# setup tinytest for checkmate functionality
library("tinytest")
library("checkmate")
using("checkmate")

# ------ fixtures ------

docs_named <- list(
  "A" = "The quick brown fox jumps over the lazy lazy dog.",
  "B" = c("Pack my brown box.", NA, "Jazz fly brown dog."),
  "C" = NA_character_
)

docs_unnamed <- c(
  "The quick brown fox jumps over the lazy lazy dog.",
  "Pack my brown box.",
  "Jazz fly brown dog."
)

d_named <- mine_text(docs_named)
d_unnamed <- mine_text(docs_unnamed)

# ------ return structure ------

# result is a data.frame
expect_true(is.data.frame(d_named))

# has exactly the three expected columns
expect_equal(colnames(d_named), c("term", "pub_id", "freq"))

# term and pub_id are factors, freq is integer
expect_true(is.factor(d_named$term))
expect_true(is.factor(d_named$pub_id))
expect_true(is.integer(d_named$freq))

# no zero-frequency rows
expect_true(all(d_named$freq > 0L))

# no NA values in any column
expect_false(anyNA(d_named))

# ------ pub_id assignment ------

# named docs: pub_ids match the list names (excluding all-NA doc "C")
expect_true(all(levels(d_named$pub_id) %in% c("A", "B", "C")))
expect_false("C" %in% as.character(d_named$pub_id)) # all-NA produces no rows

# unnamed docs: pub_ids are sequential integers as character
expect_true(all(levels(d_unnamed$pub_id) %in% c("1", "2", "3")))

# ------ term counts ------

# "lazy" appears twice in doc "A", so its freq should be 2
lazy_a <- d_named[d_named$term == "lazy" & d_named$pub_id == "A", "freq"]
expect_equal(lazy_a, 2L)

# "brown" appears in docs "A" and "B"
brown_pubs <- as.character(d_named$pub_id[d_named$term == "brown"])
expect_true("A" %in% brown_pubs)
expect_true("B" %in% brown_pubs)

# ------ stopword removal ------

# common English stopwords must not appear as terms
sw <- c("the", "my", "over")
expect_false(any(sw %in% as.character(d_named$term)))

# ------ number removal ------

d_nums <- mine_text(list("x" = "There are 3 cats and 42 dogs."))
expect_false(any(grepl("[0-9]", as.character(d_nums$term))))

# ------ URL removal ------

d_url <- mine_text(list("x" = "Visit https://example.com for details water quality."))
expect_false(any(grepl("http", as.character(d_url$term))))

# ------ sort order ------

# terms must be in ascending alphabetical order within each pub_id
terms_a <- as.character(d_named$term[d_named$pub_id == "A"])
expect_equal(terms_a, sort(terms_a))

# for a term appearing in multiple docs, rows with the same term must be
# sorted by descending frequency
if (nrow(d_named) > 1) {
  for (trm in unique(as.character(d_named$term))) {
    freqs <- d_named$freq[d_named$term == trm]
    expect_true(all(diff(freqs) <= 0L))
  }
}

# ------ all-NA document produces no rows ------

d_allna <- mine_text(list("only" = NA_character_))
expect_equal(nrow(d_allna), 0L)

# ------ sparse argument ------

# with sparse = 0.5, terms appearing in fewer than half the docs are removed;
# "brown" appears in 2 of 3 docs so it should survive
d_sparse <- mine_text(docs_unnamed, sparse = 0.5)
expect_true("brown" %in% as.character(d_sparse$term))

# with a very low sparse threshold (term must appear in nearly every doc),
# a term that appears in only one doc should be removed
d_strict <- mine_text(docs_unnamed, sparse = 0.01)
# "box" only appears in doc 2, so it must not survive
expect_false("box" %in% as.character(d_strict$term))

# ------ invalid argument errors ------

expect_error(mine_text(123))
expect_error(mine_text("text", ngmin = 0L))
expect_error(mine_text("text", ngmin = 2L, ngmax = 1L))
expect_error(mine_text("text", sparse = 0))
expect_error(mine_text("text", sparse = 1))
