;; Phase 1 — tokenize + inverted index.

(hk-test
  "tokens basic lowercases"
  (search-eval "\nresult = tokens \"The Cat sat\"\n" "result")
  (list "the" "cat" "sat"))

(hk-test
  "tokens strips punctuation"
  (search-eval "\nresult = tokens \"Hello, World!\"\n" "result")
  (list "hello" "world"))

(hk-test
  "tokens collapses whitespace"
  (search-eval "\nresult = tokens \"  a   b  \"\n" "result")
  (list "a" "b"))

(hk-test
  "tokens empty is empty"
  (search-eval "\nresult = tokens \"\"\n" "result")
  (list))

(hk-test
  "tokens keeps digits"
  (search-eval "\nresult = tokens \"abc123 x9\"\n" "result")
  (list "abc123" "x9"))

(hk-test
  "positioned attaches ordinals"
  (search-eval "\nresult = positioned \"a b a\"\n" "result")
  (list (list "a" 0) (list "b" 1) (list "a" 2)))

(hk-test
  "index + lookup single doc"
  (search-eval
    "\nresult = lookupTerm \"cat\" (indexDoc 1 \"the cat sat\" emptyIndex)\n"
    "result")
  (list (list 1 (list 1))))

(hk-test
  "lookup missing term is empty"
  (search-eval
    "\nresult = lookupTerm \"dog\" (indexDoc 1 \"the cat sat\" emptyIndex)\n"
    "result")
  (list))

(hk-test
  "lookup records all positions"
  (search-eval
    "\nresult = lookupTerm \"the\" (indexDoc 1 \"the cat the dog the\" emptyIndex)\n"
    "result")
  (list (list 1 (list 0 2 4))))

(hk-test
  "multi-doc posting list sorted by docid"
  (search-eval
    "\nresult = lookupTerm \"x\" (indexDoc 1 \"x y\" (indexDoc 2 \"x z\" emptyIndex))\n"
    "result")
  (list
    (list 1 (list 0))
    (list 2 (list 0))))

(hk-test
  "index/query case symmetry"
  (search-eval
    "\nresult = lookupTerm \"cat\" (indexDoc 1 \"CAT Cat cat\" emptyIndex)\n"
    "result")
  (list (list 1 (list 0 1 2))))

(hk-test
  "re-index replaces a doc"
  (search-eval
    "\nresult = lookupTerm \"a\" (indexDoc 1 \"a a a\" (indexDoc 1 \"a\" emptyIndex))\n"
    "result")
  (list (list 1 (list 0 1 2))))

(hk-test
  "delete removes a doc"
  (search-eval
    "\nresult = lookupTerm \"cat\" (deleteDoc 1 (indexDoc 1 \"the cat\" emptyIndex))\n"
    "result")
  (list))

(hk-test
  "delete leaves other docs"
  (search-eval
    "\nresult = lookupTerm \"cat\" (deleteDoc 2 (indexDoc 2 \"big cat\" (indexDoc 1 \"the cat\" emptyIndex)))\n"
    "result")
  (list (list 1 (list 1))))

(hk-test
  "docFreq counts docs"
  (search-eval
    "\nresult = docFreq \"cat\" (indexDoc 2 \"a cat\" (indexDoc 1 \"the cat\" emptyIndex))\n"
    "result")
  2)

(hk-test
  "docFreq zero for missing"
  (search-eval
    "\nresult = docFreq \"zzz\" (indexDoc 1 \"a b\" emptyIndex)\n"
    "result")
  0)

(hk-test
  "allTerms sorted and unique"
  (search-eval
    "\nresult = allTerms (indexDoc 1 \"banana apple cherry apple\" emptyIndex)\n"
    "result")
  (list "apple" "banana" "cherry"))

(hk-test
  "allTerms merged across docs"
  (search-eval
    "\nresult = allTerms (indexDoc 2 \"d a\" (indexDoc 1 \"c b\" emptyIndex))\n"
    "result")
  (list "a" "b" "c" "d"))

{:fail hk-test-fail :pass hk-test-pass :fails hk-test-fails}