coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

search: Phase 2 query AST + boolean/phrase eval + 28 tests
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 53s
Details

Browse Source 
Query ADT (Term|And|Or|Not|Phrase) and evalQuery over docid-sorted posting
lists: boolean ops as linear merges, Not over the allDocs universe, Phrase via
positional adjacency. Batched both test suites into one program eval each
(search-batch) so they finish under heavy CPU load. 46/46.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
giles
2026-06-06 18:47:42 +00:00
parent b8cf3eb1b8
commit 0f0da0319c
9 changed files with 264 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
lib/search/tests/index.sx
View File

@@ -1,119 +1,88 @@
;; Phase 1 — tokenize + inverted index.
;; All cases run in ONE program evaluation (search-batch) to stay fast under load.
;; Scalar results (docFreq) are wrapped as singleton lists so the batch is a list
;; of lists.
(hk-test
"tokens basic lowercases"
(search-eval "\nresult = tokens \"The Cat sat\"\n" "result")
(list "the" "cat" "sat"))
(hk-test
"tokens strips punctuation"
(search-eval "\nresult = tokens \"Hello, World!\"\n" "result")
(list "hello" "world"))
(hk-test
"tokens collapses whitespace"
(search-eval "\nresult = tokens \" a b \"\n" "result")
(list "a" "b"))
(hk-test
"tokens empty is empty"
(search-eval "\nresult = tokens \"\"\n" "result")
(list))
(hk-test
"tokens keeps digits"
(search-eval "\nresult = tokens \"abc123 x9\"\n" "result")
(list "abc123" "x9"))
(hk-test
"positioned attaches ordinals"
(search-eval "\nresult = positioned \"a b a\"\n" "result")
(list (list "a" 0) (list "b" 1) (list "a" 2)))
(hk-test
"index + lookup single doc"
(search-eval
"\nresult = lookupTerm \"cat\" (indexDoc 1 \"the cat sat\" emptyIndex)\n"
"result")
(list (list 1 (list 1))))
(hk-test
"lookup missing term is empty"
(search-eval
"\nresult = lookupTerm \"dog\" (indexDoc 1 \"the cat sat\" emptyIndex)\n"
"result")
(list))
(hk-test
"lookup records all positions"
(search-eval
"\nresult = lookupTerm \"the\" (indexDoc 1 \"the cat the dog the\" emptyIndex)\n"
"result")
(list (list 1 (list 0 2 4))))
(hk-test
"multi-doc posting list sorted by docid"
(search-eval
"\nresult = lookupTerm \"x\" (indexDoc 1 \"x y\" (indexDoc 2 \"x z\" emptyIndex))\n"
"result")
(define
index-cases
(list
(list 1 (list 0))
(list 2 (list 0))))
(list
"tokens basic lowercases"
"tokens \"The Cat sat\""
(list "the" "cat" "sat"))
(list
"tokens strips punctuation"
"tokens \"Hello, World!\""
(list "hello" "world"))
(list "tokens collapses whitespace" "tokens \" a b \"" (list "a" "b"))
(list "tokens empty is empty" "tokens \"\"" (list))
(list "tokens keeps digits" "tokens \"abc123 x9\"" (list "abc123" "x9"))
(list
"positioned attaches ordinals"
"positioned \"a b a\""
(list
(list "a" 0)
(list "b" 1)
(list "a" 2)))
(list
"index + lookup single doc"
"lookupTerm \"cat\" (indexDoc 1 \"the cat sat\" emptyIndex)"
(list (list 1 (list 1))))
(list
"lookup missing term is empty"
"lookupTerm \"dog\" (indexDoc 1 \"the cat sat\" emptyIndex)"
(list))
(list
"lookup records all positions"
"lookupTerm \"the\" (indexDoc 1 \"the cat the dog the\" emptyIndex)"
(list (list 1 (list 0 2 4))))
(list
"multi-doc posting list sorted by docid"
"lookupTerm \"x\" (indexDoc 1 \"x y\" (indexDoc 2 \"x z\" emptyIndex))"
(list
(list 1 (list 0))
(list 2 (list 0))))
(list
"index/query case symmetry"
"lookupTerm \"cat\" (indexDoc 1 \"CAT Cat cat\" emptyIndex)"
(list (list 1 (list 0 1 2))))
(list
"re-index replaces a doc"
"lookupTerm \"a\" (indexDoc 1 \"a a a\" (indexDoc 1 \"a\" emptyIndex))"
(list (list 1 (list 0 1 2))))
(list
"delete removes a doc"
"lookupTerm \"cat\" (deleteDoc 1 (indexDoc 1 \"the cat\" emptyIndex))"
(list))
(list
"delete leaves other docs"
"lookupTerm \"cat\" (deleteDoc 2 (indexDoc 2 \"big cat\" (indexDoc 1 \"the cat\" emptyIndex)))"
(list (list 1 (list 1))))
(list
"docFreq counts docs"
"[docFreq \"cat\" (indexDoc 2 \"a cat\" (indexDoc 1 \"the cat\" emptyIndex))]"
(list 2))
(list
"docFreq zero for missing"
"[docFreq \"zzz\" (indexDoc 1 \"a b\" emptyIndex)]"
(list 0))
(list
"allTerms sorted and unique"
"allTerms (indexDoc 1 \"banana apple cherry apple\" emptyIndex)"
(list "apple" "banana" "cherry"))
(list
"allTerms merged across docs"
"allTerms (indexDoc 2 \"d a\" (indexDoc 1 \"c b\" emptyIndex))"
(list "a" "b" "c" "d"))))
(hk-test
"index/query case symmetry"
(search-eval
"\nresult = lookupTerm \"cat\" (indexDoc 1 \"CAT Cat cat\" emptyIndex)\n"
"result")
(list (list 1 (list 0 1 2))))
(define
index-results
(search-batch "" (map (fn (c) (nth c 1)) index-cases)))
(hk-test
"re-index replaces a doc"
(search-eval
"\nresult = lookupTerm \"a\" (indexDoc 1 \"a a a\" (indexDoc 1 \"a\" emptyIndex))\n"
"result")
(list (list 1 (list 0 1 2))))
(hk-test
"delete removes a doc"
(search-eval
"\nresult = lookupTerm \"cat\" (deleteDoc 1 (indexDoc 1 \"the cat\" emptyIndex))\n"
"result")
(list))
(hk-test
"delete leaves other docs"
(search-eval
"\nresult = lookupTerm \"cat\" (deleteDoc 2 (indexDoc 2 \"big cat\" (indexDoc 1 \"the cat\" emptyIndex)))\n"
"result")
(list (list 1 (list 1))))
(hk-test
"docFreq counts docs"
(search-eval
"\nresult = docFreq \"cat\" (indexDoc 2 \"a cat\" (indexDoc 1 \"the cat\" emptyIndex))\n"
"result")
2)
(hk-test
"docFreq zero for missing"
(search-eval
"\nresult = docFreq \"zzz\" (indexDoc 1 \"a b\" emptyIndex)\n"
"result")
0)
(hk-test
"allTerms sorted and unique"
(search-eval
"\nresult = allTerms (indexDoc 1 \"banana apple cherry apple\" emptyIndex)\n"
"result")
(list "apple" "banana" "cherry"))
(hk-test
"allTerms merged across docs"
(search-eval
"\nresult = allTerms (indexDoc 2 \"d a\" (indexDoc 1 \"c b\" emptyIndex))\n"
"result")
(list "a" "b" "c" "d"))
(map-indexed
(fn
(i c)
(hk-test (nth c 0) (nth index-results i) (nth c 2)))
index-cases)
{:fail hk-test-fail :pass hk-test-pass :fails hk-test-fails}