diff --git a/lib/datalog/scoreboard.json b/lib/datalog/scoreboard.json index ba1017d8..c8a61bf7 100644 --- a/lib/datalog/scoreboard.json +++ b/lib/datalog/scoreboard.json @@ -1,10 +1,10 @@ { "lang": "datalog", - "total_passed": 255, + "total_passed": 256, "total_failed": 0, - "total": 255, + "total": 256, "suites": [ - {"name":"tokenize","passed":29,"failed":0,"total":29}, + {"name":"tokenize","passed":30,"failed":0,"total":30}, {"name":"parse","passed":22,"failed":0,"total":22}, {"name":"unify","passed":28,"failed":0,"total":28}, {"name":"eval","passed":38,"failed":0,"total":38}, @@ -16,5 +16,5 @@ {"name":"magic","passed":36,"failed":0,"total":36}, {"name":"demo","passed":21,"failed":0,"total":21} ], - "generated": "2026-05-10T21:13:14+00:00" + "generated": "2026-05-10T21:16:53+00:00" } diff --git a/lib/datalog/scoreboard.md b/lib/datalog/scoreboard.md index a55a1dbb..1055a8f9 100644 --- a/lib/datalog/scoreboard.md +++ b/lib/datalog/scoreboard.md @@ -1,10 +1,10 @@ # datalog scoreboard -**255 / 255 passing** (0 failure(s)). +**256 / 256 passing** (0 failure(s)). | Suite | Passed | Total | Status | |-------|--------|-------|--------| -| tokenize | 29 | 29 | ok | +| tokenize | 30 | 30 | ok | | parse | 22 | 22 | ok | | unify | 28 | 28 | ok | | eval | 38 | 38 | ok | diff --git a/lib/datalog/tests/tokenize.sx b/lib/datalog/tests/tokenize.sx index 6bf190fc..51470a44 100644 --- a/lib/datalog/tests/tokenize.sx +++ b/lib/datalog/tests/tokenize.sx @@ -118,6 +118,18 @@ "block comment" (dl-tk-types (dl-tokenize "/* a\nb */ x.")) (list "atom" "punct" "eof")) + ;; Unexpected characters surface at tokenize time rather + ;; than being silently consumed (previously `?(X)` parsed as + ;; if the leading `?` weren't there). + (dl-tk-test! + "unexpected char raises" + (let ((threw false)) + (do + (guard (e (#t (set! threw true))) + (dl-tokenize "?(X)")) + threw)) + true) + ;; Unterminated string / quoted-atom must raise. (dl-tk-test! "unterminated string raises" diff --git a/lib/datalog/tokenizer.sx b/lib/datalog/tokenizer.sx index e48ca7df..a10a7ff7 100644 --- a/lib/datalog/tokenizer.sx +++ b/lib/datalog/tokenizer.sx @@ -254,7 +254,9 @@ (dl-emit! "op" "/" start) (advance! 1) (scan!))) - (else (do (advance! 1) (scan!))))))))) + (else (error + (str "Tokenizer: unexpected character '" ch + "' at position " start))))))))) (scan!) (dl-emit! "eof" nil pos) tokens)))