datalog: tokenizer raises on unterminated block comment (249/249)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 43s

Bug: `/* unclosed` was silently consumed to EOF, swallowing any
Datalog code that followed inside the (never-closing) comment.
Programs would produce empty parses with no error.

Fix: skip-block-comment! now raises when it hits EOF without
finding `*/`. Error message includes the position where the
problem was first detected. Line comments (`%`) and properly
closed block comments (`/* ... */`) are unaffected.

1 new tokenize test verifies the error path.
This commit is contained in:
2026-05-10 20:59:33 +00:00
parent 2a1d8eeab2
commit 4b5e75dc3e
4 changed files with 19 additions and 7 deletions

View File

@@ -1,10 +1,10 @@
{ {
"lang": "datalog", "lang": "datalog",
"total_passed": 248, "total_passed": 249,
"total_failed": 0, "total_failed": 0,
"total": 248, "total": 249,
"suites": [ "suites": [
{"name":"tokenize","passed":26,"failed":0,"total":26}, {"name":"tokenize","passed":27,"failed":0,"total":27},
{"name":"parse","passed":22,"failed":0,"total":22}, {"name":"parse","passed":22,"failed":0,"total":22},
{"name":"unify","passed":28,"failed":0,"total":28}, {"name":"unify","passed":28,"failed":0,"total":28},
{"name":"eval","passed":36,"failed":0,"total":36}, {"name":"eval","passed":36,"failed":0,"total":36},
@@ -16,5 +16,5 @@
{"name":"magic","passed":34,"failed":0,"total":34}, {"name":"magic","passed":34,"failed":0,"total":34},
{"name":"demo","passed":21,"failed":0,"total":21} {"name":"demo","passed":21,"failed":0,"total":21}
], ],
"generated": "2026-05-10T20:55:23+00:00" "generated": "2026-05-10T20:59:17+00:00"
} }

View File

@@ -1,10 +1,10 @@
# datalog scoreboard # datalog scoreboard
**248 / 248 passing** (0 failure(s)). **249 / 249 passing** (0 failure(s)).
| Suite | Passed | Total | Status | | Suite | Passed | Total | Status |
|-------|--------|-------|--------| |-------|--------|-------|--------|
| tokenize | 26 | 26 | ok | | tokenize | 27 | 27 | ok |
| parse | 22 | 22 | ok | | parse | 22 | 22 | ok |
| unify | 28 | 28 | ok | | unify | 28 | 28 | ok |
| eval | 36 | 36 | ok | | eval | 36 | 36 | ok |

View File

@@ -118,6 +118,16 @@
"block comment" "block comment"
(dl-tk-types (dl-tokenize "/* a\nb */ x.")) (dl-tk-types (dl-tokenize "/* a\nb */ x."))
(list "atom" "punct" "eof")) (list "atom" "punct" "eof"))
;; Unterminated block comment must raise — previously it was
;; silently consumed to EOF.
(dl-tk-test!
"unterminated block comment raises"
(let ((threw false))
(do
(guard (e (#t (set! threw true)))
(dl-tokenize "/* unclosed comment"))
threw))
true)
(dl-tk-test! (dl-tk-test!
"whitespace" "whitespace"
(dl-tk-types (dl-tokenize " foo ,\t bar .")) (dl-tk-types (dl-tokenize " foo ,\t bar ."))

View File

@@ -63,7 +63,9 @@
(fn (fn
() ()
(cond (cond
((>= pos src-len) nil) ((>= pos src-len)
(error (str "Tokenizer: unterminated block comment "
"(started at position " pos ")")))
((and (= (cur) "*") (< (+ pos 1) src-len) (= (dl-peek 1) "/")) ((and (= (cur) "*") (< (+ pos 1) src-len) (= (dl-peek 1) "/"))
(advance! 2)) (advance! 2))
(else (do (advance! 1) (skip-block-comment!)))))) (else (do (advance! 1) (skip-block-comment!))))))