datalog: tokenizer raises on unterminated string + quoted atom (251/251)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 35s
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 35s
Bug: read-quoted ran to EOF silently when the closing quote was
missing. The token's value was whatever ran-to-end string had been
accumulated; the parser later saw an unexpected EOF, but the error
message blamed the wrong location ("expected `)` got eof") and
hid the real problem.
Fix: read-quoted now raises with a message that distinguishes
strings from quoted atoms, including the position where the
opening quote was lost. The escape-sequence handling and proper
closing are unaffected.
2 new tokenize tests.
This commit is contained in:
@@ -1,10 +1,10 @@
|
|||||||
{
|
{
|
||||||
"lang": "datalog",
|
"lang": "datalog",
|
||||||
"total_passed": 249,
|
"total_passed": 251,
|
||||||
"total_failed": 0,
|
"total_failed": 0,
|
||||||
"total": 249,
|
"total": 251,
|
||||||
"suites": [
|
"suites": [
|
||||||
{"name":"tokenize","passed":27,"failed":0,"total":27},
|
{"name":"tokenize","passed":29,"failed":0,"total":29},
|
||||||
{"name":"parse","passed":22,"failed":0,"total":22},
|
{"name":"parse","passed":22,"failed":0,"total":22},
|
||||||
{"name":"unify","passed":28,"failed":0,"total":28},
|
{"name":"unify","passed":28,"failed":0,"total":28},
|
||||||
{"name":"eval","passed":36,"failed":0,"total":36},
|
{"name":"eval","passed":36,"failed":0,"total":36},
|
||||||
@@ -16,5 +16,5 @@
|
|||||||
{"name":"magic","passed":34,"failed":0,"total":34},
|
{"name":"magic","passed":34,"failed":0,"total":34},
|
||||||
{"name":"demo","passed":21,"failed":0,"total":21}
|
{"name":"demo","passed":21,"failed":0,"total":21}
|
||||||
],
|
],
|
||||||
"generated": "2026-05-10T20:59:17+00:00"
|
"generated": "2026-05-10T21:05:12+00:00"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
# datalog scoreboard
|
# datalog scoreboard
|
||||||
|
|
||||||
**249 / 249 passing** (0 failure(s)).
|
**251 / 251 passing** (0 failure(s)).
|
||||||
|
|
||||||
| Suite | Passed | Total | Status |
|
| Suite | Passed | Total | Status |
|
||||||
|-------|--------|-------|--------|
|
|-------|--------|-------|--------|
|
||||||
| tokenize | 27 | 27 | ok |
|
| tokenize | 29 | 29 | ok |
|
||||||
| parse | 22 | 22 | ok |
|
| parse | 22 | 22 | ok |
|
||||||
| unify | 28 | 28 | ok |
|
| unify | 28 | 28 | ok |
|
||||||
| eval | 36 | 36 | ok |
|
| eval | 36 | 36 | ok |
|
||||||
|
|||||||
@@ -118,6 +118,25 @@
|
|||||||
"block comment"
|
"block comment"
|
||||||
(dl-tk-types (dl-tokenize "/* a\nb */ x."))
|
(dl-tk-types (dl-tokenize "/* a\nb */ x."))
|
||||||
(list "atom" "punct" "eof"))
|
(list "atom" "punct" "eof"))
|
||||||
|
;; Unterminated string / quoted-atom must raise.
|
||||||
|
(dl-tk-test!
|
||||||
|
"unterminated string raises"
|
||||||
|
(let ((threw false))
|
||||||
|
(do
|
||||||
|
(guard (e (#t (set! threw true)))
|
||||||
|
(dl-tokenize "\"unclosed"))
|
||||||
|
threw))
|
||||||
|
true)
|
||||||
|
|
||||||
|
(dl-tk-test!
|
||||||
|
"unterminated quoted atom raises"
|
||||||
|
(let ((threw false))
|
||||||
|
(do
|
||||||
|
(guard (e (#t (set! threw true)))
|
||||||
|
(dl-tokenize "'unclosed"))
|
||||||
|
threw))
|
||||||
|
true)
|
||||||
|
|
||||||
;; Unterminated block comment must raise — previously it was
|
;; Unterminated block comment must raise — previously it was
|
||||||
;; silently consumed to EOF.
|
;; silently consumed to EOF.
|
||||||
(dl-tk-test!
|
(dl-tk-test!
|
||||||
|
|||||||
@@ -123,7 +123,11 @@
|
|||||||
(fn
|
(fn
|
||||||
()
|
()
|
||||||
(cond
|
(cond
|
||||||
((>= pos src-len) nil)
|
((>= pos src-len)
|
||||||
|
(error
|
||||||
|
(str "Tokenizer: unterminated "
|
||||||
|
(if (= quote-char "'") "quoted atom" "string")
|
||||||
|
" (started near position " pos ")")))
|
||||||
((= (cur) "\\")
|
((= (cur) "\\")
|
||||||
(do
|
(do
|
||||||
(advance! 1)
|
(advance! 1)
|
||||||
|
|||||||
Reference in New Issue
Block a user