coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

js-on-sx: lexer handles \uXXXX and \xXX string escapes
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 1m3s
Details

Browse Source 
read-string fell through to the literal-char branch for \u and \x,
silently stripping the backslash ("A".length returned 5 instead
of 1). Added js-hex-value helper and two cond clauses that read the
hex digits via js-peek + js-hex-digit?, compute the code point, and
emit it via char-from-code. Invalid escapes fall through to the
literal-char behaviour. built-ins/String (with --restart-every 1):
65/99 → 68/99. conformance.sh: 148/148.
This commit is contained in:
giles
2026-05-07 12:02:30 +00:00
parent 89f1c0ccbe
commit 081f934cad
4 changed files with 193 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
lib/js/lexer.sx
View File

@@ -29,6 +29,16 @@
(and (>= c "a") (<= c "f")) (and (>= c "a") (<= c "f"))
(and (>= c "A") (<= c "F"))))) (and (>= c "A") (<= c "F")))))
(define
js-hex-value
(fn
(c)
(cond
((and (>= c "0") (<= c "9")) (- (char-code c) 48))
((and (>= c "a") (<= c "f")) (- (char-code c) 87))
((and (>= c "A") (<= c "F")) (- (char-code c) 55))
(else 0))))
(define (define
js-letter? js-letter?
(fn (c) (or (and (>= c "a") (<= c "z")) (and (>= c "A") (<= c "Z"))))) (fn (c) (or (and (>= c "a") (<= c "z")) (and (>= c "A") (<= c "Z")))))
@@ -37,9 +47,9 @@
(define js-ident-char? (fn (c) (or (js-ident-start? c) (js-digit? c)))) (define js-ident-char? (fn (c) (or (js-ident-start? c) (js-digit? c))))
;; ── Reserved words ────────────────────────────────────────────────
(define js-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r")))) (define js-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
;; ── Reserved words ────────────────────────────────────────────────
(define (define
js-keywords js-keywords
(list (list
@@ -86,15 +96,18 @@
"await" "await"
"of")) "of"))
;; ── Main tokenizer ────────────────────────────────────────────────
(define js-keyword? (fn (word) (contains? js-keywords word))) (define js-keyword? (fn (word) (contains? js-keywords word)))
;; ── Main tokenizer ────────────────────────────────────────────────
(define (define
js-tokenize js-tokenize
(fn (fn
(src) (src)
(let (let
((tokens (list)) (pos 0) (src-len (len src)) (nl-before false)) ((tokens (list))
(pos 0)
(src-len (len src))
(nl-before false))
(define (define
js-peek js-peek
(fn (fn
@@ -109,7 +122,7 @@
(let (let
((sl (len s))) ((sl (len s)))
(and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s))))) (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
(define js-emit! (fn (type value start) (append! tokens {:pos start :value value :type type :nl nl-before}))) (define js-emit! (fn (type value start) (append! tokens {:nl nl-before :type type :value value :pos start})))
(define (define
skip-line-comment! skip-line-comment!
(fn (fn
@@ -256,11 +269,55 @@
((= ch "b") (append! chars "\\b")) ((= ch "b") (append! chars "\\b"))
((= ch "f") (append! chars "\\f")) ((= ch "f") (append! chars "\\f"))
((= ch "v") (append! chars "\\v")) ((= ch "v") (append! chars "\\v"))
((= ch "u")
(if
(and
(< (+ pos 4) src-len)
(js-hex-digit? (js-peek 1))
(js-hex-digit? (js-peek 2))
(js-hex-digit? (js-peek 3))
(js-hex-digit? (js-peek 4)))
(do
(append!
chars
(char-from-code
(+
(*
4096
(js-hex-value
(js-peek 1)))
(*
256
(js-hex-value
(js-peek 2)))
(*
16
(js-hex-value
(js-peek 3)))
(js-hex-value (js-peek 4)))))
(advance! 4))
(append! chars ch)))
((= ch "x")
(if
(and
(< (+ pos 2) src-len)
(js-hex-digit? (js-peek 1))
(js-hex-digit? (js-peek 2)))
(do
(append!
chars
(char-from-code
(+
(* 16 (js-hex-value (js-peek 1)))
(js-hex-value (js-peek 2)))))
(advance! 2))
(append! chars ch)))
(else (append! chars ch))) (else (append! chars ch)))
(advance! 1)))) (advance! 1))))
(loop))) (loop)))
((= (cur) quote-char) (advance! 1)) ((= (cur) quote-char) (advance! 1))
(else (do (append! chars (cur)) (advance! 1) (loop)))))) (else
(do (append! chars (cur)) (advance! 1) (loop))))))
(loop) (loop)
(join "" chars)))) (join "" chars))))
(define (define
@@ -291,7 +348,8 @@
() ()
(cond (cond
((>= pos src-len) nil) ((>= pos src-len) nil)
((and (= (cur) "}") (= depth 1)) (advance! 1)) ((and (= (cur) "}") (= depth 1))
(advance! 1))
((= (cur) "}") ((= (cur) "}")
(do (do
(append! buf (cur)) (append! buf (cur))
@@ -327,7 +385,9 @@
(advance! 1))) (advance! 1)))
(sloop))) (sloop)))
((= (cur) q) ((= (cur) q)
(do (append! buf (cur)) (advance! 1))) (do
(append! buf (cur))
(advance! 1)))
(else (else
(do (do
(append! buf (cur)) (append! buf (cur))
@@ -336,7 +396,10 @@
(sloop) (sloop)
(expr-loop)))) (expr-loop))))
(else (else
(do (append! buf (cur)) (advance! 1) (expr-loop)))))) (do
(append! buf (cur))
(advance! 1)
(expr-loop))))))
(expr-loop) (expr-loop)
(join "" buf)))) (join "" buf))))
(define (define
@@ -378,14 +441,17 @@
(else (append! chars ch))) (else (append! chars ch)))
(advance! 1)))) (advance! 1))))
(loop))) (loop)))
(else (do (append! chars (cur)) (advance! 1) (loop)))))) (else
(do (append! chars (cur)) (advance! 1) (loop))))))
(loop) (loop)
(flush-chars!) (flush-chars!)
(if (if
(= (len parts) 0) (= (len parts) 0)
"" ""
(if (if
(and (= (len parts) 1) (= (nth (nth parts 0) 0) "str")) (and
(= (len parts) 1)
(= (nth (nth parts 0) 0) "str"))
(nth (nth parts 0) 1) (nth (nth parts 0) 1)
parts))))) parts)))))
(define (define
@@ -455,9 +521,13 @@
(append! buf (cur)) (append! buf (cur))
(advance! 1) (advance! 1)
(body-loop))) (body-loop)))
((and (= (cur) "/") (not in-class)) (advance! 1)) ((and (= (cur) "/") (not in-class))
(advance! 1))
(else (else
(begin (append! buf (cur)) (advance! 1) (body-loop)))))) (begin
(append! buf (cur))
(advance! 1)
(body-loop))))))
(body-loop) (body-loop)
(let (let
((flags-buf (list))) ((flags-buf (list)))
@@ -472,7 +542,7 @@
(advance! 1) (advance! 1)
(flags-loop))))) (flags-loop)))))
(flags-loop) (flags-loop)
{:pattern (join "" buf) :flags (join "" flags-buf)})))) {:flags (join "" flags-buf) :pattern (join "" buf)}))))
(define (define
try-op-4! try-op-4!
(fn (fn
@@ -512,58 +582,104 @@
(fn (fn
(start) (start)
(cond (cond
((at? "==") (do (js-emit! "op" "==" start) (advance! 2) true)) ((at? "==")
((at? "!=") (do (js-emit! "op" "!=" start) (advance! 2) true)) (do (js-emit! "op" "==" start) (advance! 2) true))
((at? "<=") (do (js-emit! "op" "<=" start) (advance! 2) true)) ((at? "!=")
((at? ">=") (do (js-emit! "op" ">=" start) (advance! 2) true)) (do (js-emit! "op" "!=" start) (advance! 2) true))
((at? "&&") (do (js-emit! "op" "&&" start) (advance! 2) true)) ((at? "<=")
((at? "||") (do (js-emit! "op" "||" start) (advance! 2) true)) (do (js-emit! "op" "<=" start) (advance! 2) true))
((at? "??") (do (js-emit! "op" "??" start) (advance! 2) true)) ((at? ">=")
((at? "=>") (do (js-emit! "op" "=>" start) (advance! 2) true)) (do (js-emit! "op" ">=" start) (advance! 2) true))
((at? "**") (do (js-emit! "op" "**" start) (advance! 2) true)) ((at? "&&")
((at? "<<") (do (js-emit! "op" "<<" start) (advance! 2) true)) (do (js-emit! "op" "&&" start) (advance! 2) true))
((at? ">>") (do (js-emit! "op" ">>" start) (advance! 2) true)) ((at? "||")
((at? "++") (do (js-emit! "op" "++" start) (advance! 2) true)) (do (js-emit! "op" "||" start) (advance! 2) true))
((at? "--") (do (js-emit! "op" "--" start) (advance! 2) true)) ((at? "??")
((at? "+=") (do (js-emit! "op" "+=" start) (advance! 2) true)) (do (js-emit! "op" "??" start) (advance! 2) true))
((at? "-=") (do (js-emit! "op" "-=" start) (advance! 2) true)) ((at? "=>")
((at? "*=") (do (js-emit! "op" "*=" start) (advance! 2) true)) (do (js-emit! "op" "=>" start) (advance! 2) true))
((at? "/=") (do (js-emit! "op" "/=" start) (advance! 2) true)) ((at? "**")
((at? "%=") (do (js-emit! "op" "%=" start) (advance! 2) true)) (do (js-emit! "op" "**" start) (advance! 2) true))
((at? "&=") (do (js-emit! "op" "&=" start) (advance! 2) true)) ((at? "<<")
((at? "|=") (do (js-emit! "op" "|=" start) (advance! 2) true)) (do (js-emit! "op" "<<" start) (advance! 2) true))
((at? "^=") (do (js-emit! "op" "^=" start) (advance! 2) true)) ((at? ">>")
((at? "?.") (do (js-emit! "op" "?." start) (advance! 2) true)) (do (js-emit! "op" ">>" start) (advance! 2) true))
((at? "++")
(do (js-emit! "op" "++" start) (advance! 2) true))
((at? "--")
(do (js-emit! "op" "--" start) (advance! 2) true))
((at? "+=")
(do (js-emit! "op" "+=" start) (advance! 2) true))
((at? "-=")
(do (js-emit! "op" "-=" start) (advance! 2) true))
((at? "*=")
(do (js-emit! "op" "*=" start) (advance! 2) true))
((at? "/=")
(do (js-emit! "op" "/=" start) (advance! 2) true))
((at? "%=")
(do (js-emit! "op" "%=" start) (advance! 2) true))
((at? "&=")
(do (js-emit! "op" "&=" start) (advance! 2) true))
((at? "|=")
(do (js-emit! "op" "|=" start) (advance! 2) true))
((at? "^=")
(do (js-emit! "op" "^=" start) (advance! 2) true))
((at? "?.")
(do (js-emit! "op" "?." start) (advance! 2) true))
(else false)))) (else false))))
(define (define
emit-one-op! emit-one-op!
(fn (fn
(ch start) (ch start)
(cond (cond
((= ch "(") (do (js-emit! "punct" "(" start) (advance! 1))) ((= ch "(")
((= ch ")") (do (js-emit! "punct" ")" start) (advance! 1))) (do (js-emit! "punct" "(" start) (advance! 1)))
((= ch "[") (do (js-emit! "punct" "[" start) (advance! 1))) ((= ch ")")
((= ch "]") (do (js-emit! "punct" "]" start) (advance! 1))) (do (js-emit! "punct" ")" start) (advance! 1)))
((= ch "{") (do (js-emit! "punct" "{" start) (advance! 1))) ((= ch "[")
((= ch "}") (do (js-emit! "punct" "}" start) (advance! 1))) (do (js-emit! "punct" "[" start) (advance! 1)))
((= ch ",") (do (js-emit! "punct" "," start) (advance! 1))) ((= ch "]")
((= ch ";") (do (js-emit! "punct" ";" start) (advance! 1))) (do (js-emit! "punct" "]" start) (advance! 1)))
((= ch ":") (do (js-emit! "punct" ":" start) (advance! 1))) ((= ch "{")
((= ch ".") (do (js-emit! "punct" "." start) (advance! 1))) (do (js-emit! "punct" "{" start) (advance! 1)))
((= ch "?") (do (js-emit! "op" "?" start) (advance! 1))) ((= ch "}")
((= ch "+") (do (js-emit! "op" "+" start) (advance! 1))) (do (js-emit! "punct" "}" start) (advance! 1)))
((= ch "-") (do (js-emit! "op" "-" start) (advance! 1))) ((= ch ",")
((= ch "*") (do (js-emit! "op" "*" start) (advance! 1))) (do (js-emit! "punct" "," start) (advance! 1)))
((= ch "/") (do (js-emit! "op" "/" start) (advance! 1))) ((= ch ";")
((= ch "%") (do (js-emit! "op" "%" start) (advance! 1))) (do (js-emit! "punct" ";" start) (advance! 1)))
((= ch "=") (do (js-emit! "op" "=" start) (advance! 1))) ((= ch ":")
((= ch "<") (do (js-emit! "op" "<" start) (advance! 1))) (do (js-emit! "punct" ":" start) (advance! 1)))
((= ch ">") (do (js-emit! "op" ">" start) (advance! 1))) ((= ch ".")
((= ch "!") (do (js-emit! "op" "!" start) (advance! 1))) (do (js-emit! "punct" "." start) (advance! 1)))
((= ch "&") (do (js-emit! "op" "&" start) (advance! 1))) ((= ch "?")
((= ch "|") (do (js-emit! "op" "|" start) (advance! 1))) (do (js-emit! "op" "?" start) (advance! 1)))
((= ch "^") (do (js-emit! "op" "^" start) (advance! 1))) ((= ch "+")
((= ch "~") (do (js-emit! "op" "~" start) (advance! 1))) (do (js-emit! "op" "+" start) (advance! 1)))
((= ch "-")
(do (js-emit! "op" "-" start) (advance! 1)))
((= ch "*")
(do (js-emit! "op" "*" start) (advance! 1)))
((= ch "/")
(do (js-emit! "op" "/" start) (advance! 1)))
((= ch "%")
(do (js-emit! "op" "%" start) (advance! 1)))
((= ch "=")
(do (js-emit! "op" "=" start) (advance! 1)))
((= ch "<")
(do (js-emit! "op" "<" start) (advance! 1)))
((= ch ">")
(do (js-emit! "op" ">" start) (advance! 1)))
((= ch "!")
(do (js-emit! "op" "!" start) (advance! 1)))
((= ch "&")
(do (js-emit! "op" "&" start) (advance! 1)))
((= ch "|")
(do (js-emit! "op" "|" start) (advance! 1)))
((= ch "^")
(do (js-emit! "op" "^" start) (advance! 1)))
((= ch "~")
(do (js-emit! "op" "~" start) (advance! 1)))
(else (advance! 1))))) (else (advance! 1)))))
(define (define
scan! scan!

18
lib/js/test262-scoreboard.json
View File

@@ -1,9 +1,9 @@
{ {
"totals": { "totals": {
"pass": 65, "pass": 65,
"fail": 26, "fail": 28,
"skip": 1, "skip": 1,
"timeout": 8, "timeout": 6,
"total": 100, "total": 100,
"runnable": 99, "runnable": 99,
"pass_rate": 65.7 "pass_rate": 65.7
@@ -13,9 +13,9 @@
"category": "built-ins/String", "category": "built-ins/String",
"total": 100, "total": 100,
"pass": 65, "pass": 65,
"fail": 26, "fail": 28,
"skip": 1, "skip": 1,
"timeout": 8, "timeout": 6,
"pass_rate": 65.7, "pass_rate": 65.7,
"top_failures": [ "top_failures": [
[ [
@@ -23,11 +23,11 @@
16 16
], ],
[ [
"Timeout", "TypeError: not a function",
8 8
], ],
[ [
"TypeError: not a function", "Timeout",
6 6
], ],
[ [
@@ -47,11 +47,11 @@
16 16
], ],
[ [
"Timeout", "TypeError: not a function",
8 8
], ],
[ [
"TypeError: not a function", "Timeout",
6 6
], ],
[ [
@@ -68,6 +68,6 @@
] ]
], ],
"pinned_commit": "d5e73fc8d2c663554fb72e2380a8c2bc1a318a33", "pinned_commit": "d5e73fc8d2c663554fb72e2380a8c2bc1a318a33",
"elapsed_seconds": 420.4, "elapsed_seconds": 353.1,
"workers": 1 "workers": 1
} }

14
lib/js/test262-scoreboard.md
View File

@@ -1,15 +1,15 @@
# test262 scoreboard # test262 scoreboard
Pinned commit: `d5e73fc8d2c663554fb72e2380a8c2bc1a318a33` Pinned commit: `d5e73fc8d2c663554fb72e2380a8c2bc1a318a33`
Wall time: 420.4s Wall time: 353.1s
**Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=26 skip=1 timeout=8 total=100. **Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=28 skip=1 timeout=6 total=100.
## Top failure modes ## Top failure modes
- **16x** Test262Error (assertion failed) - **16x** Test262Error (assertion failed)
- **8x** Timeout - **8x** TypeError: not a function
- **6x** TypeError: not a function - **6x** Timeout
- **2x** Unhandled: Not callable: \\\ - **2x** Unhandled: Not callable: \\\
- **1x** ReferenceError (undefined symbol) - **1x** ReferenceError (undefined symbol)
- **1x** SyntaxError (parse/unsupported syntax) - **1x** SyntaxError (parse/unsupported syntax)
@@ -18,14 +18,14 @@ Wall time: 420.4s
| Category | Pass | Fail | Skip | Timeout | Total | Pass % | | Category | Pass | Fail | Skip | Timeout | Total | Pass % |
|---|---:|---:|---:|---:|---:|---:| |---|---:|---:|---:|---:|---:|---:|
| built-ins/String | 65 | 26 | 1 | 8 | 100 | 65.7% | | built-ins/String | 65 | 28 | 1 | 6 | 100 | 65.7% |
## Per-category top failures (min 10 runnable, worst first) ## Per-category top failures (min 10 runnable, worst first)
### built-ins/String (65/99 — 65.7%) ### built-ins/String (65/99 — 65.7%)
- **16x** Test262Error (assertion failed) - **16x** Test262Error (assertion failed)
- **8x** Timeout - **8x** TypeError: not a function
- **6x** TypeError: not a function - **6x** Timeout
- **2x** Unhandled: Not callable: \\\ - **2x** Unhandled: Not callable: \\\
- **1x** ReferenceError (undefined symbol) - **1x** ReferenceError (undefined symbol)

2
plans/js-on-sx.md
View File

@@ -158,6 +158,8 @@ Each item: implement → tests → update progress. Mark `[x]` when tests green.
Append-only record of completed iterations. Loop writes one line per iteration: date, what was done, test count delta. Append-only record of completed iterations. Loop writes one line per iteration: date, what was done, test count delta.
- 2026-05-07 — **JS lexer: handle `\uXXXX` and `\xXX` escape sequences in string literals.** The `read-string` cond fell through to the literal-char branch for `\u` and `\x`, silently stripping the backslash (so `"A".length` returned 5 instead of 1). Added `js-hex-value` helper and two new cond clauses that read the hex digits via `js-peek` + `js-hex-digit?`, compute the code point, and emit it via `char-from-code`. Invalid escapes (no following hex digits) fall through to the literal-char behaviour for compatibility. With test isolation (`--restart-every 1`) built-ins/String 65/99 → 68/99. Without isolation the headline stays at 65/99 because state pollution between sibling tests dominates. conformance.sh: 148/148.
- 2026-05-07 — **Bump test262 runner default per-test timeout 5s→15s.** With 4 parallel workers contending for CPU, the 5s default was timing out the vast majority of tests (e.g. 85/99 on built-ins/String). Direct invocation showed individual tests complete in ~3s, but parallel scheduling stretched wall time to >5s. Bumping to 15s makes the scoreboard usable: built-ins/String 14.1% → 65.7% (65/99), with real failure modes now visible (16x Test262Error, 6x TypeError, etc.) instead of "85x Timeout" drowning the signal. Regenerated scoreboard to reflect the new state. conformance.sh: 148/148. - 2026-05-07 — **Bump test262 runner default per-test timeout 5s→15s.** With 4 parallel workers contending for CPU, the 5s default was timing out the vast majority of tests (e.g. 85/99 on built-ins/String). Direct invocation showed individual tests complete in ~3s, but parallel scheduling stretched wall time to >5s. Bumping to 15s makes the scoreboard usable: built-ins/String 14.1% → 65.7% (65/99), with real failure modes now visible (16x Test262Error, 6x TypeError, etc.) instead of "85x Timeout" drowning the signal. Regenerated scoreboard to reflect the new state. conformance.sh: 148/148.
- 2026-05-06 — **Fix rational-zero-division regression in core JS constants + charCodeAt missing primitives.** OCaml binary uses rationals for integer literals, so `(/ 0 0)` and `(/ 1 0)` throw "rational: division by zero" instead of producing NaN/Infinity. Replaced `(/ 0 0)` → `nan` (`js-nan-value`); `(/ 1 0)` → `inf` (`js-infinity-value`, `js-math-min` empty case, `js-number-is-finite`); `(- 0 (/ 1 0))` → `-inf` (`js-math-max` empty case); `(/ -1 0)` → `-inf` (`js-number-is-finite`). `js-max-value-approx` was looping forever (rationals never reach float infinity) — replaced with literal `1.7976931348623157e+308`. Fixed `charCodeAt` and string `.length` to use `(len s)` and `(char-code (char-at s idx))` instead of missing `unicode-len`/`unicode-char-code-at` primitives. conformance.sh: 0→148/148. Unit tests: 521/530 best run (baseline run was 417/530; both timeout-flaky). - 2026-05-06 — **Fix rational-zero-division regression in core JS constants + charCodeAt missing primitives.** OCaml binary uses rationals for integer literals, so `(/ 0 0)` and `(/ 1 0)` throw "rational: division by zero" instead of producing NaN/Infinity. Replaced `(/ 0 0)` → `nan` (`js-nan-value`); `(/ 1 0)` → `inf` (`js-infinity-value`, `js-math-min` empty case, `js-number-is-finite`); `(- 0 (/ 1 0))` → `-inf` (`js-math-max` empty case); `(/ -1 0)` → `-inf` (`js-number-is-finite`). `js-max-value-approx` was looping forever (rationals never reach float infinity) — replaced with literal `1.7976931348623157e+308`. Fixed `charCodeAt` and string `.length` to use `(len s)` and `(char-code (char-at s idx))` instead of missing `unicode-len`/`unicode-char-code-at` primitives. conformance.sh: 0→148/148. Unit tests: 521/530 best run (baseline run was 417/530; both timeout-flaky).