diff --git a/lib/js/lexer.sx b/lib/js/lexer.sx index abf28b75..13f82904 100644 --- a/lib/js/lexer.sx +++ b/lib/js/lexer.sx @@ -29,6 +29,16 @@ (and (>= c "a") (<= c "f")) (and (>= c "A") (<= c "F"))))) +(define + js-hex-value + (fn + (c) + (cond + ((and (>= c "0") (<= c "9")) (- (char-code c) 48)) + ((and (>= c "a") (<= c "f")) (- (char-code c) 87)) + ((and (>= c "A") (<= c "F")) (- (char-code c) 55)) + (else 0)))) + (define js-letter? (fn (c) (or (and (>= c "a") (<= c "z")) (and (>= c "A") (<= c "Z"))))) @@ -37,9 +47,9 @@ (define js-ident-char? (fn (c) (or (js-ident-start? c) (js-digit? c)))) +;; ── Reserved words ──────────────────────────────────────────────── (define js-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r")))) -;; ── Reserved words ──────────────────────────────────────────────── (define js-keywords (list @@ -86,15 +96,18 @@ "await" "of")) +;; ── Main tokenizer ──────────────────────────────────────────────── (define js-keyword? (fn (word) (contains? js-keywords word))) -;; ── Main tokenizer ──────────────────────────────────────────────── (define js-tokenize (fn (src) (let - ((tokens (list)) (pos 0) (src-len (len src)) (nl-before false)) + ((tokens (list)) + (pos 0) + (src-len (len src)) + (nl-before false)) (define js-peek (fn @@ -109,7 +122,7 @@ (let ((sl (len s))) (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s))))) - (define js-emit! (fn (type value start) (append! tokens {:pos start :value value :type type :nl nl-before}))) + (define js-emit! (fn (type value start) (append! tokens {:nl nl-before :type type :value value :pos start}))) (define skip-line-comment! (fn @@ -256,11 +269,55 @@ ((= ch "b") (append! chars "\\b")) ((= ch "f") (append! chars "\\f")) ((= ch "v") (append! chars "\\v")) + ((= ch "u") + (if + (and + (< (+ pos 4) src-len) + (js-hex-digit? (js-peek 1)) + (js-hex-digit? (js-peek 2)) + (js-hex-digit? (js-peek 3)) + (js-hex-digit? (js-peek 4))) + (do + (append! + chars + (char-from-code + (+ + (* + 4096 + (js-hex-value + (js-peek 1))) + (* + 256 + (js-hex-value + (js-peek 2))) + (* + 16 + (js-hex-value + (js-peek 3))) + (js-hex-value (js-peek 4))))) + (advance! 4)) + (append! chars ch))) + ((= ch "x") + (if + (and + (< (+ pos 2) src-len) + (js-hex-digit? (js-peek 1)) + (js-hex-digit? (js-peek 2))) + (do + (append! + chars + (char-from-code + (+ + (* 16 (js-hex-value (js-peek 1))) + (js-hex-value (js-peek 2))))) + (advance! 2)) + (append! chars ch))) (else (append! chars ch))) (advance! 1)))) (loop))) ((= (cur) quote-char) (advance! 1)) - (else (do (append! chars (cur)) (advance! 1) (loop)))))) + (else + (do (append! chars (cur)) (advance! 1) (loop)))))) (loop) (join "" chars)))) (define @@ -291,7 +348,8 @@ () (cond ((>= pos src-len) nil) - ((and (= (cur) "}") (= depth 1)) (advance! 1)) + ((and (= (cur) "}") (= depth 1)) + (advance! 1)) ((= (cur) "}") (do (append! buf (cur)) @@ -327,7 +385,9 @@ (advance! 1))) (sloop))) ((= (cur) q) - (do (append! buf (cur)) (advance! 1))) + (do + (append! buf (cur)) + (advance! 1))) (else (do (append! buf (cur)) @@ -336,7 +396,10 @@ (sloop) (expr-loop)))) (else - (do (append! buf (cur)) (advance! 1) (expr-loop)))))) + (do + (append! buf (cur)) + (advance! 1) + (expr-loop)))))) (expr-loop) (join "" buf)))) (define @@ -378,14 +441,17 @@ (else (append! chars ch))) (advance! 1)))) (loop))) - (else (do (append! chars (cur)) (advance! 1) (loop)))))) + (else + (do (append! chars (cur)) (advance! 1) (loop)))))) (loop) (flush-chars!) (if (= (len parts) 0) "" (if - (and (= (len parts) 1) (= (nth (nth parts 0) 0) "str")) + (and + (= (len parts) 1) + (= (nth (nth parts 0) 0) "str")) (nth (nth parts 0) 1) parts))))) (define @@ -455,9 +521,13 @@ (append! buf (cur)) (advance! 1) (body-loop))) - ((and (= (cur) "/") (not in-class)) (advance! 1)) + ((and (= (cur) "/") (not in-class)) + (advance! 1)) (else - (begin (append! buf (cur)) (advance! 1) (body-loop)))))) + (begin + (append! buf (cur)) + (advance! 1) + (body-loop)))))) (body-loop) (let ((flags-buf (list))) @@ -472,7 +542,7 @@ (advance! 1) (flags-loop))))) (flags-loop) - {:pattern (join "" buf) :flags (join "" flags-buf)})))) + {:flags (join "" flags-buf) :pattern (join "" buf)})))) (define try-op-4! (fn @@ -512,58 +582,104 @@ (fn (start) (cond - ((at? "==") (do (js-emit! "op" "==" start) (advance! 2) true)) - ((at? "!=") (do (js-emit! "op" "!=" start) (advance! 2) true)) - ((at? "<=") (do (js-emit! "op" "<=" start) (advance! 2) true)) - ((at? ">=") (do (js-emit! "op" ">=" start) (advance! 2) true)) - ((at? "&&") (do (js-emit! "op" "&&" start) (advance! 2) true)) - ((at? "||") (do (js-emit! "op" "||" start) (advance! 2) true)) - ((at? "??") (do (js-emit! "op" "??" start) (advance! 2) true)) - ((at? "=>") (do (js-emit! "op" "=>" start) (advance! 2) true)) - ((at? "**") (do (js-emit! "op" "**" start) (advance! 2) true)) - ((at? "<<") (do (js-emit! "op" "<<" start) (advance! 2) true)) - ((at? ">>") (do (js-emit! "op" ">>" start) (advance! 2) true)) - ((at? "++") (do (js-emit! "op" "++" start) (advance! 2) true)) - ((at? "--") (do (js-emit! "op" "--" start) (advance! 2) true)) - ((at? "+=") (do (js-emit! "op" "+=" start) (advance! 2) true)) - ((at? "-=") (do (js-emit! "op" "-=" start) (advance! 2) true)) - ((at? "*=") (do (js-emit! "op" "*=" start) (advance! 2) true)) - ((at? "/=") (do (js-emit! "op" "/=" start) (advance! 2) true)) - ((at? "%=") (do (js-emit! "op" "%=" start) (advance! 2) true)) - ((at? "&=") (do (js-emit! "op" "&=" start) (advance! 2) true)) - ((at? "|=") (do (js-emit! "op" "|=" start) (advance! 2) true)) - ((at? "^=") (do (js-emit! "op" "^=" start) (advance! 2) true)) - ((at? "?.") (do (js-emit! "op" "?." start) (advance! 2) true)) + ((at? "==") + (do (js-emit! "op" "==" start) (advance! 2) true)) + ((at? "!=") + (do (js-emit! "op" "!=" start) (advance! 2) true)) + ((at? "<=") + (do (js-emit! "op" "<=" start) (advance! 2) true)) + ((at? ">=") + (do (js-emit! "op" ">=" start) (advance! 2) true)) + ((at? "&&") + (do (js-emit! "op" "&&" start) (advance! 2) true)) + ((at? "||") + (do (js-emit! "op" "||" start) (advance! 2) true)) + ((at? "??") + (do (js-emit! "op" "??" start) (advance! 2) true)) + ((at? "=>") + (do (js-emit! "op" "=>" start) (advance! 2) true)) + ((at? "**") + (do (js-emit! "op" "**" start) (advance! 2) true)) + ((at? "<<") + (do (js-emit! "op" "<<" start) (advance! 2) true)) + ((at? ">>") + (do (js-emit! "op" ">>" start) (advance! 2) true)) + ((at? "++") + (do (js-emit! "op" "++" start) (advance! 2) true)) + ((at? "--") + (do (js-emit! "op" "--" start) (advance! 2) true)) + ((at? "+=") + (do (js-emit! "op" "+=" start) (advance! 2) true)) + ((at? "-=") + (do (js-emit! "op" "-=" start) (advance! 2) true)) + ((at? "*=") + (do (js-emit! "op" "*=" start) (advance! 2) true)) + ((at? "/=") + (do (js-emit! "op" "/=" start) (advance! 2) true)) + ((at? "%=") + (do (js-emit! "op" "%=" start) (advance! 2) true)) + ((at? "&=") + (do (js-emit! "op" "&=" start) (advance! 2) true)) + ((at? "|=") + (do (js-emit! "op" "|=" start) (advance! 2) true)) + ((at? "^=") + (do (js-emit! "op" "^=" start) (advance! 2) true)) + ((at? "?.") + (do (js-emit! "op" "?." start) (advance! 2) true)) (else false)))) (define emit-one-op! (fn (ch start) (cond - ((= ch "(") (do (js-emit! "punct" "(" start) (advance! 1))) - ((= ch ")") (do (js-emit! "punct" ")" start) (advance! 1))) - ((= ch "[") (do (js-emit! "punct" "[" start) (advance! 1))) - ((= ch "]") (do (js-emit! "punct" "]" start) (advance! 1))) - ((= ch "{") (do (js-emit! "punct" "{" start) (advance! 1))) - ((= ch "}") (do (js-emit! "punct" "}" start) (advance! 1))) - ((= ch ",") (do (js-emit! "punct" "," start) (advance! 1))) - ((= ch ";") (do (js-emit! "punct" ";" start) (advance! 1))) - ((= ch ":") (do (js-emit! "punct" ":" start) (advance! 1))) - ((= ch ".") (do (js-emit! "punct" "." start) (advance! 1))) - ((= ch "?") (do (js-emit! "op" "?" start) (advance! 1))) - ((= ch "+") (do (js-emit! "op" "+" start) (advance! 1))) - ((= ch "-") (do (js-emit! "op" "-" start) (advance! 1))) - ((= ch "*") (do (js-emit! "op" "*" start) (advance! 1))) - ((= ch "/") (do (js-emit! "op" "/" start) (advance! 1))) - ((= ch "%") (do (js-emit! "op" "%" start) (advance! 1))) - ((= ch "=") (do (js-emit! "op" "=" start) (advance! 1))) - ((= ch "<") (do (js-emit! "op" "<" start) (advance! 1))) - ((= ch ">") (do (js-emit! "op" ">" start) (advance! 1))) - ((= ch "!") (do (js-emit! "op" "!" start) (advance! 1))) - ((= ch "&") (do (js-emit! "op" "&" start) (advance! 1))) - ((= ch "|") (do (js-emit! "op" "|" start) (advance! 1))) - ((= ch "^") (do (js-emit! "op" "^" start) (advance! 1))) - ((= ch "~") (do (js-emit! "op" "~" start) (advance! 1))) + ((= ch "(") + (do (js-emit! "punct" "(" start) (advance! 1))) + ((= ch ")") + (do (js-emit! "punct" ")" start) (advance! 1))) + ((= ch "[") + (do (js-emit! "punct" "[" start) (advance! 1))) + ((= ch "]") + (do (js-emit! "punct" "]" start) (advance! 1))) + ((= ch "{") + (do (js-emit! "punct" "{" start) (advance! 1))) + ((= ch "}") + (do (js-emit! "punct" "}" start) (advance! 1))) + ((= ch ",") + (do (js-emit! "punct" "," start) (advance! 1))) + ((= ch ";") + (do (js-emit! "punct" ";" start) (advance! 1))) + ((= ch ":") + (do (js-emit! "punct" ":" start) (advance! 1))) + ((= ch ".") + (do (js-emit! "punct" "." start) (advance! 1))) + ((= ch "?") + (do (js-emit! "op" "?" start) (advance! 1))) + ((= ch "+") + (do (js-emit! "op" "+" start) (advance! 1))) + ((= ch "-") + (do (js-emit! "op" "-" start) (advance! 1))) + ((= ch "*") + (do (js-emit! "op" "*" start) (advance! 1))) + ((= ch "/") + (do (js-emit! "op" "/" start) (advance! 1))) + ((= ch "%") + (do (js-emit! "op" "%" start) (advance! 1))) + ((= ch "=") + (do (js-emit! "op" "=" start) (advance! 1))) + ((= ch "<") + (do (js-emit! "op" "<" start) (advance! 1))) + ((= ch ">") + (do (js-emit! "op" ">" start) (advance! 1))) + ((= ch "!") + (do (js-emit! "op" "!" start) (advance! 1))) + ((= ch "&") + (do (js-emit! "op" "&" start) (advance! 1))) + ((= ch "|") + (do (js-emit! "op" "|" start) (advance! 1))) + ((= ch "^") + (do (js-emit! "op" "^" start) (advance! 1))) + ((= ch "~") + (do (js-emit! "op" "~" start) (advance! 1))) (else (advance! 1))))) (define scan! diff --git a/lib/js/test262-scoreboard.json b/lib/js/test262-scoreboard.json index 58e45ede..852e7b94 100644 --- a/lib/js/test262-scoreboard.json +++ b/lib/js/test262-scoreboard.json @@ -1,9 +1,9 @@ { "totals": { "pass": 65, - "fail": 26, + "fail": 28, "skip": 1, - "timeout": 8, + "timeout": 6, "total": 100, "runnable": 99, "pass_rate": 65.7 @@ -13,9 +13,9 @@ "category": "built-ins/String", "total": 100, "pass": 65, - "fail": 26, + "fail": 28, "skip": 1, - "timeout": 8, + "timeout": 6, "pass_rate": 65.7, "top_failures": [ [ @@ -23,11 +23,11 @@ 16 ], [ - "Timeout", + "TypeError: not a function", 8 ], [ - "TypeError: not a function", + "Timeout", 6 ], [ @@ -47,11 +47,11 @@ 16 ], [ - "Timeout", + "TypeError: not a function", 8 ], [ - "TypeError: not a function", + "Timeout", 6 ], [ @@ -68,6 +68,6 @@ ] ], "pinned_commit": "d5e73fc8d2c663554fb72e2380a8c2bc1a318a33", - "elapsed_seconds": 420.4, + "elapsed_seconds": 353.1, "workers": 1 } \ No newline at end of file diff --git a/lib/js/test262-scoreboard.md b/lib/js/test262-scoreboard.md index 9eca91c0..40d688ce 100644 --- a/lib/js/test262-scoreboard.md +++ b/lib/js/test262-scoreboard.md @@ -1,15 +1,15 @@ # test262 scoreboard Pinned commit: `d5e73fc8d2c663554fb72e2380a8c2bc1a318a33` -Wall time: 420.4s +Wall time: 353.1s -**Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=26 skip=1 timeout=8 total=100. +**Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=28 skip=1 timeout=6 total=100. ## Top failure modes - **16x** Test262Error (assertion failed) -- **8x** Timeout -- **6x** TypeError: not a function +- **8x** TypeError: not a function +- **6x** Timeout - **2x** Unhandled: Not callable: \\\ - **1x** ReferenceError (undefined symbol) - **1x** SyntaxError (parse/unsupported syntax) @@ -18,14 +18,14 @@ Wall time: 420.4s | Category | Pass | Fail | Skip | Timeout | Total | Pass % | |---|---:|---:|---:|---:|---:|---:| -| built-ins/String | 65 | 26 | 1 | 8 | 100 | 65.7% | +| built-ins/String | 65 | 28 | 1 | 6 | 100 | 65.7% | ## Per-category top failures (min 10 runnable, worst first) ### built-ins/String (65/99 — 65.7%) - **16x** Test262Error (assertion failed) -- **8x** Timeout -- **6x** TypeError: not a function +- **8x** TypeError: not a function +- **6x** Timeout - **2x** Unhandled: Not callable: \\\ - **1x** ReferenceError (undefined symbol) diff --git a/plans/js-on-sx.md b/plans/js-on-sx.md index 663e0bd8..5fc77465 100644 --- a/plans/js-on-sx.md +++ b/plans/js-on-sx.md @@ -158,6 +158,8 @@ Each item: implement → tests → update progress. Mark `[x]` when tests green. Append-only record of completed iterations. Loop writes one line per iteration: date, what was done, test count delta. +- 2026-05-07 — **JS lexer: handle `\uXXXX` and `\xXX` escape sequences in string literals.** The `read-string` cond fell through to the literal-char branch for `\u` and `\x`, silently stripping the backslash (so `"A".length` returned 5 instead of 1). Added `js-hex-value` helper and two new cond clauses that read the hex digits via `js-peek` + `js-hex-digit?`, compute the code point, and emit it via `char-from-code`. Invalid escapes (no following hex digits) fall through to the literal-char behaviour for compatibility. With test isolation (`--restart-every 1`) built-ins/String 65/99 → 68/99. Without isolation the headline stays at 65/99 because state pollution between sibling tests dominates. conformance.sh: 148/148. + - 2026-05-07 — **Bump test262 runner default per-test timeout 5s→15s.** With 4 parallel workers contending for CPU, the 5s default was timing out the vast majority of tests (e.g. 85/99 on built-ins/String). Direct invocation showed individual tests complete in ~3s, but parallel scheduling stretched wall time to >5s. Bumping to 15s makes the scoreboard usable: built-ins/String 14.1% → 65.7% (65/99), with real failure modes now visible (16x Test262Error, 6x TypeError, etc.) instead of "85x Timeout" drowning the signal. Regenerated scoreboard to reflect the new state. conformance.sh: 148/148. - 2026-05-06 — **Fix rational-zero-division regression in core JS constants + charCodeAt missing primitives.** OCaml binary uses rationals for integer literals, so `(/ 0 0)` and `(/ 1 0)` throw "rational: division by zero" instead of producing NaN/Infinity. Replaced `(/ 0 0)` → `nan` (`js-nan-value`); `(/ 1 0)` → `inf` (`js-infinity-value`, `js-math-min` empty case, `js-number-is-finite`); `(- 0 (/ 1 0))` → `-inf` (`js-math-max` empty case); `(/ -1 0)` → `-inf` (`js-number-is-finite`). `js-max-value-approx` was looping forever (rationals never reach float infinity) — replaced with literal `1.7976931348623157e+308`. Fixed `charCodeAt` and string `.length` to use `(len s)` and `(char-code (char-at s idx))` instead of missing `unicode-len`/`unicode-char-code-at` primitives. conformance.sh: 0→148/148. Unit tests: 521/530 best run (baseline run was 417/530; both timeout-flaky).