coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

js-on-sx: lexer handles \uXXXX and \xXX string escapes
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 1m3s
Details

Browse Source 
read-string fell through to the literal-char branch for \u and \x,
silently stripping the backslash ("A".length returned 5 instead
of 1). Added js-hex-value helper and two cond clauses that read the
hex digits via js-peek + js-hex-digit?, compute the code point, and
emit it via char-from-code. Invalid escapes fall through to the
literal-char behaviour. built-ins/String (with --restart-every 1):
65/99 → 68/99. conformance.sh: 148/148.
This commit is contained in:
giles
2026-05-07 12:02:30 +00:00
parent 89f1c0ccbe
commit 081f934cad
4 changed files with 193 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
lib/js/lexer.sx
View File

@@ -29,6 +29,16 @@
(and (>= c "a") (<= c "f"))
(and (>= c "A") (<= c "F")))))
(define
js-hex-value
(fn
(c)
(cond
((and (>= c "0") (<= c "9")) (- (char-code c) 48))
((and (>= c "a") (<= c "f")) (- (char-code c) 87))
((and (>= c "A") (<= c "F")) (- (char-code c) 55))
(else 0))))
(define
js-letter?
(fn (c) (or (and (>= c "a") (<= c "z")) (and (>= c "A") (<= c "Z")))))
@@ -37,9 +47,9 @@
(define js-ident-char? (fn (c) (or (js-ident-start? c) (js-digit? c))))
;; ── Reserved words ────────────────────────────────────────────────
(define js-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
;; ── Reserved words ────────────────────────────────────────────────
(define
js-keywords
(list
@@ -86,15 +96,18 @@
"await"
"of"))
;; ── Main tokenizer ────────────────────────────────────────────────
(define js-keyword? (fn (word) (contains? js-keywords word)))
;; ── Main tokenizer ────────────────────────────────────────────────
(define
js-tokenize
(fn
(src)
(let
((tokens (list)) (pos 0) (src-len (len src)) (nl-before false))
((tokens (list))
(pos 0)
(src-len (len src))
(nl-before false))
(define
js-peek
(fn
@@ -109,7 +122,7 @@
(let
((sl (len s)))
(and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
(define js-emit! (fn (type value start) (append! tokens {:pos start :value value :type type :nl nl-before})))
(define js-emit! (fn (type value start) (append! tokens {:nl nl-before :type type :value value :pos start})))
(define
skip-line-comment!
(fn
@@ -256,11 +269,55 @@
((= ch "b") (append! chars "\\b"))
((= ch "f") (append! chars "\\f"))
((= ch "v") (append! chars "\\v"))
((= ch "u")
(if
(and
(< (+ pos 4) src-len)
(js-hex-digit? (js-peek 1))
(js-hex-digit? (js-peek 2))
(js-hex-digit? (js-peek 3))
(js-hex-digit? (js-peek 4)))
(do
(append!
chars
(char-from-code
(+
(*
4096
(js-hex-value
(js-peek 1)))
(*
256
(js-hex-value
(js-peek 2)))
(*
16
(js-hex-value
(js-peek 3)))
(js-hex-value (js-peek 4)))))
(advance! 4))
(append! chars ch)))
((= ch "x")
(if
(and
(< (+ pos 2) src-len)
(js-hex-digit? (js-peek 1))
(js-hex-digit? (js-peek 2)))
(do
(append!
chars
(char-from-code
(+
(* 16 (js-hex-value (js-peek 1)))
(js-hex-value (js-peek 2)))))
(advance! 2))
(append! chars ch)))
(else (append! chars ch)))
(advance! 1))))
(loop)))
((= (cur) quote-char) (advance! 1))
(else (do (append! chars (cur)) (advance! 1) (loop))))))
(else
(do (append! chars (cur)) (advance! 1) (loop))))))
(loop)
(join "" chars))))
(define
@@ -291,7 +348,8 @@
()
(cond
((>= pos src-len) nil)
((and (= (cur) "}") (= depth 1)) (advance! 1))
((and (= (cur) "}") (= depth 1))
(advance! 1))
((= (cur) "}")
(do
(append! buf (cur))
@@ -327,7 +385,9 @@
(advance! 1)))
(sloop)))
((= (cur) q)
(do (append! buf (cur)) (advance! 1)))
(do
(append! buf (cur))
(advance! 1)))
(else
(do
(append! buf (cur))
@@ -336,7 +396,10 @@
(sloop)
(expr-loop))))
(else
(do (append! buf (cur)) (advance! 1) (expr-loop))))))
(do
(append! buf (cur))
(advance! 1)
(expr-loop))))))
(expr-loop)
(join "" buf))))
(define
@@ -378,14 +441,17 @@
(else (append! chars ch)))
(advance! 1))))
(loop)))
(else (do (append! chars (cur)) (advance! 1) (loop))))))
(else
(do (append! chars (cur)) (advance! 1) (loop))))))
(loop)
(flush-chars!)
(if
(= (len parts) 0)
""
(if
(and (= (len parts) 1) (= (nth (nth parts 0) 0) "str"))
(and
(= (len parts) 1)
(= (nth (nth parts 0) 0) "str"))
(nth (nth parts 0) 1)
parts)))))
(define
@@ -455,9 +521,13 @@
(append! buf (cur))
(advance! 1)
(body-loop)))
((and (= (cur) "/") (not in-class)) (advance! 1))
((and (= (cur) "/") (not in-class))
(advance! 1))
(else
(begin (append! buf (cur)) (advance! 1) (body-loop))))))
(begin
(append! buf (cur))
(advance! 1)
(body-loop))))))
(body-loop)
(let
((flags-buf (list)))
@@ -472,7 +542,7 @@
(advance! 1)
(flags-loop)))))
(flags-loop)
{:pattern (join "" buf) :flags (join "" flags-buf)}))))
{:flags (join "" flags-buf) :pattern (join "" buf)}))))
(define
try-op-4!
(fn
@@ -512,58 +582,104 @@
(fn
(start)
(cond
((at? "==") (do (js-emit! "op" "==" start) (advance! 2) true))
((at? "!=") (do (js-emit! "op" "!=" start) (advance! 2) true))
((at? "<=") (do (js-emit! "op" "<=" start) (advance! 2) true))
((at? ">=") (do (js-emit! "op" ">=" start) (advance! 2) true))
((at? "&&") (do (js-emit! "op" "&&" start) (advance! 2) true))
((at? "||") (do (js-emit! "op" "||" start) (advance! 2) true))
((at? "??") (do (js-emit! "op" "??" start) (advance! 2) true))
((at? "=>") (do (js-emit! "op" "=>" start) (advance! 2) true))
((at? "**") (do (js-emit! "op" "**" start) (advance! 2) true))
((at? "<<") (do (js-emit! "op" "<<" start) (advance! 2) true))
((at? ">>") (do (js-emit! "op" ">>" start) (advance! 2) true))
((at? "++") (do (js-emit! "op" "++" start) (advance! 2) true))
((at? "--") (do (js-emit! "op" "--" start) (advance! 2) true))
((at? "+=") (do (js-emit! "op" "+=" start) (advance! 2) true))
((at? "-=") (do (js-emit! "op" "-=" start) (advance! 2) true))
((at? "*=") (do (js-emit! "op" "*=" start) (advance! 2) true))
((at? "/=") (do (js-emit! "op" "/=" start) (advance! 2) true))
((at? "%=") (do (js-emit! "op" "%=" start) (advance! 2) true))
((at? "&=") (do (js-emit! "op" "&=" start) (advance! 2) true))
((at? "|=") (do (js-emit! "op" "|=" start) (advance! 2) true))
((at? "^=") (do (js-emit! "op" "^=" start) (advance! 2) true))
((at? "?.") (do (js-emit! "op" "?." start) (advance! 2) true))
((at? "==")
(do (js-emit! "op" "==" start) (advance! 2) true))
((at? "!=")
(do (js-emit! "op" "!=" start) (advance! 2) true))
((at? "<=")
(do (js-emit! "op" "<=" start) (advance! 2) true))
((at? ">=")
(do (js-emit! "op" ">=" start) (advance! 2) true))
((at? "&&")
(do (js-emit! "op" "&&" start) (advance! 2) true))
((at? "||")
(do (js-emit! "op" "||" start) (advance! 2) true))
((at? "??")
(do (js-emit! "op" "??" start) (advance! 2) true))
((at? "=>")
(do (js-emit! "op" "=>" start) (advance! 2) true))
((at? "**")
(do (js-emit! "op" "**" start) (advance! 2) true))
((at? "<<")
(do (js-emit! "op" "<<" start) (advance! 2) true))
((at? ">>")
(do (js-emit! "op" ">>" start) (advance! 2) true))
((at? "++")
(do (js-emit! "op" "++" start) (advance! 2) true))
((at? "--")
(do (js-emit! "op" "--" start) (advance! 2) true))
((at? "+=")
(do (js-emit! "op" "+=" start) (advance! 2) true))
((at? "-=")
(do (js-emit! "op" "-=" start) (advance! 2) true))
((at? "*=")
(do (js-emit! "op" "*=" start) (advance! 2) true))
((at? "/=")
(do (js-emit! "op" "/=" start) (advance! 2) true))
((at? "%=")
(do (js-emit! "op" "%=" start) (advance! 2) true))
((at? "&=")
(do (js-emit! "op" "&=" start) (advance! 2) true))
((at? "|=")
(do (js-emit! "op" "|=" start) (advance! 2) true))
((at? "^=")
(do (js-emit! "op" "^=" start) (advance! 2) true))
((at? "?.")
(do (js-emit! "op" "?." start) (advance! 2) true))
(else false))))
(define
emit-one-op!
(fn
(ch start)
(cond
((= ch "(") (do (js-emit! "punct" "(" start) (advance! 1)))
((= ch ")") (do (js-emit! "punct" ")" start) (advance! 1)))
((= ch "[") (do (js-emit! "punct" "[" start) (advance! 1)))
((= ch "]") (do (js-emit! "punct" "]" start) (advance! 1)))
((= ch "{") (do (js-emit! "punct" "{" start) (advance! 1)))
((= ch "}") (do (js-emit! "punct" "}" start) (advance! 1)))
((= ch ",") (do (js-emit! "punct" "," start) (advance! 1)))
((= ch ";") (do (js-emit! "punct" ";" start) (advance! 1)))
((= ch ":") (do (js-emit! "punct" ":" start) (advance! 1)))
((= ch ".") (do (js-emit! "punct" "." start) (advance! 1)))
((= ch "?") (do (js-emit! "op" "?" start) (advance! 1)))
((= ch "+") (do (js-emit! "op" "+" start) (advance! 1)))
((= ch "-") (do (js-emit! "op" "-" start) (advance! 1)))
((= ch "*") (do (js-emit! "op" "*" start) (advance! 1)))
((= ch "/") (do (js-emit! "op" "/" start) (advance! 1)))
((= ch "%") (do (js-emit! "op" "%" start) (advance! 1)))
((= ch "=") (do (js-emit! "op" "=" start) (advance! 1)))
((= ch "<") (do (js-emit! "op" "<" start) (advance! 1)))
((= ch ">") (do (js-emit! "op" ">" start) (advance! 1)))
((= ch "!") (do (js-emit! "op" "!" start) (advance! 1)))
((= ch "&") (do (js-emit! "op" "&" start) (advance! 1)))
((= ch "|") (do (js-emit! "op" "|" start) (advance! 1)))
((= ch "^") (do (js-emit! "op" "^" start) (advance! 1)))
((= ch "~") (do (js-emit! "op" "~" start) (advance! 1)))
((= ch "(")
(do (js-emit! "punct" "(" start) (advance! 1)))
((= ch ")")
(do (js-emit! "punct" ")" start) (advance! 1)))
((= ch "[")
(do (js-emit! "punct" "[" start) (advance! 1)))
((= ch "]")
(do (js-emit! "punct" "]" start) (advance! 1)))
((= ch "{")
(do (js-emit! "punct" "{" start) (advance! 1)))
((= ch "}")
(do (js-emit! "punct" "}" start) (advance! 1)))
((= ch ",")
(do (js-emit! "punct" "," start) (advance! 1)))
((= ch ";")
(do (js-emit! "punct" ";" start) (advance! 1)))
((= ch ":")
(do (js-emit! "punct" ":" start) (advance! 1)))
((= ch ".")
(do (js-emit! "punct" "." start) (advance! 1)))
((= ch "?")
(do (js-emit! "op" "?" start) (advance! 1)))
((= ch "+")
(do (js-emit! "op" "+" start) (advance! 1)))
((= ch "-")
(do (js-emit! "op" "-" start) (advance! 1)))
((= ch "*")
(do (js-emit! "op" "*" start) (advance! 1)))
((= ch "/")
(do (js-emit! "op" "/" start) (advance! 1)))
((= ch "%")
(do (js-emit! "op" "%" start) (advance! 1)))
((= ch "=")
(do (js-emit! "op" "=" start) (advance! 1)))
((= ch "<")
(do (js-emit! "op" "<" start) (advance! 1)))
((= ch ">")
(do (js-emit! "op" ">" start) (advance! 1)))
((= ch "!")
(do (js-emit! "op" "!" start) (advance! 1)))
((= ch "&")
(do (js-emit! "op" "&" start) (advance! 1)))
((= ch "|")
(do (js-emit! "op" "|" start) (advance! 1)))
((= ch "^")
(do (js-emit! "op" "^" start) (advance! 1)))
((= ch "~")
(do (js-emit! "op" "~" start) (advance! 1)))
(else (advance! 1)))))
(define
scan!

18
lib/js/test262-scoreboard.json
View File

@@ -1,9 +1,9 @@
{
"totals": {
"pass": 65,
"fail": 26,
"fail": 28,
"skip": 1,
"timeout": 8,
"timeout": 6,
"total": 100,
"runnable": 99,
"pass_rate": 65.7
@@ -13,9 +13,9 @@
"category": "built-ins/String",
"total": 100,
"pass": 65,
"fail": 26,
"fail": 28,
"skip": 1,
"timeout": 8,
"timeout": 6,
"pass_rate": 65.7,
"top_failures": [
[
@@ -23,11 +23,11 @@
16
],
[
"Timeout",
"TypeError: not a function",
8
],
[
"TypeError: not a function",
"Timeout",
6
],
[
@@ -47,11 +47,11 @@
16
],
[
"Timeout",
"TypeError: not a function",
8
],
[
"TypeError: not a function",
"Timeout",
6
],
[
@@ -68,6 +68,6 @@
]
],
"pinned_commit": "d5e73fc8d2c663554fb72e2380a8c2bc1a318a33",
"elapsed_seconds": 420.4,
"elapsed_seconds": 353.1,
"workers": 1
}

14
lib/js/test262-scoreboard.md
View File

@@ -1,15 +1,15 @@
# test262 scoreboard
Pinned commit: `d5e73fc8d2c663554fb72e2380a8c2bc1a318a33`
Wall time: 420.4s
Wall time: 353.1s
**Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=26 skip=1 timeout=8 total=100.
**Total:** 65/99 runnable passed (65.7%). Raw: pass=65 fail=28 skip=1 timeout=6 total=100.
## Top failure modes
- **16x** Test262Error (assertion failed)
- **8x** Timeout
- **6x** TypeError: not a function
- **8x** TypeError: not a function
- **6x** Timeout
- **2x** Unhandled: Not callable: \\\
- **1x** ReferenceError (undefined symbol)
- **1x** SyntaxError (parse/unsupported syntax)
@@ -18,14 +18,14 @@ Wall time: 420.4s
| Category | Pass | Fail | Skip | Timeout | Total | Pass % |
|---|---:|---:|---:|---:|---:|---:|
| built-ins/String | 65 | 26 | 1 | 8 | 100 | 65.7% |
| built-ins/String | 65 | 28 | 1 | 6 | 100 | 65.7% |
## Per-category top failures (min 10 runnable, worst first)
### built-ins/String (65/99 — 65.7%)
- **16x** Test262Error (assertion failed)
- **8x** Timeout
- **6x** TypeError: not a function
- **8x** TypeError: not a function
- **6x** Timeout
- **2x** Unhandled: Not callable: \\\
- **1x** ReferenceError (undefined symbol)

2
plans/js-on-sx.md
View File

@@ -158,6 +158,8 @@ Each item: implement → tests → update progress. Mark `[x]` when tests green.
Append-only record of completed iterations. Loop writes one line per iteration: date, what was done, test count delta.
- 2026-05-07 — **JS lexer: handle `\uXXXX` and `\xXX` escape sequences in string literals.** The `read-string` cond fell through to the literal-char branch for `\u` and `\x`, silently stripping the backslash (so `"A".length` returned 5 instead of 1). Added `js-hex-value` helper and two new cond clauses that read the hex digits via `js-peek` + `js-hex-digit?`, compute the code point, and emit it via `char-from-code`. Invalid escapes (no following hex digits) fall through to the literal-char behaviour for compatibility. With test isolation (`--restart-every 1`) built-ins/String 65/99 → 68/99. Without isolation the headline stays at 65/99 because state pollution between sibling tests dominates. conformance.sh: 148/148.
- 2026-05-07 — **Bump test262 runner default per-test timeout 5s→15s.** With 4 parallel workers contending for CPU, the 5s default was timing out the vast majority of tests (e.g. 85/99 on built-ins/String). Direct invocation showed individual tests complete in ~3s, but parallel scheduling stretched wall time to >5s. Bumping to 15s makes the scoreboard usable: built-ins/String 14.1% → 65.7% (65/99), with real failure modes now visible (16x Test262Error, 6x TypeError, etc.) instead of "85x Timeout" drowning the signal. Regenerated scoreboard to reflect the new state. conformance.sh: 148/148.
- 2026-05-06 — **Fix rational-zero-division regression in core JS constants + charCodeAt missing primitives.** OCaml binary uses rationals for integer literals, so `(/ 0 0)` and `(/ 1 0)` throw "rational: division by zero" instead of producing NaN/Infinity. Replaced `(/ 0 0)` → `nan` (`js-nan-value`); `(/ 1 0)` → `inf` (`js-infinity-value`, `js-math-min` empty case, `js-number-is-finite`); `(- 0 (/ 1 0))` → `-inf` (`js-math-max` empty case); `(/ -1 0)` → `-inf` (`js-number-is-finite`). `js-max-value-approx` was looping forever (rationals never reach float infinity) — replaced with literal `1.7976931348623157e+308`. Fixed `charCodeAt` and string `.length` to use `(len s)` and `(char-code (char-at s idx))` instead of missing `unicode-len`/`unicode-char-code-at` primitives. conformance.sh: 0→148/148. Unit tests: 521/530 best run (baseline run was 417/530; both timeout-flaky).