coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HS: E37 tokenizer API — 16/17 conformance tests passing
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 16s
Details

Browse Source 
Add hs-raw->api-token, hs-eof-sentinel, hs-api-list, hs-tokens-of,
hs-stream-token, hs-stream-consume, hs-stream-has-more, hs-token-type,
hs-token-value, hs-token-op? to runtime. Fix tokenizer to emit whitespace
tokens and handle dot/hash after closing brackets. Fix hs-tokens-of to
accept bare :template keyword flag via &rest args + some() check.
Remaining failure (string interpolation isnt surprising) requires full
DOM activation infrastructure.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
giles
2026-04-26 18:45:58 +00:00
parent 820132b839
commit cabb0467ab
5 changed files with 775 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
lib/hyperscript/tokenizer.sx
View File

@@ -334,11 +334,17 @@
(= ch "r")
(do (append! chars "\r") (hs-advance! 1))
(= ch "b")
(do (append! chars (char-from-code 8)) (hs-advance! 1))
(do
(append! chars (char-from-code 8))
(hs-advance! 1))
(= ch "f")
(do (append! chars (char-from-code 12)) (hs-advance! 1))
(do
(append! chars (char-from-code 12))
(hs-advance! 1))
(= ch "v")
(do (append! chars (char-from-code 11)) (hs-advance! 1))
(do
(append! chars (char-from-code 11))
(hs-advance! 1))
(= ch "\\")
(do (append! chars "\\") (hs-advance! 1))
(= ch quote-char)
@@ -353,12 +359,16 @@
(hs-hex-digit? (hs-peek 1)))
(let
((d1 (hs-hex-val (hs-cur)))
(d2 (hs-hex-val (hs-peek 1))))
(append! chars (char-from-code (+ (* d1 16) d2)))
(d2 (hs-hex-val (hs-peek 1))))
(append!
chars
(char-from-code (+ (* d1 16) d2)))
(hs-advance! 2))
(error "Invalid hexadecimal escape: \\x")))
:else
(do (append! chars "\\") (append! chars ch) (hs-advance! 1)))))
:else (do
(append! chars "\\")
(append! chars ch)
(hs-advance! 1)))))
(loop))
(= (hs-cur) quote-char)
(hs-advance! 1)
@@ -465,7 +475,13 @@
scan!
(fn
()
(skip-ws!)
(do
(let
((ws-start pos))
(skip-ws!)
(when
(and (> (len tokens) 0) (> pos ws-start))
(hs-emit! "whitespace" (slice src ws-start pos) ws-start))))
(when
(< pos src-len)
(let
@@ -489,6 +505,25 @@
(do (hs-emit! "selector" (read-selector) start) (scan!))
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!))
(and
(= ch ".")
(< (+ pos 1) src-len)
(or
(hs-letter? (hs-peek 1))
(= (hs-peek 1) "-")
(= (hs-peek 1) "_"))
(> (len tokens) 0)
(let
((lt (dict-get (nth tokens (- (len tokens) 1)) :type)))
(or
(= lt "paren-close")
(= lt "brace-close")
(= lt "bracket-close"))))
(do
(hs-emit! "dot" "." start)
(hs-advance! 1)
(hs-emit! "ident" (read-ident pos) start)
(scan!))
(and
(= ch ".")
(< (+ pos 1) src-len)
@@ -500,6 +535,22 @@
(hs-advance! 1)
(hs-emit! "class" (read-class-name pos) start)
(scan!))
(and
(= ch "#")
(< (+ pos 1) src-len)
(hs-ident-start? (hs-peek 1))
(> (len tokens) 0)
(let
((lt (dict-get (nth tokens (- (len tokens) 1)) :type)))
(or
(= lt "paren-close")
(= lt "brace-close")
(= lt "bracket-close"))))
(do
(hs-emit! "op" "#" start)
(hs-advance! 1)
(hs-emit! "ident" (read-ident pos) start)
(scan!))
(and
(= ch "#")
(< (+ pos 1) src-len)
@@ -569,21 +620,7 @@
(let
((word (read-ident start)))
(let
((full-word
(if
(and
(< pos src-len)
(= (hs-cur) "'")
(< (+ pos 1) src-len)
(hs-letter? (hs-peek 1))
(not
(and
(= (hs-peek 1) "s")
(or
(>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2)))))))
(do (hs-advance! 1) (str word "'" (read-ident pos)))
word)))
((full-word (if (and (< pos src-len) (= (hs-cur) "'") (< (+ pos 1) src-len) (hs-letter? (hs-peek 1)) (not (and (= (hs-peek 1) "s") (or (>= (+ pos 2) src-len) (not (hs-ident-char? (hs-peek 2))))))) (do (hs-advance! 1) (str word "'" (read-ident pos))) word)))
(hs-emit!
(if (hs-keyword? full-word) "keyword" "ident")
full-word