coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HS: tokenizer-stream API → 13 tests pass (-13 skips)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 54s
Details

Browse Source 
lib/hyperscript/tokenizer.sx — added cursor + follow-set wrapper over
the existing flat-list tokenize output:

  hs-stream src                 → {:tokens :pos :follows :last-match :last-ws}
  hs-stream-current  s          → next non-WS token (skips WS, captures :last-ws)
  hs-stream-match    s value    → consume if value matches & not in follow set
  hs-stream-match-type s ...types → consume if upstream type name matches
  hs-stream-match-any  s ...names → consume if value matches any name
  hs-stream-match-any-op s ...ops → consume if op token & value matches
  hs-stream-peek     s value n  → look n non-WS tokens ahead, no consume
  hs-stream-consume-until s marker     → collect tokens until marker
  hs-stream-consume-until-ws  s        → collect until next whitespace
  hs-stream-push-follow! / pop-follow!
  hs-stream-push-follows! / pop-follows! n
  hs-stream-clear-follows! → saved   /  restore-follows! saved
  hs-stream-last-match / last-ws

hs-stream-type-map maps our lowercase type names to upstream's
("ident" → "IDENTIFIER", "number" → "NUMBER", etc.) so type-based
matching works against upstream test expectations.

13 tokenizer-stream tests now pass; 30/30 in hs-upstream-core/tokenizer.

Skips remaining: 5 (down from 18).
  - 2 template-component scope tests
  - 1 async event dispatch (until event keyword works)
  - left for later: needs more architectural work

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
giles
2026-05-08 07:22:40 +00:00
parent d0b358eca2
commit a9eb821cce
4 changed files with 403 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tests/hs-run-filtered.js
View File

@@ -967,25 +967,6 @@ for(let i=startTest;i<Math.min(endTest,testCount);i++){
// 'repeat until event' loop suspends the OCaml kernel waiting for an
// event that is never fired from outside the K.eval call chain.
"until event keyword works",
// === Tokenizer-stream API tests (13) — upstream exposes a streaming token
// API on _hyperscript.internals.tokenizer (matchToken, peekToken, consumeUntil,
// pushFollow, etc.). Our lib/hyperscript/tokenizer.sx returns a flat token list
// and the parser keeps stream state internally as closures. Making these tests
// pass would require exposing a token-stream wrapper as a primitive. The
// tokenizer is correct; it just doesn't expose this API surface. ===
"matchToken consumes and returns on match",
"matchToken honors the follow set",
"matchTokenType matches by type",
"matchOpToken matches operators by value",
"matchAnyToken and matchAnyOpToken try each option",
"peekToken skips whitespace when looking ahead",
"consumeUntil collects tokens up to a marker",
"consumeUntilWhitespace stops at first whitespace",
"pushFollow/popFollow nest follow-set boundaries",
"pushFollows/popFollows push and pop in bulk",
"clearFollows/restoreFollows round-trip the follow set",
"lastMatch returns the last consumed token",
"lastWhitespace reflects whitespace before the current token",
// === Template-component scope tests (2) — upstream uses
// <script type="text/hyperscript-template" component="...">
// for HTML-template-based custom-element components. Our defcomp uses SX

96
tests/playwright/generate-sx-tests.py
View File

@@ -109,6 +109,102 @@ SKIP_TEST_NAMES = {
# Manually-written SX test bodies for tests whose upstream body cannot be
# auto-translated. Key = test name; value = SX lines to emit inside deftest.
MANUAL_TEST_BODIES = {
# === Tokenizer-stream API tests (13) — exercise hs-stream and friends in
# lib/hyperscript/tokenizer.sx, which wraps hs-tokenize output with the
# cursor + follow-set semantics upstream exposes on Tokens objects. ===
"matchToken consumes and returns on match": [
' (let ((s (hs-stream "foo bar baz")))',
' (assert= (get (hs-stream-match s "foo") :value) "foo")',
' (assert (nil? (hs-stream-match s "baz")))',
' (assert= (get (hs-stream-current s) :value) "bar")',
' (assert= (get (hs-stream-match s "bar") :value) "bar"))',
],
"matchToken honors the follow set": [
' (let ((s (hs-stream "and or not")))',
' (hs-stream-push-follow! s "and")',
' (assert (nil? (hs-stream-match s "and")))',
' (hs-stream-pop-follow! s)',
' (assert= (get (hs-stream-match s "and") :value) "and"))',
],
"matchTokenType matches by type": [
' (let ((s (hs-stream "foo 42")))',
' (assert= (get (hs-stream-match-type s "IDENTIFIER") :value) "foo")',
' (assert (nil? (hs-stream-match-type s "STRING")))',
' (assert= (get (hs-stream-match-type s "STRING" "NUMBER") :value) "42"))',
],
"matchOpToken matches operators by value": [
' (let ((s (hs-stream "1 + 2")))',
' (assert= (get (hs-stream-match-type s "NUMBER") :value) "1")',
' (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+"))',
],
"matchAnyToken and matchAnyOpToken try each option": [
' (let ((s (hs-stream "bar + baz")))',
' (assert= (get (hs-stream-match-any s "foo" "bar" "baz") :value) "bar")',
' (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+")',
' (assert (nil? (hs-stream-match-any s "foo" "quux"))))',
],
"peekToken skips whitespace when looking ahead": [
' (let ((s (hs-stream "for x in items")))',
' (assert= (get (hs-stream-peek s "for" 0) :value) "for")',
' (assert= (get (hs-stream-peek s "x" 1) :value) "x")',
' (assert= (get (hs-stream-peek s "in" 2) :value) "in")',
' (assert= (get (hs-stream-peek s "items" 3) :value) "items")',
' (assert (nil? (hs-stream-peek s "wrong" 1))))',
],
"consumeUntil collects tokens up to a marker": [
' (let ((s (hs-stream "a b c end d")))',
' (let ((collected (filter (fn (t) (not (= (get t :type) "whitespace")))',
' (hs-stream-consume-until s "end"))))',
' (assert= (map (fn (t) (get t :value)) collected) (list "a" "b" "c"))',
' (assert= (get (hs-stream-current s) :value) "end")))',
],
"consumeUntilWhitespace stops at first whitespace": [
' (let ((s (hs-stream "abc def")))',
' (let ((collected (hs-stream-consume-until-ws s)))',
' (assert= (len collected) 1)',
' (assert= (get (first collected) :value) "abc")',
' (assert= (get (hs-stream-current s) :value) "def")))',
],
"pushFollow/popFollow nest follow-set boundaries": [
' (let ((s (hs-stream "and or not")))',
' (hs-stream-push-follow! s "and")',
' (hs-stream-push-follow! s "or")',
' (assert (nil? (hs-stream-match s "and")))',
' (hs-stream-pop-follow! s)',
' (assert (nil? (hs-stream-match s "and")))',
' (hs-stream-pop-follow! s)',
' (assert= (get (hs-stream-match s "and") :value) "and"))',
],
"pushFollows/popFollows push and pop in bulk": [
' (let ((s (hs-stream "and or not")))',
' (hs-stream-push-follows! s (list "and" "or"))',
' (assert (nil? (hs-stream-match s "and")))',
' (assert (nil? (hs-stream-match s "or")))',
' (hs-stream-pop-follows! s 2)',
' (assert= (get (hs-stream-match s "and") :value) "and"))',
],
"clearFollows/restoreFollows round-trip the follow set": [
' (let ((s (hs-stream "and or not")))',
' (hs-stream-push-follow! s "and")',
' (hs-stream-push-follow! s "or")',
' (let ((saved (hs-stream-clear-follows! s)))',
' (assert= (get (hs-stream-match s "and") :value) "and")',
' (hs-stream-restore-follows! s saved)',
' (assert (nil? (hs-stream-match s "or")))))',
],
"lastMatch returns the last consumed token": [
' (let ((s (hs-stream "foo bar baz")))',
' (hs-stream-match s "foo")',
' (assert= (get (hs-stream-last-match s) :value) "foo")',
' (hs-stream-match s "bar")',
' (assert= (get (hs-stream-last-match s) :value) "bar"))',
],
"lastWhitespace reflects whitespace before the current token": [
' (let ((s (hs-stream "foo bar")))',
' (hs-stream-match s "foo")',
' (hs-stream-skip-ws! s)',
' (assert= (hs-stream-last-ws s) " "))',
],
# throttle: first click fires, subsequent within 200ms dropped.
# In the synchronous mock no time passes between two dom-dispatch calls.
"throttled at <time> drops events within the window": [