HS: tokenizer-stream API → 13 tests pass (-13 skips)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 54s
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 54s
lib/hyperscript/tokenizer.sx — added cursor + follow-set wrapper over
the existing flat-list tokenize output:
hs-stream src → {:tokens :pos :follows :last-match :last-ws}
hs-stream-current s → next non-WS token (skips WS, captures :last-ws)
hs-stream-match s value → consume if value matches & not in follow set
hs-stream-match-type s ...types → consume if upstream type name matches
hs-stream-match-any s ...names → consume if value matches any name
hs-stream-match-any-op s ...ops → consume if op token & value matches
hs-stream-peek s value n → look n non-WS tokens ahead, no consume
hs-stream-consume-until s marker → collect tokens until marker
hs-stream-consume-until-ws s → collect until next whitespace
hs-stream-push-follow! / pop-follow!
hs-stream-push-follows! / pop-follows! n
hs-stream-clear-follows! → saved / restore-follows! saved
hs-stream-last-match / last-ws
hs-stream-type-map maps our lowercase type names to upstream's
("ident" → "IDENTIFIER", "number" → "NUMBER", etc.) so type-based
matching works against upstream test expectations.
13 tokenizer-stream tests now pass; 30/30 in hs-upstream-core/tokenizer.
Skips remaining: 5 (down from 18).
- 2 template-component scope tests
- 1 async event dispatch (until event keyword works)
- left for later: needs more architectural work
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -109,6 +109,102 @@ SKIP_TEST_NAMES = {
|
||||
# Manually-written SX test bodies for tests whose upstream body cannot be
|
||||
# auto-translated. Key = test name; value = SX lines to emit inside deftest.
|
||||
MANUAL_TEST_BODIES = {
|
||||
# === Tokenizer-stream API tests (13) — exercise hs-stream and friends in
|
||||
# lib/hyperscript/tokenizer.sx, which wraps hs-tokenize output with the
|
||||
# cursor + follow-set semantics upstream exposes on Tokens objects. ===
|
||||
"matchToken consumes and returns on match": [
|
||||
' (let ((s (hs-stream "foo bar baz")))',
|
||||
' (assert= (get (hs-stream-match s "foo") :value) "foo")',
|
||||
' (assert (nil? (hs-stream-match s "baz")))',
|
||||
' (assert= (get (hs-stream-current s) :value) "bar")',
|
||||
' (assert= (get (hs-stream-match s "bar") :value) "bar"))',
|
||||
],
|
||||
"matchToken honors the follow set": [
|
||||
' (let ((s (hs-stream "and or not")))',
|
||||
' (hs-stream-push-follow! s "and")',
|
||||
' (assert (nil? (hs-stream-match s "and")))',
|
||||
' (hs-stream-pop-follow! s)',
|
||||
' (assert= (get (hs-stream-match s "and") :value) "and"))',
|
||||
],
|
||||
"matchTokenType matches by type": [
|
||||
' (let ((s (hs-stream "foo 42")))',
|
||||
' (assert= (get (hs-stream-match-type s "IDENTIFIER") :value) "foo")',
|
||||
' (assert (nil? (hs-stream-match-type s "STRING")))',
|
||||
' (assert= (get (hs-stream-match-type s "STRING" "NUMBER") :value) "42"))',
|
||||
],
|
||||
"matchOpToken matches operators by value": [
|
||||
' (let ((s (hs-stream "1 + 2")))',
|
||||
' (assert= (get (hs-stream-match-type s "NUMBER") :value) "1")',
|
||||
' (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+"))',
|
||||
],
|
||||
"matchAnyToken and matchAnyOpToken try each option": [
|
||||
' (let ((s (hs-stream "bar + baz")))',
|
||||
' (assert= (get (hs-stream-match-any s "foo" "bar" "baz") :value) "bar")',
|
||||
' (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+")',
|
||||
' (assert (nil? (hs-stream-match-any s "foo" "quux"))))',
|
||||
],
|
||||
"peekToken skips whitespace when looking ahead": [
|
||||
' (let ((s (hs-stream "for x in items")))',
|
||||
' (assert= (get (hs-stream-peek s "for" 0) :value) "for")',
|
||||
' (assert= (get (hs-stream-peek s "x" 1) :value) "x")',
|
||||
' (assert= (get (hs-stream-peek s "in" 2) :value) "in")',
|
||||
' (assert= (get (hs-stream-peek s "items" 3) :value) "items")',
|
||||
' (assert (nil? (hs-stream-peek s "wrong" 1))))',
|
||||
],
|
||||
"consumeUntil collects tokens up to a marker": [
|
||||
' (let ((s (hs-stream "a b c end d")))',
|
||||
' (let ((collected (filter (fn (t) (not (= (get t :type) "whitespace")))',
|
||||
' (hs-stream-consume-until s "end"))))',
|
||||
' (assert= (map (fn (t) (get t :value)) collected) (list "a" "b" "c"))',
|
||||
' (assert= (get (hs-stream-current s) :value) "end")))',
|
||||
],
|
||||
"consumeUntilWhitespace stops at first whitespace": [
|
||||
' (let ((s (hs-stream "abc def")))',
|
||||
' (let ((collected (hs-stream-consume-until-ws s)))',
|
||||
' (assert= (len collected) 1)',
|
||||
' (assert= (get (first collected) :value) "abc")',
|
||||
' (assert= (get (hs-stream-current s) :value) "def")))',
|
||||
],
|
||||
"pushFollow/popFollow nest follow-set boundaries": [
|
||||
' (let ((s (hs-stream "and or not")))',
|
||||
' (hs-stream-push-follow! s "and")',
|
||||
' (hs-stream-push-follow! s "or")',
|
||||
' (assert (nil? (hs-stream-match s "and")))',
|
||||
' (hs-stream-pop-follow! s)',
|
||||
' (assert (nil? (hs-stream-match s "and")))',
|
||||
' (hs-stream-pop-follow! s)',
|
||||
' (assert= (get (hs-stream-match s "and") :value) "and"))',
|
||||
],
|
||||
"pushFollows/popFollows push and pop in bulk": [
|
||||
' (let ((s (hs-stream "and or not")))',
|
||||
' (hs-stream-push-follows! s (list "and" "or"))',
|
||||
' (assert (nil? (hs-stream-match s "and")))',
|
||||
' (assert (nil? (hs-stream-match s "or")))',
|
||||
' (hs-stream-pop-follows! s 2)',
|
||||
' (assert= (get (hs-stream-match s "and") :value) "and"))',
|
||||
],
|
||||
"clearFollows/restoreFollows round-trip the follow set": [
|
||||
' (let ((s (hs-stream "and or not")))',
|
||||
' (hs-stream-push-follow! s "and")',
|
||||
' (hs-stream-push-follow! s "or")',
|
||||
' (let ((saved (hs-stream-clear-follows! s)))',
|
||||
' (assert= (get (hs-stream-match s "and") :value) "and")',
|
||||
' (hs-stream-restore-follows! s saved)',
|
||||
' (assert (nil? (hs-stream-match s "or")))))',
|
||||
],
|
||||
"lastMatch returns the last consumed token": [
|
||||
' (let ((s (hs-stream "foo bar baz")))',
|
||||
' (hs-stream-match s "foo")',
|
||||
' (assert= (get (hs-stream-last-match s) :value) "foo")',
|
||||
' (hs-stream-match s "bar")',
|
||||
' (assert= (get (hs-stream-last-match s) :value) "bar"))',
|
||||
],
|
||||
"lastWhitespace reflects whitespace before the current token": [
|
||||
' (let ((s (hs-stream "foo bar")))',
|
||||
' (hs-stream-match s "foo")',
|
||||
' (hs-stream-skip-ws! s)',
|
||||
' (assert= (hs-stream-last-ws s) " "))',
|
||||
],
|
||||
# throttle: first click fires, subsequent within 200ms dropped.
|
||||
# In the synchronous mock no time passes between two dom-dispatch calls.
|
||||
"throttled at <time> drops events within the window": [
|
||||
|
||||
Reference in New Issue
Block a user