HS: tokenizer-stream API → 13 tests pass (-13 skips)

lib/hyperscript/tokenizer.sx — added cursor + follow-set wrapper over
the existing flat-list tokenize output:

  hs-stream src                 → {:tokens :pos :follows :last-match :last-ws}
  hs-stream-current  s          → next non-WS token (skips WS, captures :last-ws)
  hs-stream-match    s value    → consume if value matches & not in follow set
  hs-stream-match-type s ...types → consume if upstream type name matches
  hs-stream-match-any  s ...names → consume if value matches any name
  hs-stream-match-any-op s ...ops → consume if op token & value matches
  hs-stream-peek     s value n  → look n non-WS tokens ahead, no consume
  hs-stream-consume-until s marker     → collect tokens until marker
  hs-stream-consume-until-ws  s        → collect until next whitespace
  hs-stream-push-follow! / pop-follow!
  hs-stream-push-follows! / pop-follows! n
  hs-stream-clear-follows! → saved   /  restore-follows! saved
  hs-stream-last-match / last-ws

hs-stream-type-map maps our lowercase type names to upstream's
("ident" → "IDENTIFIER", "number" → "NUMBER", etc.) so type-based
matching works against upstream test expectations.

13 tokenizer-stream tests now pass; 30/30 in hs-upstream-core/tokenizer.

Skips remaining: 5 (down from 18).
  - 2 template-component scope tests
  - 1 async event dispatch (until event keyword works)
  - left for later: needs more architectural work

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

spec/tests/test-hyperscript-behavioral.sx

@@ -2877,31 +2877,98 @@
       (assert= (dom-text-content _el-div) "test${x} test 42 test$x test 42 test $x test ${x} test42 test_42 test_42 test-42 test.42")
     ))
   (deftest "clearFollows/restoreFollows round-trip the follow set"
-    (error "SKIP (untranslated): clearFollows/restoreFollows round-trip the follow set"))
+    (let ((s (hs-stream "and or not")))
+      (hs-stream-push-follow! s "and")
+      (hs-stream-push-follow! s "or")
+      (let ((saved (hs-stream-clear-follows! s)))
+        (assert= (get (hs-stream-match s "and") :value) "and")
+        (hs-stream-restore-follows! s saved)
+        (assert (nil? (hs-stream-match s "or")))))
+  )
   (deftest "consumeUntil collects tokens up to a marker"
-    (error "SKIP (untranslated): consumeUntil collects tokens up to a marker"))
+    (let ((s (hs-stream "a b c end d")))
+      (let ((collected (filter (fn (t) (not (= (get t :type) "whitespace")))
+                                (hs-stream-consume-until s "end"))))
+        (assert= (map (fn (t) (get t :value)) collected) (list "a" "b" "c"))
+        (assert= (get (hs-stream-current s) :value) "end")))
+  )
   (deftest "consumeUntilWhitespace stops at first whitespace"
-    (error "SKIP (untranslated): consumeUntilWhitespace stops at first whitespace"))
+    (let ((s (hs-stream "abc def")))
+      (let ((collected (hs-stream-consume-until-ws s)))
+        (assert= (len collected) 1)
+        (assert= (get (first collected) :value) "abc")
+        (assert= (get (hs-stream-current s) :value) "def")))
+  )
   (deftest "lastMatch returns the last consumed token"
-    (error "SKIP (untranslated): lastMatch returns the last consumed token"))
+    (let ((s (hs-stream "foo bar baz")))
+      (hs-stream-match s "foo")
+      (assert= (get (hs-stream-last-match s) :value) "foo")
+      (hs-stream-match s "bar")
+      (assert= (get (hs-stream-last-match s) :value) "bar"))
+  )
   (deftest "lastWhitespace reflects whitespace before the current token"
-    (error "SKIP (untranslated): lastWhitespace reflects whitespace before the current token"))
+    (let ((s (hs-stream "foo  bar")))
+      (hs-stream-match s "foo")
+      (hs-stream-skip-ws! s)
+      (assert= (hs-stream-last-ws s) "  "))
+  )
   (deftest "matchAnyToken and matchAnyOpToken try each option"
-    (error "SKIP (untranslated): matchAnyToken and matchAnyOpToken try each option"))
+    (let ((s (hs-stream "bar + baz")))
+      (assert= (get (hs-stream-match-any s "foo" "bar" "baz") :value) "bar")
+      (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+")
+      (assert (nil? (hs-stream-match-any s "foo" "quux"))))
+  )
   (deftest "matchOpToken matches operators by value"
-    (error "SKIP (untranslated): matchOpToken matches operators by value"))
+    (let ((s (hs-stream "1 + 2")))
+      (assert= (get (hs-stream-match-type s "NUMBER") :value) "1")
+      (assert= (get (hs-stream-match-any-op s "-" "+") :value) "+"))
+  )
   (deftest "matchToken consumes and returns on match"
-    (error "SKIP (untranslated): matchToken consumes and returns on match"))
+    (let ((s (hs-stream "foo bar baz")))
+      (assert= (get (hs-stream-match s "foo") :value) "foo")
+      (assert (nil? (hs-stream-match s "baz")))
+      (assert= (get (hs-stream-current s) :value) "bar")
+      (assert= (get (hs-stream-match s "bar") :value) "bar"))
+  )
   (deftest "matchToken honors the follow set"
-    (error "SKIP (untranslated): matchToken honors the follow set"))
+    (let ((s (hs-stream "and or not")))
+      (hs-stream-push-follow! s "and")
+      (assert (nil? (hs-stream-match s "and")))
+      (hs-stream-pop-follow! s)
+      (assert= (get (hs-stream-match s "and") :value) "and"))
+  )
   (deftest "matchTokenType matches by type"
-    (error "SKIP (untranslated): matchTokenType matches by type"))
+    (let ((s (hs-stream "foo 42")))
+      (assert= (get (hs-stream-match-type s "IDENTIFIER") :value) "foo")
+      (assert (nil? (hs-stream-match-type s "STRING")))
+      (assert= (get (hs-stream-match-type s "STRING" "NUMBER") :value) "42"))
+  )
   (deftest "peekToken skips whitespace when looking ahead"
-    (error "SKIP (untranslated): peekToken skips whitespace when looking ahead"))
+    (let ((s (hs-stream "for x in items")))
+      (assert= (get (hs-stream-peek s "for" 0) :value) "for")
+      (assert= (get (hs-stream-peek s "x" 1) :value) "x")
+      (assert= (get (hs-stream-peek s "in" 2) :value) "in")
+      (assert= (get (hs-stream-peek s "items" 3) :value) "items")
+      (assert (nil? (hs-stream-peek s "wrong" 1))))
+  )
   (deftest "pushFollow/popFollow nest follow-set boundaries"
-    (error "SKIP (untranslated): pushFollow/popFollow nest follow-set boundaries"))
+    (let ((s (hs-stream "and or not")))
+      (hs-stream-push-follow! s "and")
+      (hs-stream-push-follow! s "or")
+      (assert (nil? (hs-stream-match s "and")))
+      (hs-stream-pop-follow! s)
+      (assert (nil? (hs-stream-match s "and")))
+      (hs-stream-pop-follow! s)
+      (assert= (get (hs-stream-match s "and") :value) "and"))
+  )
   (deftest "pushFollows/popFollows push and pop in bulk"
-    (error "SKIP (untranslated): pushFollows/popFollows push and pop in bulk"))
+    (let ((s (hs-stream "and or not")))
+      (hs-stream-push-follows! s (list "and" "or"))
+      (assert (nil? (hs-stream-match s "and")))
+      (assert (nil? (hs-stream-match s "or")))
+      (hs-stream-pop-follows! s 2)
+      (assert= (get (hs-stream-match s "and") :value) "and"))
+  )
 )
 
 ;; ── def (27 tests) ──