HS E37: tokenizer-as-API 17/17 (+fixes)

- runtime.sx: fix extra ) in hs-tokens-of (parse error); add hs-eof-sentinel,
  hs-raw->api-token, hs-normalize-raw-tokens, hs-tokens-of, stream helpers,
  hs-token-type/value/op?; add \$ escape to hs-template
- tokenizer.sx: fix read-number double-dot bug (1.1.1 → 3 tokens); fix t-emit!
  eof call (3→2 args); add bare $ case to scan-template!
- compiler.sx: add \$ escape to tpl-collect template interpolation
- generate-sx-tests.py: preserve \$ in process_hs_val; add generate_tokenizer_test
- regen spec/tests/test-hyperscript-behavioral.sx: 17 tokenizer tests generated
- plans/hs-conformance-to-100.md: row 37 marked done +17

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

shared/static/wasm/sx/hs-runtime.sx

@@ -2021,6 +2021,12 @@
             (let
               ((ch (nth raw i)))
               (if
+                (and (= ch "\\") (< (+ i 1) n) (= (nth raw (+ i 1)) "$"))
+                (do
+                  (set! result (str result "$"))
+                  (set! i (+ i 2))
+                  (tpl-loop))
+                (if
                 (and (= ch "$") (< (+ i 1) n))
                 (if
                   (= (nth raw (+ i 1)) "{")
@@ -2089,7 +2095,7 @@
                 (do
                   (set! result (str result ch))
                   (set! i (+ i 1))
-                  (tpl-loop)))))))
+                  (tpl-loop))))))))
       (do (tpl-loop) result))))
 
 (define
@@ -2525,3 +2531,188 @@
   (fn
     (fn-name args)
     (let ((fn (host-global fn-name))) (if fn (host-call-fn fn args) nil))))
+
+;; ── E37 Tokenizer-as-API ─────────────────────────────────────────────
+
+(define hs-eof-sentinel (fn () {:type "EOF" :value "<<<EOF>>>" :op false}))
+
+(define
+  hs-op-type
+  (fn
+    (val)
+    (cond
+      ((= val "+")   "PLUS")
+      ((= val "-")   "MINUS")
+      ((= val "*")   "MULTIPLY")
+      ((= val "/")   "SLASH")
+      ((= val "%")   "PERCENT")
+      ((= val "|")   "PIPE")
+      ((= val "!")   "EXCLAMATION")
+      ((= val "?")   "QUESTION")
+      ((= val "#")   "POUND")
+      ((= val "&")   "AMPERSAND")
+      ((= val ";")   "SEMI")
+      ((= val "=")   "EQUALS")
+      ((= val "<")   "L_ANG")
+      ((= val ">")   "R_ANG")
+      ((= val "<=")  "LTE_ANG")
+      ((= val ">=")  "GTE_ANG")
+      ((= val "==")  "EQ")
+      ((= val "===") "EQQ")
+      ((= val "\\")  "BACKSLASH")
+      (true          (str "OP_" val)))))
+
+(define
+  hs-raw->api-token
+  (fn
+    (tok)
+    (let
+      ((raw-type (get tok "type"))
+       (raw-val  (get tok "value")))
+      (let
+        ((up-type
+           (cond
+             ((or (= raw-type "ident") (= raw-type "keyword")) "IDENTIFIER")
+             ((= raw-type "number")        "NUMBER")
+             ((= raw-type "string")        "STRING")
+             ((= raw-type "class")         "CLASS_REF")
+             ((= raw-type "id")            "ID_REF")
+             ((= raw-type "attr")          "ATTRIBUTE_REF")
+             ((= raw-type "style")         "STYLE_REF")
+             ((= raw-type "selector")      "QUERY_REF")
+             ((= raw-type "eof")           "EOF")
+             ((= raw-type "paren-open")    "L_PAREN")
+             ((= raw-type "paren-close")   "R_PAREN")
+             ((= raw-type "bracket-open")  "L_BRACKET")
+             ((= raw-type "bracket-close") "R_BRACKET")
+             ((= raw-type "brace-open")    "L_BRACE")
+             ((= raw-type "brace-close")   "R_BRACE")
+             ((= raw-type "comma")         "COMMA")
+             ((= raw-type "dot")           "PERIOD")
+             ((= raw-type "colon")         "COLON")
+             ((= raw-type "op")            (hs-op-type raw-val))
+             (true                         (str "UNKNOWN_" raw-type))))
+         (up-val
+           (cond
+             ((= raw-type "class") (str "." raw-val))
+             ((= raw-type "id")    (str "#" raw-val))
+             ((= raw-type "eof")   "<<<EOF>>>")
+             (true                 raw-val)))
+         (is-op
+           (or
+             (= raw-type "paren-open")
+             (= raw-type "paren-close")
+             (= raw-type "bracket-open")
+             (= raw-type "bracket-close")
+             (= raw-type "brace-open")
+             (= raw-type "brace-close")
+             (= raw-type "comma")
+             (= raw-type "dot")
+             (= raw-type "colon")
+             (= raw-type "op"))))
+        {:type up-type :value up-val :op is-op}))))
+
+;; Expand "class" and "id" tokens that follow a closing bracket into
+;; separate dot/hash + ident tokens, matching upstream context-sensitive
+;; behaviour: after ) ] } the dot is property access, not a CLASS_REF.
+(define
+  hs-normalize-raw-tokens
+  (fn
+    (raw-real)
+    (let
+      ((result (list))
+       (prev-type nil))
+      (for-each
+        (fn
+          (tok)
+          (let
+            ((typ (get tok "type"))
+             (val (get tok "value"))
+             (tok-pos (get tok "pos")))
+            (if
+              (and
+                (or (= typ "class") (= typ "id"))
+                (or
+                  (= prev-type "paren-close")
+                  (= prev-type "bracket-close")
+                  (= prev-type "brace-close")))
+              (do
+                (if
+                  (= typ "class")
+                  (do
+                    (append! result {:type "dot" :value "." :pos tok-pos})
+                    (append! result {:type "ident" :value val :pos (+ tok-pos 1)}))
+                  (do
+                    (append! result {:type "op" :value "#" :pos tok-pos})
+                    (append! result {:type "ident" :value val :pos (+ tok-pos 1)})))
+                (set! prev-type "ident"))
+              (do
+                (append! result tok)
+                (set! prev-type typ)))))
+        raw-real)
+      result)))
+
+(define
+  hs-tokens-of
+  (fn
+    (src &rest rest)
+    (let
+      ((template? (and (> (len rest) 0) (= (first rest) :template)))
+       (raw (if template? (hs-tokenize-template src) (hs-tokenize src))))
+      (if
+        template?
+        {:source src :list (map hs-raw->api-token raw) :pos 0}
+        ;; Normal mode: filter EOF, context-normalise, add trailing-WS sentinel
+        (let
+          ((real (filter (fn (t) (not (= (get t "type") "eof"))) raw)))
+          (let
+            ((norm (hs-normalize-raw-tokens real)))
+            (let
+              ((api (map hs-raw->api-token norm)))
+              (let
+                ((with-sep
+                   (if
+                     (and
+                       (> (len norm) 0)
+                       (let
+                         ((last-tok (nth norm (- (len norm) 1))))
+                         (let
+                           ((end-pos
+                              (+ (get last-tok "pos")
+                                 (len (get last-tok "value")))))
+                           (and
+                             (< end-pos (len src))
+                             (hs-ws? (nth src end-pos))))))
+                     (append api (list {:type "WHITESPACE" :value " " :op false}))
+                     api)))
+                {:source src :list with-sep :pos 0}))))))))
+
+
+(define
+  hs-stream-token
+  (fn
+    (s i)
+    (let
+      ((lst (get s "list"))
+       (pos (get s "pos")))
+      (or (nth lst (+ pos i))
+          (hs-eof-sentinel)))))
+
+(define
+  hs-stream-consume
+  (fn
+    (s)
+    (let
+      ((tok (hs-stream-token s 0)))
+      (when
+        (not (= (get tok "type") "EOF"))
+        (dict-set! s "pos" (+ (get s "pos") 1)))
+      tok)))
+
+(define
+  hs-stream-has-more
+  (fn (s) (not (= (get (hs-stream-token s 0) "type") "EOF"))))
+
+(define hs-token-type  (fn (tok) (get tok "type")))
+(define hs-token-value (fn (tok) (get tok "value")))
+(define hs-token-op?   (fn (tok) (get tok "op")))