From 3003c8a0693c4fa6ace01206abfe12d73df511ff Mon Sep 17 00:00:00 2001
From: giles <giles.bradshaw@rose-ash.com>
Date: Sat, 25 Apr 2026 19:08:38 +0000
Subject: [PATCH] HS E37 step 5: hs-tokenize-template + template routing in
 hs-tokens-of

Add hs-tokenize-template: scans " as single STRING token, ${ ... }
as dollar+brace+inner-tokens (inner tokenized with hs-tokenize), and
} as brace-close. Update hs-tokens-of to call hs-tokenize-template
when :template keyword arg is passed. Unlocks tests 1 and 15.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 lib/hyperscript/runtime.sx   |  3 +-
 lib/hyperscript/tokenizer.sx | 65 ++++++++++++++++++++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/lib/hyperscript/runtime.sx b/lib/hyperscript/runtime.sx
index e851c0d6..7369dab0 100644
--- a/lib/hyperscript/runtime.sx
+++ b/lib/hyperscript/runtime.sx
@@ -2611,7 +2611,8 @@
   (fn
     (src &rest rest)
     (let
-      ((raw (hs-tokenize src)))
+      ((template? (and (> (len rest) 0) (= (first rest) :template)))
+       (raw (if template? (hs-tokenize-template src) (hs-tokenize src))))
       {:source src
        :list   (map hs-raw->api-token raw)
        :pos    0})))
diff --git a/lib/hyperscript/tokenizer.sx b/lib/hyperscript/tokenizer.sx
index 98933329..ca61680a 100644
--- a/lib/hyperscript/tokenizer.sx
+++ b/lib/hyperscript/tokenizer.sx
@@ -666,4 +666,69 @@
                 :else (do (hs-advance! 1) (scan!)))))))
       (scan!)
       (hs-emit! "eof" nil pos)
+      tokens)))
+
+;; ── Template-mode tokenizer (E37 API) ────────────────────────────────
+;; Used by hs-tokens-of when :template flag is set.
+;; Emits outer " chars as single STRING tokens; ${ ... } as $ { <inner-tokens> };
+;; inner content is tokenized with the regular hs-tokenize.
+
+(define
+  hs-tokenize-template
+  (fn
+    (src)
+    (let
+      ((tokens (list)) (pos 0) (src-len (len src)))
+      (define t-cur (fn () (if (< pos src-len) (nth src pos) nil)))
+      (define t-peek (fn (n) (if (< (+ pos n) src-len) (nth src (+ pos n)) nil)))
+      (define t-advance! (fn (n) (set! pos (+ pos n))))
+      (define t-emit! (fn (type value) (append! tokens (hs-make-token type value pos))))
+      (define
+        scan-to-close!
+        (fn
+          (depth)
+          (when
+            (and (< pos src-len) (> depth 0))
+            (cond
+              (= (t-cur) "{")
+              (do (t-advance! 1) (scan-to-close! (+ depth 1)))
+              (= (t-cur) "}")
+              (when (> (- depth 1) 0) (t-advance! 1) (scan-to-close! (- depth 1)))
+              :else (do (t-advance! 1) (scan-to-close! depth))))))
+      (define
+        scan-template!
+        (fn
+          ()
+          (when
+            (< pos src-len)
+            (let
+              ((ch (t-cur)))
+              (cond
+                (= ch "\"")
+                (do (t-emit! "string" "\"") (t-advance! 1) (scan-template!))
+                (and (= ch "$") (= (t-peek 1) "{"))
+                (do
+                  (t-emit! "op" "$")
+                  (t-advance! 1)
+                  (t-emit! "brace-open" "{")
+                  (t-advance! 1)
+                  (let
+                    ((inner-start pos))
+                    (scan-to-close! 1)
+                    (let
+                      ((inner-src (slice src inner-start pos))
+                       (inner-toks (hs-tokenize inner-src)))
+                      (for-each
+                        (fn (tok)
+                          (when (not (= (get tok "type") "eof"))
+                            (append! tokens tok)))
+                        inner-toks))
+                    (t-emit! "brace-close" "}")
+                    (when (< pos src-len) (t-advance! 1)))
+                  (scan-template!))
+                (hs-ws? ch)
+                (do (t-advance! 1) (scan-template!))
+                :else (do (t-advance! 1) (scan-template!)))))))
+      (scan-template!)
+      (t-emit! "eof" nil pos)
       tokens)))
\ No newline at end of file