From 8ac669c7396f5f504edfe2999ee6e3903c0cd70c Mon Sep 17 00:00:00 2001
From: giles <giles.bradshaw@rose-ash.com>
Date: Sat, 25 Apr 2026 18:56:26 +0000
Subject: [PATCH] HS E37 step 1: hs-api-tokens + stream/token helpers in
 runtime.sx
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add hs-eof-sentinel, hs-op-type, hs-raw->api-token, hs-tokens-of,
hs-stream-token, hs-stream-consume, hs-stream-has-more, and the
three token accessors (hs-token-type, hs-token-value, hs-token-op?).
No test delta yet — API-only, generator comes in step 6.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 lib/hyperscript/runtime.sx | 119 +++++++++++++++++++++++++++++++++++++
 1 file changed, 119 insertions(+)

diff --git a/lib/hyperscript/runtime.sx b/lib/hyperscript/runtime.sx
index 4daa71d9..e851c0d6 100644
--- a/lib/hyperscript/runtime.sx
+++ b/lib/hyperscript/runtime.sx
@@ -2525,3 +2525,122 @@
   (fn
     (fn-name args)
     (let ((fn (host-global fn-name))) (if fn (host-call-fn fn args) nil))))
+
+;; ── E37 Tokenizer-as-API ─────────────────────────────────────────────
+
+(define hs-eof-sentinel (fn () {:type "EOF" :value "<<<EOF>>>" :op false}))
+
+(define
+  hs-op-type
+  (fn
+    (val)
+    (cond
+      ((= val "+")   "PLUS")
+      ((= val "-")   "MINUS")
+      ((= val "*")   "MULTIPLY")
+      ((= val "/")   "SLASH")
+      ((= val "%")   "PERCENT")
+      ((= val "|")   "PIPE")
+      ((= val "!")   "EXCLAMATION")
+      ((= val "?")   "QUESTION")
+      ((= val "#")   "POUND")
+      ((= val "&")   "AMPERSAND")
+      ((= val ";")   "SEMI")
+      ((= val "=")   "EQUALS")
+      ((= val "<")   "L_ANG")
+      ((= val ">")   "R_ANG")
+      ((= val "<=")  "LTE_ANG")
+      ((= val ">=")  "GTE_ANG")
+      ((= val "==")  "EQ")
+      ((= val "===") "EQQ")
+      ((= val "\\")  "BACKSLASH")
+      (true          (str "OP_" val)))))
+
+(define
+  hs-raw->api-token
+  (fn
+    (tok)
+    (let
+      ((raw-type (get tok "type"))
+       (raw-val  (get tok "value")))
+      (let
+        ((up-type
+           (cond
+             ((or (= raw-type "ident") (= raw-type "keyword")) "IDENTIFIER")
+             ((= raw-type "number")        "NUMBER")
+             ((= raw-type "string")        "STRING")
+             ((= raw-type "class")         "CLASS_REF")
+             ((= raw-type "id")            "ID_REF")
+             ((= raw-type "attr")          "ATTRIBUTE_REF")
+             ((= raw-type "style")         "STYLE_REF")
+             ((= raw-type "selector")      "QUERY_REF")
+             ((= raw-type "eof")           "EOF")
+             ((= raw-type "paren-open")    "L_PAREN")
+             ((= raw-type "paren-close")   "R_PAREN")
+             ((= raw-type "bracket-open")  "L_BRACKET")
+             ((= raw-type "bracket-close") "R_BRACKET")
+             ((= raw-type "brace-open")    "L_BRACE")
+             ((= raw-type "brace-close")   "R_BRACE")
+             ((= raw-type "comma")         "COMMA")
+             ((= raw-type "dot")           "PERIOD")
+             ((= raw-type "colon")         "COLON")
+             ((= raw-type "op")            (hs-op-type raw-val))
+             (true                         (str "UNKNOWN_" raw-type))))
+         (up-val
+           (cond
+             ((= raw-type "class") (str "." raw-val))
+             ((= raw-type "id")    (str "#" raw-val))
+             ((= raw-type "eof")   "<<<EOF>>>")
+             (true                 raw-val)))
+         (is-op
+           (or
+             (= raw-type "paren-open")
+             (= raw-type "paren-close")
+             (= raw-type "bracket-open")
+             (= raw-type "bracket-close")
+             (= raw-type "brace-open")
+             (= raw-type "brace-close")
+             (= raw-type "comma")
+             (= raw-type "dot")
+             (= raw-type "colon")
+             (= raw-type "op"))))
+        {:type up-type :value up-val :op is-op}))))
+
+(define
+  hs-tokens-of
+  (fn
+    (src &rest rest)
+    (let
+      ((raw (hs-tokenize src)))
+      {:source src
+       :list   (map hs-raw->api-token raw)
+       :pos    0})))
+
+(define
+  hs-stream-token
+  (fn
+    (s i)
+    (let
+      ((lst (get s "list"))
+       (pos (get s "pos")))
+      (or (nth lst (+ pos i))
+          (hs-eof-sentinel)))))
+
+(define
+  hs-stream-consume
+  (fn
+    (s)
+    (let
+      ((tok (hs-stream-token s 0)))
+      (when
+        (not (= (get tok "type") "EOF"))
+        (dict-set! s "pos" (+ (get s "pos") 1)))
+      tok)))
+
+(define
+  hs-stream-has-more
+  (fn (s) (not (= (get (hs-stream-token s 0) "type") "EOF"))))
+
+(define hs-token-type  (fn (tok) (get tok "type")))
+(define hs-token-value (fn (tok) (get tok "value")))
+(define hs-token-op?   (fn (tok) (get tok "op")))