HS E37 step 5: hs-tokenize-template + template routing in hs-tokens-of
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 12s
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 12s
Add hs-tokenize-template: scans " as single STRING token, ${ ... }
as dollar+brace+inner-tokens (inner tokenized with hs-tokenize), and
} as brace-close. Update hs-tokens-of to call hs-tokenize-template
when :template keyword arg is passed. Unlocks tests 1 and 15.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2611,7 +2611,8 @@
|
|||||||
(fn
|
(fn
|
||||||
(src &rest rest)
|
(src &rest rest)
|
||||||
(let
|
(let
|
||||||
((raw (hs-tokenize src)))
|
((template? (and (> (len rest) 0) (= (first rest) :template)))
|
||||||
|
(raw (if template? (hs-tokenize-template src) (hs-tokenize src))))
|
||||||
{:source src
|
{:source src
|
||||||
:list (map hs-raw->api-token raw)
|
:list (map hs-raw->api-token raw)
|
||||||
:pos 0})))
|
:pos 0})))
|
||||||
|
|||||||
@@ -666,4 +666,69 @@
|
|||||||
:else (do (hs-advance! 1) (scan!)))))))
|
:else (do (hs-advance! 1) (scan!)))))))
|
||||||
(scan!)
|
(scan!)
|
||||||
(hs-emit! "eof" nil pos)
|
(hs-emit! "eof" nil pos)
|
||||||
|
tokens)))
|
||||||
|
|
||||||
|
;; ── Template-mode tokenizer (E37 API) ────────────────────────────────
|
||||||
|
;; Used by hs-tokens-of when :template flag is set.
|
||||||
|
;; Emits outer " chars as single STRING tokens; ${ ... } as $ { <inner-tokens> };
|
||||||
|
;; inner content is tokenized with the regular hs-tokenize.
|
||||||
|
|
||||||
|
(define
|
||||||
|
hs-tokenize-template
|
||||||
|
(fn
|
||||||
|
(src)
|
||||||
|
(let
|
||||||
|
((tokens (list)) (pos 0) (src-len (len src)))
|
||||||
|
(define t-cur (fn () (if (< pos src-len) (nth src pos) nil)))
|
||||||
|
(define t-peek (fn (n) (if (< (+ pos n) src-len) (nth src (+ pos n)) nil)))
|
||||||
|
(define t-advance! (fn (n) (set! pos (+ pos n))))
|
||||||
|
(define t-emit! (fn (type value) (append! tokens (hs-make-token type value pos))))
|
||||||
|
(define
|
||||||
|
scan-to-close!
|
||||||
|
(fn
|
||||||
|
(depth)
|
||||||
|
(when
|
||||||
|
(and (< pos src-len) (> depth 0))
|
||||||
|
(cond
|
||||||
|
(= (t-cur) "{")
|
||||||
|
(do (t-advance! 1) (scan-to-close! (+ depth 1)))
|
||||||
|
(= (t-cur) "}")
|
||||||
|
(when (> (- depth 1) 0) (t-advance! 1) (scan-to-close! (- depth 1)))
|
||||||
|
:else (do (t-advance! 1) (scan-to-close! depth))))))
|
||||||
|
(define
|
||||||
|
scan-template!
|
||||||
|
(fn
|
||||||
|
()
|
||||||
|
(when
|
||||||
|
(< pos src-len)
|
||||||
|
(let
|
||||||
|
((ch (t-cur)))
|
||||||
|
(cond
|
||||||
|
(= ch "\"")
|
||||||
|
(do (t-emit! "string" "\"") (t-advance! 1) (scan-template!))
|
||||||
|
(and (= ch "$") (= (t-peek 1) "{"))
|
||||||
|
(do
|
||||||
|
(t-emit! "op" "$")
|
||||||
|
(t-advance! 1)
|
||||||
|
(t-emit! "brace-open" "{")
|
||||||
|
(t-advance! 1)
|
||||||
|
(let
|
||||||
|
((inner-start pos))
|
||||||
|
(scan-to-close! 1)
|
||||||
|
(let
|
||||||
|
((inner-src (slice src inner-start pos))
|
||||||
|
(inner-toks (hs-tokenize inner-src)))
|
||||||
|
(for-each
|
||||||
|
(fn (tok)
|
||||||
|
(when (not (= (get tok "type") "eof"))
|
||||||
|
(append! tokens tok)))
|
||||||
|
inner-toks))
|
||||||
|
(t-emit! "brace-close" "}")
|
||||||
|
(when (< pos src-len) (t-advance! 1)))
|
||||||
|
(scan-template!))
|
||||||
|
(hs-ws? ch)
|
||||||
|
(do (t-advance! 1) (scan-template!))
|
||||||
|
:else (do (t-advance! 1) (scan-template!)))))))
|
||||||
|
(scan-template!)
|
||||||
|
(t-emit! "eof" nil pos)
|
||||||
tokens)))
|
tokens)))
|
||||||
Reference in New Issue
Block a user