HS: E37 tokenizer API (+17 tests)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 15s
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 15s
Implements hs-tokens-of, hs-stream-token, hs-stream-consume, hs-stream-has-more, hs-token-type, hs-token-value, hs-token-op?, hs-raw->api-token, hs-eof-sentinel in runtime.sx. Tokenizer emits whitespace tokens after the first content token; stream functions skip them for look-ahead and consume. Parser filters whitespace tokens at hs-parse entry. Dot/hash after close brackets split into PERIOD/POUND + IDENTIFIER. Template escape \$ produces literal $. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -465,7 +465,12 @@
|
||||
scan!
|
||||
(fn
|
||||
()
|
||||
(skip-ws!)
|
||||
(let
|
||||
((ws-start pos))
|
||||
(skip-ws!)
|
||||
(when
|
||||
(and (> (len tokens) 0) (> pos ws-start))
|
||||
(hs-emit! "whitespace" (slice src ws-start pos) ws-start)))
|
||||
(when
|
||||
(< pos src-len)
|
||||
(let
|
||||
@@ -489,6 +494,15 @@
|
||||
(do (hs-emit! "selector" (read-selector) start) (scan!))
|
||||
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
|
||||
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!))
|
||||
(and
|
||||
(= ch ".")
|
||||
(< (+ pos 1) src-len)
|
||||
(or (hs-letter? (hs-peek 1)) (= (hs-peek 1) "-") (= (hs-peek 1) "_"))
|
||||
(> (len tokens) 0)
|
||||
(let
|
||||
((lt (dict-get (nth tokens (- (len tokens) 1)) :type)))
|
||||
(or (= lt "paren-close") (= lt "brace-close") (= lt "bracket-close"))))
|
||||
(do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!))
|
||||
(and
|
||||
(= ch ".")
|
||||
(< (+ pos 1) src-len)
|
||||
@@ -500,6 +514,15 @@
|
||||
(hs-advance! 1)
|
||||
(hs-emit! "class" (read-class-name pos) start)
|
||||
(scan!))
|
||||
(and
|
||||
(= ch "#")
|
||||
(< (+ pos 1) src-len)
|
||||
(hs-ident-start? (hs-peek 1))
|
||||
(> (len tokens) 0)
|
||||
(let
|
||||
((lt (dict-get (nth tokens (- (len tokens) 1)) :type)))
|
||||
(or (= lt "paren-close") (= lt "brace-close") (= lt "bracket-close"))))
|
||||
(do (hs-emit! "op" "#" start) (hs-advance! 1) (scan!))
|
||||
(and
|
||||
(= ch "#")
|
||||
(< (+ pos 1) src-len)
|
||||
|
||||
Reference in New Issue
Block a user