coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HS: tokenizer tracks :end and :line

Browse Source
This commit is contained in:
giles
2026-04-25 18:54:59 +00:00
parent 912649c426
commit 20a643806b
2 changed files with 106 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
lib/hyperscript/tokenizer.sx
View File

@@ -1,6 +1,6 @@
;; _hyperscript tokenizer — produces token stream from hyperscript source
;;
;; Tokens: {:type T :value V :pos P}
;; Tokens: {:type T :value V :pos P :end E :line L}
;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style"
;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open"
;; "bracket-close" "brace-open" "brace-close" "comma" "colon"
@@ -8,7 +8,7 @@
;; ── Token constructor ─────────────────────────────────────────────
(define hs-make-token (fn (type value pos) {:pos pos :value value :type type}))
(define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type}))
;; ── Character predicates ──────────────────────────────────────────
@@ -198,14 +198,22 @@
(fn
(src)
(let
((tokens (list)) (pos 0) (src-len (len src)))
((tokens (list)) (pos 0) (src-len (len src)) (current-line 1))
(define
hs-peek
(fn
(offset)
(if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
(define hs-cur (fn () (hs-peek 0)))
(define hs-advance! (fn (n) (set! pos (+ pos n))))
(define
hs-advance!
(fn
(n)
(when
(> n 0)
(when (= (hs-cur) "\n") (set! current-line (+ current-line 1)))
(set! pos (+ pos 1))
(hs-advance! (- n 1)))))
(define
skip-ws!
(fn
@@ -427,8 +435,8 @@
(define
hs-emit!
(fn
(type value start)
(append! tokens (hs-make-token type value start))))
(type value start start-line)
(append! tokens (hs-make-token type value start pos start-line))))
(define
scan!
(fn
@@ -437,7 +445,7 @@
(when
(< pos src-len)
(let
((ch (hs-cur)) (start pos))
((ch (hs-cur)) (start pos) (start-line current-line))
(cond
(and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-"))
(do (hs-advance! 2) (skip-comment!) (scan!))
@@ -454,9 +462,9 @@
(= (hs-peek 1) "[")
(= (hs-peek 1) "*")
(= (hs-peek 1) ":")))
(do (hs-emit! "selector" (read-selector) start) (scan!))
(do (hs-emit! "selector" (read-selector) start start-line) (scan!))
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!))
(do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!))
(and
(= ch ".")
(< (+ pos 1) src-len)
@@ -466,7 +474,7 @@
(= (hs-peek 1) "_")))
(do
(hs-advance! 1)
(hs-emit! "class" (read-class-name pos) start)
(hs-emit! "class" (read-class-name pos) start start-line)
(scan!))
(and
(= ch "#")
@@ -474,7 +482,7 @@
(hs-ident-start? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "id" (read-ident pos) start)
(hs-emit! "id" (read-ident pos) start start-line)
(scan!))
(and
(= ch "@")
@@ -482,7 +490,7 @@
(hs-ident-char? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "attr" (read-ident pos) start)
(hs-emit! "attr" (read-ident pos) start start-line)
(scan!))
(and
(= ch "^")
@@ -490,7 +498,7 @@
(hs-ident-char? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "hat" (read-ident pos) start)
(hs-emit! "hat" (read-ident pos) start start-line)
(scan!))
(and
(= ch "~")
@@ -498,7 +506,7 @@
(hs-letter? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "component" (str "~" (read-ident pos)) start)
(hs-emit! "component" (str "~" (read-ident pos)) start start-line)
(scan!))
(and
(= ch "*")
@@ -506,7 +514,7 @@
(hs-letter? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "style" (read-ident pos) start)
(hs-emit! "style" (read-ident pos) start start-line)
(scan!))
(and
(= ch ":")
@@ -514,7 +522,7 @@
(hs-ident-start? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "local" (read-ident pos) start)
(hs-emit! "local" (read-ident pos) start start-line)
(scan!))
(or
(= ch "\"")
@@ -527,11 +535,11 @@
(or
(>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2))))))))
(do (hs-emit! "string" (read-string ch) start) (scan!))
(do (hs-emit! "string" (read-string ch) start start-line) (scan!))
(= ch "`")
(do (hs-emit! "template" (read-template) start) (scan!))
(do (hs-emit! "template" (read-template) start start-line) (scan!))
(hs-digit? ch)
(do (hs-emit! "number" (read-number start) start) (scan!))
(do (hs-emit! "number" (read-number start) start start-line) (scan!))
(hs-ident-start? ch)
(do
(let
@@ -539,7 +547,8 @@
(hs-emit!
(if (hs-keyword? word) "keyword" "ident")
word
start))
start
start-line))
(scan!))
(and
(or (= ch "=") (= ch "!") (= ch "<") (= ch ">"))
@@ -551,8 +560,8 @@
(or (= ch "=") (= ch "!"))
(< (+ pos 2) src-len)
(= (hs-peek 2) "="))
(do (hs-emit! "op" (str ch "==") start) (hs-advance! 3))
(do (hs-emit! "op" (str ch "=") start) (hs-advance! 2)))
(do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line))
(do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line)))
(scan!))
(and
(= ch "'")
@@ -561,66 +570,66 @@
(or
(>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2)))))
(do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!))
(do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!))
(= ch "(")
(do
(hs-emit! "paren-open" "(" start)
(hs-advance! 1)
(hs-emit! "paren-open" "(" start start-line)
(scan!))
(= ch ")")
(do
(hs-emit! "paren-close" ")" start)
(hs-advance! 1)
(hs-emit! "paren-close" ")" start start-line)
(scan!))
(= ch "[")
(do
(hs-emit! "bracket-open" "[" start)
(hs-advance! 1)
(hs-emit! "bracket-open" "[" start start-line)
(scan!))
(= ch "]")
(do
(hs-emit! "bracket-close" "]" start)
(hs-advance! 1)
(hs-emit! "bracket-close" "]" start start-line)
(scan!))
(= ch "{")
(do
(hs-emit! "brace-open" "{" start)
(hs-advance! 1)
(hs-emit! "brace-open" "{" start start-line)
(scan!))
(= ch "}")
(do
(hs-emit! "brace-close" "}" start)
(hs-advance! 1)
(hs-emit! "brace-close" "}" start start-line)
(scan!))
(= ch ",")
(do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!))
(= ch "+")
(do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!))
(= ch "-")
(do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!))
(= ch "/")
(do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!))
(= ch "=")
(do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!))
(= ch "<")
(do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!))
(= ch ">")
(do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!))
(= ch "!")
(do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!))
(= ch "*")
(do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!))
(= ch "%")
(do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!))
(= ch ".")
(do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!))
(= ch "\\")
(do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!))
(= ch ":")
(do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!))
(= ch "|")
(do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!))
:else (do (hs-advance! 1) (scan!)))))))
(scan!)
(hs-emit! "eof" nil pos)
(hs-emit! "eof" nil pos current-line)
tokens)))

97
shared/static/wasm/sx/hs-tokenizer.sx
View File

@@ -1,6 +1,6 @@
;; _hyperscript tokenizer — produces token stream from hyperscript source
;;
;; Tokens: {:type T :value V :pos P}
;; Tokens: {:type T :value V :pos P :end E :line L}
;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style"
;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open"
;; "bracket-close" "brace-open" "brace-close" "comma" "colon"
@@ -8,7 +8,7 @@
;; ── Token constructor ─────────────────────────────────────────────
(define hs-make-token (fn (type value pos) {:pos pos :value value :type type}))
(define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type}))
;; ── Character predicates ──────────────────────────────────────────
@@ -198,14 +198,22 @@
(fn
(src)
(let
((tokens (list)) (pos 0) (src-len (len src)))
((tokens (list)) (pos 0) (src-len (len src)) (current-line 1))
(define
hs-peek
(fn
(offset)
(if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
(define hs-cur (fn () (hs-peek 0)))
(define hs-advance! (fn (n) (set! pos (+ pos n))))
(define
hs-advance!
(fn
(n)
(when
(> n 0)
(when (= (hs-cur) "\n") (set! current-line (+ current-line 1)))
(set! pos (+ pos 1))
(hs-advance! (- n 1)))))
(define
skip-ws!
(fn
@@ -427,8 +435,8 @@
(define
hs-emit!
(fn
(type value start)
(append! tokens (hs-make-token type value start))))
(type value start start-line)
(append! tokens (hs-make-token type value start pos start-line))))
(define
scan!
(fn
@@ -437,7 +445,7 @@
(when
(< pos src-len)
(let
((ch (hs-cur)) (start pos))
((ch (hs-cur)) (start pos) (start-line current-line))
(cond
(and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-"))
(do (hs-advance! 2) (skip-comment!) (scan!))
@@ -454,9 +462,9 @@
(= (hs-peek 1) "[")
(= (hs-peek 1) "*")
(= (hs-peek 1) ":")))
(do (hs-emit! "selector" (read-selector) start) (scan!))
(do (hs-emit! "selector" (read-selector) start start-line) (scan!))
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!))
(do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!))
(and
(= ch ".")
(< (+ pos 1) src-len)
@@ -466,7 +474,7 @@
(= (hs-peek 1) "_")))
(do
(hs-advance! 1)
(hs-emit! "class" (read-class-name pos) start)
(hs-emit! "class" (read-class-name pos) start start-line)
(scan!))
(and
(= ch "#")
@@ -474,7 +482,7 @@
(hs-ident-start? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "id" (read-ident pos) start)
(hs-emit! "id" (read-ident pos) start start-line)
(scan!))
(and
(= ch "@")
@@ -482,7 +490,7 @@
(hs-ident-char? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "attr" (read-ident pos) start)
(hs-emit! "attr" (read-ident pos) start start-line)
(scan!))
(and
(= ch "^")
@@ -490,7 +498,7 @@
(hs-ident-char? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "hat" (read-ident pos) start)
(hs-emit! "hat" (read-ident pos) start start-line)
(scan!))
(and
(= ch "~")
@@ -498,7 +506,7 @@
(hs-letter? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "component" (str "~" (read-ident pos)) start)
(hs-emit! "component" (str "~" (read-ident pos)) start start-line)
(scan!))
(and
(= ch "*")
@@ -506,7 +514,7 @@
(hs-letter? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "style" (read-ident pos) start)
(hs-emit! "style" (read-ident pos) start start-line)
(scan!))
(and
(= ch ":")
@@ -514,7 +522,7 @@
(hs-ident-start? (hs-peek 1)))
(do
(hs-advance! 1)
(hs-emit! "local" (read-ident pos) start)
(hs-emit! "local" (read-ident pos) start start-line)
(scan!))
(or
(= ch "\"")
@@ -527,11 +535,11 @@
(or
(>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2))))))))
(do (hs-emit! "string" (read-string ch) start) (scan!))
(do (hs-emit! "string" (read-string ch) start start-line) (scan!))
(= ch "`")
(do (hs-emit! "template" (read-template) start) (scan!))
(do (hs-emit! "template" (read-template) start start-line) (scan!))
(hs-digit? ch)
(do (hs-emit! "number" (read-number start) start) (scan!))
(do (hs-emit! "number" (read-number start) start start-line) (scan!))
(hs-ident-start? ch)
(do
(let
@@ -539,7 +547,8 @@
(hs-emit!
(if (hs-keyword? word) "keyword" "ident")
word
start))
start
start-line))
(scan!))
(and
(or (= ch "=") (= ch "!") (= ch "<") (= ch ">"))
@@ -551,8 +560,8 @@
(or (= ch "=") (= ch "!"))
(< (+ pos 2) src-len)
(= (hs-peek 2) "="))
(do (hs-emit! "op" (str ch "==") start) (hs-advance! 3))
(do (hs-emit! "op" (str ch "=") start) (hs-advance! 2)))
(do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line))
(do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line)))
(scan!))
(and
(= ch "'")
@@ -561,66 +570,66 @@
(or
(>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2)))))
(do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!))
(do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!))
(= ch "(")
(do
(hs-emit! "paren-open" "(" start)
(hs-advance! 1)
(hs-emit! "paren-open" "(" start start-line)
(scan!))
(= ch ")")
(do
(hs-emit! "paren-close" ")" start)
(hs-advance! 1)
(hs-emit! "paren-close" ")" start start-line)
(scan!))
(= ch "[")
(do
(hs-emit! "bracket-open" "[" start)
(hs-advance! 1)
(hs-emit! "bracket-open" "[" start start-line)
(scan!))
(= ch "]")
(do
(hs-emit! "bracket-close" "]" start)
(hs-advance! 1)
(hs-emit! "bracket-close" "]" start start-line)
(scan!))
(= ch "{")
(do
(hs-emit! "brace-open" "{" start)
(hs-advance! 1)
(hs-emit! "brace-open" "{" start start-line)
(scan!))
(= ch "}")
(do
(hs-emit! "brace-close" "}" start)
(hs-advance! 1)
(hs-emit! "brace-close" "}" start start-line)
(scan!))
(= ch ",")
(do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!))
(= ch "+")
(do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!))
(= ch "-")
(do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!))
(= ch "/")
(do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!))
(= ch "=")
(do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!))
(= ch "<")
(do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!))
(= ch ">")
(do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!))
(= ch "!")
(do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!))
(= ch "*")
(do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!))
(= ch "%")
(do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!))
(= ch ".")
(do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!))
(= ch "\\")
(do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!))
(= ch ":")
(do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!))
(= ch "|")
(do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!))
(do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!))
:else (do (hs-advance! 1) (scan!)))))))
(scan!)
(hs-emit! "eof" nil pos)
(hs-emit! "eof" nil pos current-line)
tokens)))