coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HS: tokenizer tracks :end and :line

Browse Source
This commit is contained in:
giles
2026-04-25 18:54:59 +00:00
parent 912649c426
commit 20a643806b
2 changed files with 106 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
lib/hyperscript/tokenizer.sx
View File

@@ -1,6 +1,6 @@
;; _hyperscript tokenizer — produces token stream from hyperscript source ;; _hyperscript tokenizer — produces token stream from hyperscript source
;; ;;
;; Tokens: {:type T :value V :pos P} ;; Tokens: {:type T :value V :pos P :end E :line L}
;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style" ;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style"
;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open" ;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open"
;; "bracket-close" "brace-open" "brace-close" "comma" "colon" ;; "bracket-close" "brace-open" "brace-close" "comma" "colon"
@@ -8,7 +8,7 @@
;; ── Token constructor ───────────────────────────────────────────── ;; ── Token constructor ─────────────────────────────────────────────
(define hs-make-token (fn (type value pos) {:pos pos :value value :type type})) (define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type}))
;; ── Character predicates ────────────────────────────────────────── ;; ── Character predicates ──────────────────────────────────────────
@@ -198,14 +198,22 @@
(fn (fn
(src) (src)
(let (let
((tokens (list)) (pos 0) (src-len (len src))) ((tokens (list)) (pos 0) (src-len (len src)) (current-line 1))
(define (define
hs-peek hs-peek
(fn (fn
(offset) (offset)
(if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil))) (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
(define hs-cur (fn () (hs-peek 0))) (define hs-cur (fn () (hs-peek 0)))
(define hs-advance! (fn (n) (set! pos (+ pos n)))) (define
hs-advance!
(fn
(n)
(when
(> n 0)
(when (= (hs-cur) "\n") (set! current-line (+ current-line 1)))
(set! pos (+ pos 1))
(hs-advance! (- n 1)))))
(define (define
skip-ws! skip-ws!
(fn (fn
@@ -427,8 +435,8 @@
(define (define
hs-emit! hs-emit!
(fn (fn
(type value start) (type value start start-line)
(append! tokens (hs-make-token type value start)))) (append! tokens (hs-make-token type value start pos start-line))))
(define (define
scan! scan!
(fn (fn
@@ -437,7 +445,7 @@
(when (when
(< pos src-len) (< pos src-len)
(let (let
((ch (hs-cur)) (start pos)) ((ch (hs-cur)) (start pos) (start-line current-line))
(cond (cond
(and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-")) (and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-"))
(do (hs-advance! 2) (skip-comment!) (scan!)) (do (hs-advance! 2) (skip-comment!) (scan!))
@@ -454,9 +462,9 @@
(= (hs-peek 1) "[") (= (hs-peek 1) "[")
(= (hs-peek 1) "*") (= (hs-peek 1) "*")
(= (hs-peek 1) ":"))) (= (hs-peek 1) ":")))
(do (hs-emit! "selector" (read-selector) start) (scan!)) (do (hs-emit! "selector" (read-selector) start start-line) (scan!))
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) ".")) (and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!)) (do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!))
(and (and
(= ch ".") (= ch ".")
(< (+ pos 1) src-len) (< (+ pos 1) src-len)
@@ -466,7 +474,7 @@
(= (hs-peek 1) "_"))) (= (hs-peek 1) "_")))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "class" (read-class-name pos) start) (hs-emit! "class" (read-class-name pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "#") (= ch "#")
@@ -474,7 +482,7 @@
(hs-ident-start? (hs-peek 1))) (hs-ident-start? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "id" (read-ident pos) start) (hs-emit! "id" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "@") (= ch "@")
@@ -482,7 +490,7 @@
(hs-ident-char? (hs-peek 1))) (hs-ident-char? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "attr" (read-ident pos) start) (hs-emit! "attr" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "^") (= ch "^")
@@ -490,7 +498,7 @@
(hs-ident-char? (hs-peek 1))) (hs-ident-char? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "hat" (read-ident pos) start) (hs-emit! "hat" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "~") (= ch "~")
@@ -498,7 +506,7 @@
(hs-letter? (hs-peek 1))) (hs-letter? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "component" (str "~" (read-ident pos)) start) (hs-emit! "component" (str "~" (read-ident pos)) start start-line)
(scan!)) (scan!))
(and (and
(= ch "*") (= ch "*")
@@ -506,7 +514,7 @@
(hs-letter? (hs-peek 1))) (hs-letter? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "style" (read-ident pos) start) (hs-emit! "style" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch ":") (= ch ":")
@@ -514,7 +522,7 @@
(hs-ident-start? (hs-peek 1))) (hs-ident-start? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "local" (read-ident pos) start) (hs-emit! "local" (read-ident pos) start start-line)
(scan!)) (scan!))
(or (or
(= ch "\"") (= ch "\"")
@@ -527,11 +535,11 @@
(or (or
(>= (+ pos 2) src-len) (>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2)))))))) (not (hs-ident-char? (hs-peek 2))))))))
(do (hs-emit! "string" (read-string ch) start) (scan!)) (do (hs-emit! "string" (read-string ch) start start-line) (scan!))
(= ch "`") (= ch "`")
(do (hs-emit! "template" (read-template) start) (scan!)) (do (hs-emit! "template" (read-template) start start-line) (scan!))
(hs-digit? ch) (hs-digit? ch)
(do (hs-emit! "number" (read-number start) start) (scan!)) (do (hs-emit! "number" (read-number start) start start-line) (scan!))
(hs-ident-start? ch) (hs-ident-start? ch)
(do (do
(let (let
@@ -539,7 +547,8 @@
(hs-emit! (hs-emit!
(if (hs-keyword? word) "keyword" "ident") (if (hs-keyword? word) "keyword" "ident")
word word
start)) start
start-line))
(scan!)) (scan!))
(and (and
(or (= ch "=") (= ch "!") (= ch "<") (= ch ">")) (or (= ch "=") (= ch "!") (= ch "<") (= ch ">"))
@@ -551,8 +560,8 @@
(or (= ch "=") (= ch "!")) (or (= ch "=") (= ch "!"))
(< (+ pos 2) src-len) (< (+ pos 2) src-len)
(= (hs-peek 2) "=")) (= (hs-peek 2) "="))
(do (hs-emit! "op" (str ch "==") start) (hs-advance! 3)) (do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line))
(do (hs-emit! "op" (str ch "=") start) (hs-advance! 2))) (do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line)))
(scan!)) (scan!))
(and (and
(= ch "'") (= ch "'")
@@ -561,66 +570,66 @@
(or (or
(>= (+ pos 2) src-len) (>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2))))) (not (hs-ident-char? (hs-peek 2)))))
(do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!)) (do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!))
(= ch "(") (= ch "(")
(do (do
(hs-emit! "paren-open" "(" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "paren-open" "(" start start-line)
(scan!)) (scan!))
(= ch ")") (= ch ")")
(do (do
(hs-emit! "paren-close" ")" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "paren-close" ")" start start-line)
(scan!)) (scan!))
(= ch "[") (= ch "[")
(do (do
(hs-emit! "bracket-open" "[" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "bracket-open" "[" start start-line)
(scan!)) (scan!))
(= ch "]") (= ch "]")
(do (do
(hs-emit! "bracket-close" "]" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "bracket-close" "]" start start-line)
(scan!)) (scan!))
(= ch "{") (= ch "{")
(do (do
(hs-emit! "brace-open" "{" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "brace-open" "{" start start-line)
(scan!)) (scan!))
(= ch "}") (= ch "}")
(do (do
(hs-emit! "brace-close" "}" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "brace-close" "}" start start-line)
(scan!)) (scan!))
(= ch ",") (= ch ",")
(do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!))
(= ch "+") (= ch "+")
(do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!))
(= ch "-") (= ch "-")
(do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!))
(= ch "/") (= ch "/")
(do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!))
(= ch "=") (= ch "=")
(do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!))
(= ch "<") (= ch "<")
(do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!))
(= ch ">") (= ch ">")
(do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!))
(= ch "!") (= ch "!")
(do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!))
(= ch "*") (= ch "*")
(do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!))
(= ch "%") (= ch "%")
(do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!))
(= ch ".") (= ch ".")
(do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!))
(= ch "\\") (= ch "\\")
(do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!))
(= ch ":") (= ch ":")
(do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!))
(= ch "|") (= ch "|")
(do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!))
:else (do (hs-advance! 1) (scan!))))))) :else (do (hs-advance! 1) (scan!)))))))
(scan!) (scan!)
(hs-emit! "eof" nil pos) (hs-emit! "eof" nil pos current-line)
tokens))) tokens)))

97
shared/static/wasm/sx/hs-tokenizer.sx
View File

@@ -1,6 +1,6 @@
;; _hyperscript tokenizer — produces token stream from hyperscript source ;; _hyperscript tokenizer — produces token stream from hyperscript source
;; ;;
;; Tokens: {:type T :value V :pos P} ;; Tokens: {:type T :value V :pos P :end E :line L}
;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style" ;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style"
;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open" ;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open"
;; "bracket-close" "brace-open" "brace-close" "comma" "colon" ;; "bracket-close" "brace-open" "brace-close" "comma" "colon"
@@ -8,7 +8,7 @@
;; ── Token constructor ───────────────────────────────────────────── ;; ── Token constructor ─────────────────────────────────────────────
(define hs-make-token (fn (type value pos) {:pos pos :value value :type type})) (define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type}))
;; ── Character predicates ────────────────────────────────────────── ;; ── Character predicates ──────────────────────────────────────────
@@ -198,14 +198,22 @@
(fn (fn
(src) (src)
(let (let
((tokens (list)) (pos 0) (src-len (len src))) ((tokens (list)) (pos 0) (src-len (len src)) (current-line 1))
(define (define
hs-peek hs-peek
(fn (fn
(offset) (offset)
(if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil))) (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
(define hs-cur (fn () (hs-peek 0))) (define hs-cur (fn () (hs-peek 0)))
(define hs-advance! (fn (n) (set! pos (+ pos n)))) (define
hs-advance!
(fn
(n)
(when
(> n 0)
(when (= (hs-cur) "\n") (set! current-line (+ current-line 1)))
(set! pos (+ pos 1))
(hs-advance! (- n 1)))))
(define (define
skip-ws! skip-ws!
(fn (fn
@@ -427,8 +435,8 @@
(define (define
hs-emit! hs-emit!
(fn (fn
(type value start) (type value start start-line)
(append! tokens (hs-make-token type value start)))) (append! tokens (hs-make-token type value start pos start-line))))
(define (define
scan! scan!
(fn (fn
@@ -437,7 +445,7 @@
(when (when
(< pos src-len) (< pos src-len)
(let (let
((ch (hs-cur)) (start pos)) ((ch (hs-cur)) (start pos) (start-line current-line))
(cond (cond
(and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-")) (and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-"))
(do (hs-advance! 2) (skip-comment!) (scan!)) (do (hs-advance! 2) (skip-comment!) (scan!))
@@ -454,9 +462,9 @@
(= (hs-peek 1) "[") (= (hs-peek 1) "[")
(= (hs-peek 1) "*") (= (hs-peek 1) "*")
(= (hs-peek 1) ":"))) (= (hs-peek 1) ":")))
(do (hs-emit! "selector" (read-selector) start) (scan!)) (do (hs-emit! "selector" (read-selector) start start-line) (scan!))
(and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) ".")) (and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) "."))
(do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!)) (do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!))
(and (and
(= ch ".") (= ch ".")
(< (+ pos 1) src-len) (< (+ pos 1) src-len)
@@ -466,7 +474,7 @@
(= (hs-peek 1) "_"))) (= (hs-peek 1) "_")))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "class" (read-class-name pos) start) (hs-emit! "class" (read-class-name pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "#") (= ch "#")
@@ -474,7 +482,7 @@
(hs-ident-start? (hs-peek 1))) (hs-ident-start? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "id" (read-ident pos) start) (hs-emit! "id" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "@") (= ch "@")
@@ -482,7 +490,7 @@
(hs-ident-char? (hs-peek 1))) (hs-ident-char? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "attr" (read-ident pos) start) (hs-emit! "attr" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "^") (= ch "^")
@@ -490,7 +498,7 @@
(hs-ident-char? (hs-peek 1))) (hs-ident-char? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "hat" (read-ident pos) start) (hs-emit! "hat" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch "~") (= ch "~")
@@ -498,7 +506,7 @@
(hs-letter? (hs-peek 1))) (hs-letter? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "component" (str "~" (read-ident pos)) start) (hs-emit! "component" (str "~" (read-ident pos)) start start-line)
(scan!)) (scan!))
(and (and
(= ch "*") (= ch "*")
@@ -506,7 +514,7 @@
(hs-letter? (hs-peek 1))) (hs-letter? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "style" (read-ident pos) start) (hs-emit! "style" (read-ident pos) start start-line)
(scan!)) (scan!))
(and (and
(= ch ":") (= ch ":")
@@ -514,7 +522,7 @@
(hs-ident-start? (hs-peek 1))) (hs-ident-start? (hs-peek 1)))
(do (do
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "local" (read-ident pos) start) (hs-emit! "local" (read-ident pos) start start-line)
(scan!)) (scan!))
(or (or
(= ch "\"") (= ch "\"")
@@ -527,11 +535,11 @@
(or (or
(>= (+ pos 2) src-len) (>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2)))))))) (not (hs-ident-char? (hs-peek 2))))))))
(do (hs-emit! "string" (read-string ch) start) (scan!)) (do (hs-emit! "string" (read-string ch) start start-line) (scan!))
(= ch "`") (= ch "`")
(do (hs-emit! "template" (read-template) start) (scan!)) (do (hs-emit! "template" (read-template) start start-line) (scan!))
(hs-digit? ch) (hs-digit? ch)
(do (hs-emit! "number" (read-number start) start) (scan!)) (do (hs-emit! "number" (read-number start) start start-line) (scan!))
(hs-ident-start? ch) (hs-ident-start? ch)
(do (do
(let (let
@@ -539,7 +547,8 @@
(hs-emit! (hs-emit!
(if (hs-keyword? word) "keyword" "ident") (if (hs-keyword? word) "keyword" "ident")
word word
start)) start
start-line))
(scan!)) (scan!))
(and (and
(or (= ch "=") (= ch "!") (= ch "<") (= ch ">")) (or (= ch "=") (= ch "!") (= ch "<") (= ch ">"))
@@ -551,8 +560,8 @@
(or (= ch "=") (= ch "!")) (or (= ch "=") (= ch "!"))
(< (+ pos 2) src-len) (< (+ pos 2) src-len)
(= (hs-peek 2) "=")) (= (hs-peek 2) "="))
(do (hs-emit! "op" (str ch "==") start) (hs-advance! 3)) (do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line))
(do (hs-emit! "op" (str ch "=") start) (hs-advance! 2))) (do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line)))
(scan!)) (scan!))
(and (and
(= ch "'") (= ch "'")
@@ -561,66 +570,66 @@
(or (or
(>= (+ pos 2) src-len) (>= (+ pos 2) src-len)
(not (hs-ident-char? (hs-peek 2))))) (not (hs-ident-char? (hs-peek 2)))))
(do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!)) (do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!))
(= ch "(") (= ch "(")
(do (do
(hs-emit! "paren-open" "(" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "paren-open" "(" start start-line)
(scan!)) (scan!))
(= ch ")") (= ch ")")
(do (do
(hs-emit! "paren-close" ")" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "paren-close" ")" start start-line)
(scan!)) (scan!))
(= ch "[") (= ch "[")
(do (do
(hs-emit! "bracket-open" "[" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "bracket-open" "[" start start-line)
(scan!)) (scan!))
(= ch "]") (= ch "]")
(do (do
(hs-emit! "bracket-close" "]" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "bracket-close" "]" start start-line)
(scan!)) (scan!))
(= ch "{") (= ch "{")
(do (do
(hs-emit! "brace-open" "{" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "brace-open" "{" start start-line)
(scan!)) (scan!))
(= ch "}") (= ch "}")
(do (do
(hs-emit! "brace-close" "}" start)
(hs-advance! 1) (hs-advance! 1)
(hs-emit! "brace-close" "}" start start-line)
(scan!)) (scan!))
(= ch ",") (= ch ",")
(do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!))
(= ch "+") (= ch "+")
(do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!))
(= ch "-") (= ch "-")
(do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!))
(= ch "/") (= ch "/")
(do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!))
(= ch "=") (= ch "=")
(do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!))
(= ch "<") (= ch "<")
(do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!))
(= ch ">") (= ch ">")
(do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!))
(= ch "!") (= ch "!")
(do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!))
(= ch "*") (= ch "*")
(do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!))
(= ch "%") (= ch "%")
(do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!))
(= ch ".") (= ch ".")
(do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!))
(= ch "\\") (= ch "\\")
(do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!))
(= ch ":") (= ch ":")
(do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!))
(= ch "|") (= ch "|")
(do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!)) (do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!))
:else (do (hs-advance! 1) (scan!))))))) :else (do (hs-advance! 1) (scan!)))))))
(scan!) (scan!)
(hs-emit! "eof" nil pos) (hs-emit! "eof" nil pos current-line)
tokens))) tokens)))