diff --git a/lib/hyperscript/tokenizer.sx b/lib/hyperscript/tokenizer.sx index 2483ea8c..6f652dea 100644 --- a/lib/hyperscript/tokenizer.sx +++ b/lib/hyperscript/tokenizer.sx @@ -1,6 +1,6 @@ ;; _hyperscript tokenizer — produces token stream from hyperscript source ;; -;; Tokens: {:type T :value V :pos P} +;; Tokens: {:type T :value V :pos P :end E :line L} ;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style" ;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open" ;; "bracket-close" "brace-open" "brace-close" "comma" "colon" @@ -8,7 +8,7 @@ ;; ── Token constructor ───────────────────────────────────────────── -(define hs-make-token (fn (type value pos) {:pos pos :value value :type type})) +(define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type})) ;; ── Character predicates ────────────────────────────────────────── @@ -198,14 +198,22 @@ (fn (src) (let - ((tokens (list)) (pos 0) (src-len (len src))) + ((tokens (list)) (pos 0) (src-len (len src)) (current-line 1)) (define hs-peek (fn (offset) (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil))) (define hs-cur (fn () (hs-peek 0))) - (define hs-advance! (fn (n) (set! pos (+ pos n)))) + (define + hs-advance! + (fn + (n) + (when + (> n 0) + (when (= (hs-cur) "\n") (set! current-line (+ current-line 1))) + (set! pos (+ pos 1)) + (hs-advance! (- n 1))))) (define skip-ws! (fn @@ -427,8 +435,8 @@ (define hs-emit! (fn - (type value start) - (append! tokens (hs-make-token type value start)))) + (type value start start-line) + (append! tokens (hs-make-token type value start pos start-line)))) (define scan! (fn @@ -437,7 +445,7 @@ (when (< pos src-len) (let - ((ch (hs-cur)) (start pos)) + ((ch (hs-cur)) (start pos) (start-line current-line)) (cond (and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-")) (do (hs-advance! 2) (skip-comment!) (scan!)) @@ -454,9 +462,9 @@ (= (hs-peek 1) "[") (= (hs-peek 1) "*") (= (hs-peek 1) ":"))) - (do (hs-emit! "selector" (read-selector) start) (scan!)) + (do (hs-emit! "selector" (read-selector) start start-line) (scan!)) (and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) ".")) - (do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!)) + (do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!)) (and (= ch ".") (< (+ pos 1) src-len) @@ -466,7 +474,7 @@ (= (hs-peek 1) "_"))) (do (hs-advance! 1) - (hs-emit! "class" (read-class-name pos) start) + (hs-emit! "class" (read-class-name pos) start start-line) (scan!)) (and (= ch "#") @@ -474,7 +482,7 @@ (hs-ident-start? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "id" (read-ident pos) start) + (hs-emit! "id" (read-ident pos) start start-line) (scan!)) (and (= ch "@") @@ -482,7 +490,7 @@ (hs-ident-char? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "attr" (read-ident pos) start) + (hs-emit! "attr" (read-ident pos) start start-line) (scan!)) (and (= ch "^") @@ -490,7 +498,7 @@ (hs-ident-char? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "hat" (read-ident pos) start) + (hs-emit! "hat" (read-ident pos) start start-line) (scan!)) (and (= ch "~") @@ -498,7 +506,7 @@ (hs-letter? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "component" (str "~" (read-ident pos)) start) + (hs-emit! "component" (str "~" (read-ident pos)) start start-line) (scan!)) (and (= ch "*") @@ -506,7 +514,7 @@ (hs-letter? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "style" (read-ident pos) start) + (hs-emit! "style" (read-ident pos) start start-line) (scan!)) (and (= ch ":") @@ -514,7 +522,7 @@ (hs-ident-start? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "local" (read-ident pos) start) + (hs-emit! "local" (read-ident pos) start start-line) (scan!)) (or (= ch "\"") @@ -527,11 +535,11 @@ (or (>= (+ pos 2) src-len) (not (hs-ident-char? (hs-peek 2)))))))) - (do (hs-emit! "string" (read-string ch) start) (scan!)) + (do (hs-emit! "string" (read-string ch) start start-line) (scan!)) (= ch "`") - (do (hs-emit! "template" (read-template) start) (scan!)) + (do (hs-emit! "template" (read-template) start start-line) (scan!)) (hs-digit? ch) - (do (hs-emit! "number" (read-number start) start) (scan!)) + (do (hs-emit! "number" (read-number start) start start-line) (scan!)) (hs-ident-start? ch) (do (let @@ -539,7 +547,8 @@ (hs-emit! (if (hs-keyword? word) "keyword" "ident") word - start)) + start + start-line)) (scan!)) (and (or (= ch "=") (= ch "!") (= ch "<") (= ch ">")) @@ -551,8 +560,8 @@ (or (= ch "=") (= ch "!")) (< (+ pos 2) src-len) (= (hs-peek 2) "=")) - (do (hs-emit! "op" (str ch "==") start) (hs-advance! 3)) - (do (hs-emit! "op" (str ch "=") start) (hs-advance! 2))) + (do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line)) + (do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line))) (scan!)) (and (= ch "'") @@ -561,66 +570,66 @@ (or (>= (+ pos 2) src-len) (not (hs-ident-char? (hs-peek 2))))) - (do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!)) + (do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!)) (= ch "(") (do - (hs-emit! "paren-open" "(" start) (hs-advance! 1) + (hs-emit! "paren-open" "(" start start-line) (scan!)) (= ch ")") (do - (hs-emit! "paren-close" ")" start) (hs-advance! 1) + (hs-emit! "paren-close" ")" start start-line) (scan!)) (= ch "[") (do - (hs-emit! "bracket-open" "[" start) (hs-advance! 1) + (hs-emit! "bracket-open" "[" start start-line) (scan!)) (= ch "]") (do - (hs-emit! "bracket-close" "]" start) (hs-advance! 1) + (hs-emit! "bracket-close" "]" start start-line) (scan!)) (= ch "{") (do - (hs-emit! "brace-open" "{" start) (hs-advance! 1) + (hs-emit! "brace-open" "{" start start-line) (scan!)) (= ch "}") (do - (hs-emit! "brace-close" "}" start) (hs-advance! 1) + (hs-emit! "brace-close" "}" start start-line) (scan!)) (= ch ",") - (do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!)) (= ch "+") - (do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!)) (= ch "-") - (do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!)) (= ch "/") - (do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!)) (= ch "=") - (do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!)) (= ch "<") - (do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!)) (= ch ">") - (do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!)) (= ch "!") - (do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!)) (= ch "*") - (do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!)) (= ch "%") - (do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!)) (= ch ".") - (do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!)) (= ch "\\") - (do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!)) (= ch ":") - (do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!)) (= ch "|") - (do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!)) :else (do (hs-advance! 1) (scan!))))))) (scan!) - (hs-emit! "eof" nil pos) + (hs-emit! "eof" nil pos current-line) tokens))) \ No newline at end of file diff --git a/shared/static/wasm/sx/hs-tokenizer.sx b/shared/static/wasm/sx/hs-tokenizer.sx index 2483ea8c..6f652dea 100644 --- a/shared/static/wasm/sx/hs-tokenizer.sx +++ b/shared/static/wasm/sx/hs-tokenizer.sx @@ -1,6 +1,6 @@ ;; _hyperscript tokenizer — produces token stream from hyperscript source ;; -;; Tokens: {:type T :value V :pos P} +;; Tokens: {:type T :value V :pos P :end E :line L} ;; Types: "keyword" "ident" "number" "string" "class" "id" "attr" "style" ;; "selector" "op" "dot" "paren-open" "paren-close" "bracket-open" ;; "bracket-close" "brace-open" "brace-close" "comma" "colon" @@ -8,7 +8,7 @@ ;; ── Token constructor ───────────────────────────────────────────── -(define hs-make-token (fn (type value pos) {:pos pos :value value :type type})) +(define hs-make-token (fn (type value pos end line) {:pos pos :end end :line line :value value :type type})) ;; ── Character predicates ────────────────────────────────────────── @@ -198,14 +198,22 @@ (fn (src) (let - ((tokens (list)) (pos 0) (src-len (len src))) + ((tokens (list)) (pos 0) (src-len (len src)) (current-line 1)) (define hs-peek (fn (offset) (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil))) (define hs-cur (fn () (hs-peek 0))) - (define hs-advance! (fn (n) (set! pos (+ pos n)))) + (define + hs-advance! + (fn + (n) + (when + (> n 0) + (when (= (hs-cur) "\n") (set! current-line (+ current-line 1))) + (set! pos (+ pos 1)) + (hs-advance! (- n 1))))) (define skip-ws! (fn @@ -427,8 +435,8 @@ (define hs-emit! (fn - (type value start) - (append! tokens (hs-make-token type value start)))) + (type value start start-line) + (append! tokens (hs-make-token type value start pos start-line)))) (define scan! (fn @@ -437,7 +445,7 @@ (when (< pos src-len) (let - ((ch (hs-cur)) (start pos)) + ((ch (hs-cur)) (start pos) (start-line current-line)) (cond (and (= ch "-") (< (+ pos 1) src-len) (= (hs-peek 1) "-")) (do (hs-advance! 2) (skip-comment!) (scan!)) @@ -454,9 +462,9 @@ (= (hs-peek 1) "[") (= (hs-peek 1) "*") (= (hs-peek 1) ":"))) - (do (hs-emit! "selector" (read-selector) start) (scan!)) + (do (hs-emit! "selector" (read-selector) start start-line) (scan!)) (and (= ch ".") (< (+ pos 1) src-len) (= (hs-peek 1) ".")) - (do (hs-emit! "op" ".." start) (hs-advance! 2) (scan!)) + (do (hs-advance! 2) (hs-emit! "op" ".." start start-line) (scan!)) (and (= ch ".") (< (+ pos 1) src-len) @@ -466,7 +474,7 @@ (= (hs-peek 1) "_"))) (do (hs-advance! 1) - (hs-emit! "class" (read-class-name pos) start) + (hs-emit! "class" (read-class-name pos) start start-line) (scan!)) (and (= ch "#") @@ -474,7 +482,7 @@ (hs-ident-start? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "id" (read-ident pos) start) + (hs-emit! "id" (read-ident pos) start start-line) (scan!)) (and (= ch "@") @@ -482,7 +490,7 @@ (hs-ident-char? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "attr" (read-ident pos) start) + (hs-emit! "attr" (read-ident pos) start start-line) (scan!)) (and (= ch "^") @@ -490,7 +498,7 @@ (hs-ident-char? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "hat" (read-ident pos) start) + (hs-emit! "hat" (read-ident pos) start start-line) (scan!)) (and (= ch "~") @@ -498,7 +506,7 @@ (hs-letter? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "component" (str "~" (read-ident pos)) start) + (hs-emit! "component" (str "~" (read-ident pos)) start start-line) (scan!)) (and (= ch "*") @@ -506,7 +514,7 @@ (hs-letter? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "style" (read-ident pos) start) + (hs-emit! "style" (read-ident pos) start start-line) (scan!)) (and (= ch ":") @@ -514,7 +522,7 @@ (hs-ident-start? (hs-peek 1))) (do (hs-advance! 1) - (hs-emit! "local" (read-ident pos) start) + (hs-emit! "local" (read-ident pos) start start-line) (scan!)) (or (= ch "\"") @@ -527,11 +535,11 @@ (or (>= (+ pos 2) src-len) (not (hs-ident-char? (hs-peek 2)))))))) - (do (hs-emit! "string" (read-string ch) start) (scan!)) + (do (hs-emit! "string" (read-string ch) start start-line) (scan!)) (= ch "`") - (do (hs-emit! "template" (read-template) start) (scan!)) + (do (hs-emit! "template" (read-template) start start-line) (scan!)) (hs-digit? ch) - (do (hs-emit! "number" (read-number start) start) (scan!)) + (do (hs-emit! "number" (read-number start) start start-line) (scan!)) (hs-ident-start? ch) (do (let @@ -539,7 +547,8 @@ (hs-emit! (if (hs-keyword? word) "keyword" "ident") word - start)) + start + start-line)) (scan!)) (and (or (= ch "=") (= ch "!") (= ch "<") (= ch ">")) @@ -551,8 +560,8 @@ (or (= ch "=") (= ch "!")) (< (+ pos 2) src-len) (= (hs-peek 2) "=")) - (do (hs-emit! "op" (str ch "==") start) (hs-advance! 3)) - (do (hs-emit! "op" (str ch "=") start) (hs-advance! 2))) + (do (hs-advance! 3) (hs-emit! "op" (str ch "==") start start-line)) + (do (hs-advance! 2) (hs-emit! "op" (str ch "=") start start-line))) (scan!)) (and (= ch "'") @@ -561,66 +570,66 @@ (or (>= (+ pos 2) src-len) (not (hs-ident-char? (hs-peek 2))))) - (do (hs-emit! "op" "'s" start) (hs-advance! 2) (scan!)) + (do (hs-advance! 2) (hs-emit! "op" "'s" start start-line) (scan!)) (= ch "(") (do - (hs-emit! "paren-open" "(" start) (hs-advance! 1) + (hs-emit! "paren-open" "(" start start-line) (scan!)) (= ch ")") (do - (hs-emit! "paren-close" ")" start) (hs-advance! 1) + (hs-emit! "paren-close" ")" start start-line) (scan!)) (= ch "[") (do - (hs-emit! "bracket-open" "[" start) (hs-advance! 1) + (hs-emit! "bracket-open" "[" start start-line) (scan!)) (= ch "]") (do - (hs-emit! "bracket-close" "]" start) (hs-advance! 1) + (hs-emit! "bracket-close" "]" start start-line) (scan!)) (= ch "{") (do - (hs-emit! "brace-open" "{" start) (hs-advance! 1) + (hs-emit! "brace-open" "{" start start-line) (scan!)) (= ch "}") (do - (hs-emit! "brace-close" "}" start) (hs-advance! 1) + (hs-emit! "brace-close" "}" start start-line) (scan!)) (= ch ",") - (do (hs-emit! "comma" "," start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "comma" "," start start-line) (scan!)) (= ch "+") - (do (hs-emit! "op" "+" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "+" start start-line) (scan!)) (= ch "-") - (do (hs-emit! "op" "-" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "-" start start-line) (scan!)) (= ch "/") - (do (hs-emit! "op" "/" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "/" start start-line) (scan!)) (= ch "=") - (do (hs-emit! "op" "=" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "=" start start-line) (scan!)) (= ch "<") - (do (hs-emit! "op" "<" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "<" start start-line) (scan!)) (= ch ">") - (do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" ">" start start-line) (scan!)) (= ch "!") - (do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "!" start start-line) (scan!)) (= ch "*") - (do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "*" start start-line) (scan!)) (= ch "%") - (do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "%" start start-line) (scan!)) (= ch ".") - (do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "dot" "." start start-line) (scan!)) (= ch "\\") - (do (hs-emit! "op" "\\" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "\\" start start-line) (scan!)) (= ch ":") - (do (hs-emit! "colon" ":" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "colon" ":" start start-line) (scan!)) (= ch "|") - (do (hs-emit! "op" "|" start) (hs-advance! 1) (scan!)) + (do (hs-advance! 1) (hs-emit! "op" "|" start start-line) (scan!)) :else (do (hs-advance! 1) (scan!))))))) (scan!) - (hs-emit! "eof" nil pos) + (hs-emit! "eof" nil pos current-line) tokens))) \ No newline at end of file