diff --git a/lib/hyperscript/tokenizer.sx b/lib/hyperscript/tokenizer.sx index 2483ea8c..98933329 100644 --- a/lib/hyperscript/tokenizer.sx +++ b/lib/hyperscript/tokenizer.sx @@ -28,6 +28,27 @@ (define hs-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r")))) +(define + hs-hex-digit? + (fn + (c) + (or + (and (>= c "0") (<= c "9")) + (and (>= c "a") (<= c "f")) + (and (>= c "A") (<= c "F"))))) + +(define + hs-hex-val + (fn + (c) + (let + ((code (char-code c))) + (cond + ((and (>= code 48) (<= code 57)) (- code 48)) + ((and (>= code 65) (<= code 70)) (- code 55)) + ((and (>= code 97) (<= code 102)) (- code 87)) + (true 0))))) + ;; ── Keyword set ─────────────────────────────────────────────────── (define @@ -308,7 +329,7 @@ () (cond (>= pos src-len) - nil + (error "Unterminated string") (= (hs-cur) "\\") (do (hs-advance! 1) @@ -318,15 +339,37 @@ ((ch (hs-cur))) (cond (= ch "n") - (append! chars "\n") + (do (append! chars "\n") (hs-advance! 1)) (= ch "t") - (append! chars "\t") + (do (append! chars "\t") (hs-advance! 1)) + (= ch "r") + (do (append! chars "\r") (hs-advance! 1)) + (= ch "b") + (do (append! chars (char-from-code 8)) (hs-advance! 1)) + (= ch "f") + (do (append! chars (char-from-code 12)) (hs-advance! 1)) + (= ch "v") + (do (append! chars (char-from-code 11)) (hs-advance! 1)) (= ch "\\") - (append! chars "\\") + (do (append! chars "\\") (hs-advance! 1)) (= ch quote-char) - (append! chars quote-char) - :else (do (append! chars "\\") (append! chars ch))) - (hs-advance! 1))) + (do (append! chars quote-char) (hs-advance! 1)) + (= ch "x") + (do + (hs-advance! 1) + (if + (and + (< (+ pos 1) src-len) + (hs-hex-digit? (hs-cur)) + (hs-hex-digit? (hs-peek 1))) + (let + ((d1 (hs-hex-val (hs-cur))) + (d2 (hs-hex-val (hs-peek 1)))) + (append! chars (char-from-code (+ (* d1 16) d2))) + (hs-advance! 2)) + (error "Invalid hexadecimal escape: \\x"))) + :else + (do (append! chars "\\") (append! chars ch) (hs-advance! 1))))) (loop)) (= (hs-cur) quote-char) (hs-advance! 1)