coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4cd0e77331ea3ea81191f006d6f9792d4daa915f
rose-ash/spec/tests/test-hyperscript-tokenizer.sx
giles 3336c4e957 Step 18 (part 1): _hyperscript tokenizer — 38 tests 
lib/hyperscript/tokenizer.sx — tokenizes real _hyperscript syntax into
typed token stream. Handles:
  Keywords (on, set, add, toggle, if, then, from, etc.)
  DOM literals (.class, #id, @attr, *style, :local, <sel/>)
  Strings (single/double quoted, escapes), template literals
  Numbers (integers, decimals, time units: 100ms, 2s)
  Operators (==, !=, +, -, 's possessive)
  Punctuation (parens, brackets, braces, commas, dots)
  Line comments (// to EOL)

Parser will disambiguate .name as class vs property access from context.
Possessive 's correctly distinguished from single-quote strings.

2952/2952 tests, zero failures.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-05 21:49:13 +00:00

314 lines
11 KiB
Plaintext
Raw Blame History

;; _hyperscript tokenizer tests
;; helper: get token types as a flat list
(define hs-types (fn (tokens) (map (fn (t) (get t "type")) tokens)))
(define hs-vals (fn (tokens) (map (fn (t) (get t "value")) tokens)))
(define hs-tok (fn (tokens n) (nth tokens n)))
(defsuite
"hs-tokenize-basics"
(deftest
"empty input"
(let
((tokens (hs-tokenize "")))
(assert= 1 (len tokens))
(assert= "eof" (get (first tokens) "type"))))
(deftest
"single keyword"
(let
((tokens (hs-tokenize "on")))
(assert= 2 (len tokens))
(assert= "keyword" (get (first tokens) "type"))
(assert= "on" (get (first tokens) "value"))))
(deftest
"identifier"
(let
((tokens (hs-tokenize "myVar")))
(assert= "ident" (get (first tokens) "type"))
(assert= "myVar" (get (first tokens) "value"))))
(deftest
"keywords vs identifiers"
(let
((tokens (hs-tokenize "on click add foo")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "ident" (get (hs-tok tokens 3) "type"))))
(deftest
"whitespace skipped"
(let
((tokens (hs-tokenize " on click ")))
(assert= 3 (len tokens))
(assert= "on" (get (first tokens) "value")))))
(defsuite
"hs-tokenize-literals"
(deftest
"integer"
(let
((t (first (hs-tokenize "42"))))
(assert= "number" (get t "type"))
(assert= "42" (get t "value"))))
(deftest
"decimal"
(let
((t (first (hs-tokenize "3.14"))))
(assert= "number" (get t "type"))
(assert= "3.14" (get t "value"))))
(deftest
"number with ms unit"
(let
((t (first (hs-tokenize "100ms"))))
(assert= "number" (get t "type"))
(assert= "100ms" (get t "value"))))
(deftest
"number with s unit"
(let
((t (first (hs-tokenize "2s"))))
(assert= "number" (get t "type"))
(assert= "2s" (get t "value"))))
(deftest
"double-quoted string"
(let
((t (first (hs-tokenize "\"hello world\""))))
(assert= "string" (get t "type"))
(assert= "hello world" (get t "value"))))
(deftest
"single-quoted string"
(let
((t (first (hs-tokenize "'foo'"))))
(assert= "string" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"string with escapes"
(let
((t (first (hs-tokenize "\"a\\nb\""))))
(assert= "string" (get t "type"))
(assert= "a\nb" (get t "value"))))
(deftest
"template literal"
(let
((t (first (hs-tokenize "`hello ${name}`"))))
(assert= "template" (get t "type"))
(assert= "hello ${name}" (get t "value")))))
(defsuite
"hs-tokenize-dom-refs"
(deftest
"class literal"
(let
((t (first (hs-tokenize ".foo"))))
(assert= "class" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"class with dashes"
(let
((t (first (hs-tokenize ".foo--bar"))))
(assert= "class" (get t "type"))
(assert= "foo--bar" (get t "value"))))
(deftest
"id literal"
(let
((t (first (hs-tokenize "#bar"))))
(assert= "id" (get t "type"))
(assert= "bar" (get t "value"))))
(deftest
"attribute ref"
(let
((t (first (hs-tokenize "@foo"))))
(assert= "attr" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"style ref"
(let
((t (first (hs-tokenize "*color"))))
(assert= "style" (get t "type"))
(assert= "color" (get t "value"))))
(deftest
"element-scoped local"
(let
((t (first (hs-tokenize ":myVar"))))
(assert= "local" (get t "type"))
(assert= "myVar" (get t "value"))))
(deftest
"CSS selector"
(let
((t (first (hs-tokenize "<p/>"))))
(assert= "selector" (get t "type"))
(assert= "p" (get t "value"))))
(deftest
"CSS selector complex"
(let
((t (first (hs-tokenize "<div.foo/>"))))
(assert= "selector" (get t "type"))
(assert= "div.foo" (get t "value")))))
(defsuite
"hs-tokenize-operators"
(deftest
"equals"
(let
((t (first (hs-tokenize "=="))))
(assert= "op" (get t "type"))
(assert= "==" (get t "value"))))
(deftest
"not-equals"
(let
((t (first (hs-tokenize "!="))))
(assert= "op" (get t "type"))
(assert= "!=" (get t "value"))))
(deftest
"plus"
(let
((t (first (hs-tokenize "+"))))
(assert= "op" (get t "type"))
(assert= "+" (get t "value"))))
(deftest
"possessive"
(let
((t (first (hs-tokenize "#d1's"))))
(assert= "id" (get (hs-tok (hs-tokenize "#d1's") 0) "type"))
(assert= "op" (get (hs-tok (hs-tokenize "#d1's") 1) "type"))
(assert= "'s" (get (hs-tok (hs-tokenize "#d1's") 1) "value"))))
(deftest
"parens"
(let
((tokens (hs-tokenize "(foo)")))
(assert= "paren-open" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "paren-close" (get (hs-tok tokens 2) "type"))))
(deftest
"brackets"
(let
((tokens (hs-tokenize "[0]")))
(assert= "bracket-open" (get (hs-tok tokens 0) "type"))
(assert= "number" (get (hs-tok tokens 1) "type"))
(assert= "bracket-close" (get (hs-tok tokens 2) "type")))))
(defsuite
"hs-tokenize-comments"
(deftest
"line comment skipped"
(let
((tokens (hs-tokenize "on // this is a comment\nclick")))
(assert= 3 (len tokens))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "click" (get (hs-tok tokens 1) "value")))))
(defsuite
"hs-tokenize-full-expressions"
(deftest
"on click add .called"
(let
((tokens (hs-tokenize "on click add .called")))
(assert= 5 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "add" (get (hs-tok tokens 2) "value"))
(assert= "class" (get (hs-tok tokens 3) "type"))
(assert= "called" (get (hs-tok tokens 3) "value"))
(assert= "eof" (get (hs-tok tokens 4) "type"))))
(deftest
"set #d1.innerHTML to foo"
(let
((tokens (hs-tokenize "set #d1.innerHTML to \"foo\"")))
(assert= 6 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "id" (get (hs-tok tokens 1) "type"))
(assert= "d1" (get (hs-tok tokens 1) "value"))
(assert= "class" (get (hs-tok tokens 2) "type"))
(assert= "innerHTML" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "to" (get (hs-tok tokens 3) "value"))
(assert= "string" (get (hs-tok tokens 4) "type"))))
(deftest
"put \"Clicked\" into my.innerHTML"
(let
((tokens (hs-tokenize "put \"Clicked\" into my.innerHTML")))
(assert= 6 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "put" (get (hs-tok tokens 0) "value"))
(assert= "string" (get (hs-tok tokens 1) "type"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "into" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "my" (get (hs-tok tokens 3) "value"))))
(deftest
"on click send custom(foo:\"fromBar\") to #d2"
(let
((tokens (hs-tokenize "on click send custom(foo:\"fromBar\") to #d2")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "send" (get (hs-tok tokens 2) "value"))))
(deftest
"toggle between .foo and .bar"
(let
((tokens (hs-tokenize "toggle between .foo and .bar")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "toggle" (get (hs-tok tokens 0) "value"))
(assert= "keyword" (get (hs-tok tokens 1) "type"))
(assert= "between" (get (hs-tok tokens 1) "value"))
(assert= "class" (get (hs-tok tokens 2) "type"))
(assert= "foo" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "and" (get (hs-tok tokens 3) "value"))
(assert= "class" (get (hs-tok tokens 4) "type"))
(assert= "bar" (get (hs-tok tokens 4) "value"))))
(deftest
"if true put \"foo\" into me.innerHTML else put \"bar\" into me.innerHTML end"
(let
((tokens (hs-tokenize "if true put \"foo\" into me.innerHTML else put \"bar\" into me.innerHTML end")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "if" (get (hs-tok tokens 0) "value"))
(assert= "keyword" (get (hs-tok tokens 1) "type"))
(assert= "true" (get (hs-tok tokens 1) "value"))))
(deftest
"on click[buttons==0] log event"
(let
((tokens (hs-tokenize "on click[buttons==0] log event")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "bracket-open" (get (hs-tok tokens 2) "type"))
(assert= "ident" (get (hs-tok tokens 3) "type"))
(assert= "buttons" (get (hs-tok tokens 3) "value"))
(assert= "op" (get (hs-tok tokens 4) "type"))
(assert= "==" (get (hs-tok tokens 4) "value"))))
(deftest
"wait 100ms then add .done"
(let
((tokens (hs-tokenize "wait 100ms then add .done")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "wait" (get (hs-tok tokens 0) "value"))
(assert= "number" (get (hs-tok tokens 1) "type"))
(assert= "100ms" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "then" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "add" (get (hs-tok tokens 3) "value"))
(assert= "class" (get (hs-tok tokens 4) "type"))
(assert= "done" (get (hs-tok tokens 4) "value"))))
(deftest
"increment @count then put it into me"
(let
((tokens (hs-tokenize "increment @count then put it into me")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "increment" (get (hs-tok tokens 0) "value"))
(assert= "attr" (get (hs-tok tokens 1) "type"))
(assert= "count" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 4) "type"))
(assert= "it" (get (hs-tok tokens 4) "value"))))
(deftest
"on click from #bar add .clicked"
(let
((tokens (hs-tokenize "on click from #bar add .clicked")))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "from" (get (hs-tok tokens 2) "value"))
(assert= "id" (get (hs-tok tokens 3) "type"))
(assert= "bar" (get (hs-tok tokens 3) "value")))))
Reference in New Issue View Git Blame Copy Permalink