coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
577d09f443c17ef70641f7ad3754cb554e769470
rose-ash/spec/tests/test-hyperscript-tokenizer.sx
giles f1ba7177e7 Step 18 (part 3): Expand parser — expressions, commands, features 
Tokenizer:
  * and % now emit as operators (were silently swallowed)
  Added keywords: install, measure, behavior, called
  5 new arithmetic operator tests

Parser — expression layer:
  Arithmetic (+, -, *, /, %) via parse-arith
  Unary not, no, unary minus
  the X of Y possessive (parse-the-expr)
  as Type conversion, X in Y membership, array literals [...]
  fetch URL parsing fixed — no longer consumes "as" meant for fetch

Parser — 8 new commands:
  return, throw, append...to, tell...end, for...in...end,
  make a Type, install Behavior, measure

Parser — 2 new features:
  def name(params)...end, behavior Name(params)...end

Parser — enhanced:
  wait for event [from target], on every event modifier

33 new parser tests (16 suites), 5 tokenizer tests.
3043/3043 full build, zero regressions.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-06 08:21:02 +00:00

351 lines
12 KiB
Plaintext
Raw Blame History

;; _hyperscript tokenizer tests
;; helper: get token types as a flat list
(define hs-types (fn (tokens) (map (fn (t) (get t "type")) tokens)))
(define hs-vals (fn (tokens) (map (fn (t) (get t "value")) tokens)))
(define hs-tok (fn (tokens n) (nth tokens n)))
(defsuite
"hs-tokenize-basics"
(deftest
"empty input"
(let
((tokens (hs-tokenize "")))
(assert= 1 (len tokens))
(assert= "eof" (get (first tokens) "type"))))
(deftest
"single keyword"
(let
((tokens (hs-tokenize "on")))
(assert= 2 (len tokens))
(assert= "keyword" (get (first tokens) "type"))
(assert= "on" (get (first tokens) "value"))))
(deftest
"identifier"
(let
((tokens (hs-tokenize "myVar")))
(assert= "ident" (get (first tokens) "type"))
(assert= "myVar" (get (first tokens) "value"))))
(deftest
"keywords vs identifiers"
(let
((tokens (hs-tokenize "on click add foo")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "ident" (get (hs-tok tokens 3) "type"))))
(deftest
"whitespace skipped"
(let
((tokens (hs-tokenize " on click ")))
(assert= 3 (len tokens))
(assert= "on" (get (first tokens) "value")))))
(defsuite
"hs-tokenize-literals"
(deftest
"integer"
(let
((t (first (hs-tokenize "42"))))
(assert= "number" (get t "type"))
(assert= "42" (get t "value"))))
(deftest
"decimal"
(let
((t (first (hs-tokenize "3.14"))))
(assert= "number" (get t "type"))
(assert= "3.14" (get t "value"))))
(deftest
"number with ms unit"
(let
((t (first (hs-tokenize "100ms"))))
(assert= "number" (get t "type"))
(assert= "100ms" (get t "value"))))
(deftest
"number with s unit"
(let
((t (first (hs-tokenize "2s"))))
(assert= "number" (get t "type"))
(assert= "2s" (get t "value"))))
(deftest
"double-quoted string"
(let
((t (first (hs-tokenize "\"hello world\""))))
(assert= "string" (get t "type"))
(assert= "hello world" (get t "value"))))
(deftest
"single-quoted string"
(let
((t (first (hs-tokenize "'foo'"))))
(assert= "string" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"string with escapes"
(let
((t (first (hs-tokenize "\"a\\nb\""))))
(assert= "string" (get t "type"))
(assert= "a\nb" (get t "value"))))
(deftest
"template literal"
(let
((t (first (hs-tokenize "`hello ${name}`"))))
(assert= "template" (get t "type"))
(assert= "hello ${name}" (get t "value")))))
(defsuite
"hs-tokenize-dom-refs"
(deftest
"class literal"
(let
((t (first (hs-tokenize ".foo"))))
(assert= "class" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"class with dashes"
(let
((t (first (hs-tokenize ".foo--bar"))))
(assert= "class" (get t "type"))
(assert= "foo--bar" (get t "value"))))
(deftest
"id literal"
(let
((t (first (hs-tokenize "#bar"))))
(assert= "id" (get t "type"))
(assert= "bar" (get t "value"))))
(deftest
"attribute ref"
(let
((t (first (hs-tokenize "@foo"))))
(assert= "attr" (get t "type"))
(assert= "foo" (get t "value"))))
(deftest
"style ref"
(let
((t (first (hs-tokenize "*color"))))
(assert= "style" (get t "type"))
(assert= "color" (get t "value"))))
(deftest
"element-scoped local"
(let
((t (first (hs-tokenize ":myVar"))))
(assert= "local" (get t "type"))
(assert= "myVar" (get t "value"))))
(deftest
"CSS selector"
(let
((t (first (hs-tokenize "<p/>"))))
(assert= "selector" (get t "type"))
(assert= "p" (get t "value"))))
(deftest
"CSS selector complex"
(let
((t (first (hs-tokenize "<div.foo/>"))))
(assert= "selector" (get t "type"))
(assert= "div.foo" (get t "value")))))
(defsuite
"hs-tokenize-operators"
(deftest
"equals"
(let
((t (first (hs-tokenize "=="))))
(assert= "op" (get t "type"))
(assert= "==" (get t "value"))))
(deftest
"not-equals"
(let
((t (first (hs-tokenize "!="))))
(assert= "op" (get t "type"))
(assert= "!=" (get t "value"))))
(deftest
"plus"
(let
((t (first (hs-tokenize "+"))))
(assert= "op" (get t "type"))
(assert= "+" (get t "value"))))
(deftest
"possessive"
(let
((t (first (hs-tokenize "#d1's"))))
(assert= "id" (get (hs-tok (hs-tokenize "#d1's") 0) "type"))
(assert= "op" (get (hs-tok (hs-tokenize "#d1's") 1) "type"))
(assert= "'s" (get (hs-tok (hs-tokenize "#d1's") 1) "value"))))
(deftest
"parens"
(let
((tokens (hs-tokenize "(foo)")))
(assert= "paren-open" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "paren-close" (get (hs-tok tokens 2) "type"))))
(deftest
"brackets"
(let
((tokens (hs-tokenize "[0]")))
(assert= "bracket-open" (get (hs-tok tokens 0) "type"))
(assert= "number" (get (hs-tok tokens 1) "type"))
(assert= "bracket-close" (get (hs-tok tokens 2) "type")))))
(defsuite
"hs-tokenize-arithmetic-ops"
(deftest
"multiply operator"
(let
((toks (hs-tokenize "4 * 5")))
(assert= "number" (get (nth toks 0) "type"))
(assert= "op" (get (nth toks 1) "type"))
(assert= "*" (get (nth toks 1) "value"))
(assert= "number" (get (nth toks 2) "type"))))
(deftest
"modulo operator"
(let
((toks (hs-tokenize "10 % 3")))
(assert= "op" (get (nth toks 1) "type"))
(assert= "%" (get (nth toks 1) "value"))))
(deftest
"star as style not operator"
(let
((toks (hs-tokenize "*color")))
(assert= "style" (get (nth toks 0) "type"))
(assert= "color" (get (nth toks 0) "value"))))
(deftest
"division operator"
(let
((toks (hs-tokenize "10 / 2")))
(assert= "op" (get (nth toks 1) "type"))
(assert= "/" (get (nth toks 1) "value"))))
(deftest
"mixed arithmetic"
(let
((toks (hs-tokenize "1 + 2 * 3")))
(assert= "op" (get (nth toks 1) "type"))
(assert= "+" (get (nth toks 1) "value"))
(assert= "op" (get (nth toks 3) "type"))
(assert= "*" (get (nth toks 3) "value")))))
(defsuite
"hs-tokenize-comments"
(deftest
"line comment skipped"
(let
((tokens (hs-tokenize "on // this is a comment\nclick")))
(assert= 3 (len tokens))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "click" (get (hs-tok tokens 1) "value")))))
(defsuite
"hs-tokenize-full-expressions"
(deftest
"on click add .called"
(let
((tokens (hs-tokenize "on click add .called")))
(assert= 5 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "add" (get (hs-tok tokens 2) "value"))
(assert= "class" (get (hs-tok tokens 3) "type"))
(assert= "called" (get (hs-tok tokens 3) "value"))
(assert= "eof" (get (hs-tok tokens 4) "type"))))
(deftest
"set #d1.innerHTML to foo"
(let
((tokens (hs-tokenize "set #d1.innerHTML to \"foo\"")))
(assert= 6 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "id" (get (hs-tok tokens 1) "type"))
(assert= "d1" (get (hs-tok tokens 1) "value"))
(assert= "class" (get (hs-tok tokens 2) "type"))
(assert= "innerHTML" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "to" (get (hs-tok tokens 3) "value"))
(assert= "string" (get (hs-tok tokens 4) "type"))))
(deftest
"put \"Clicked\" into my.innerHTML"
(let
((tokens (hs-tokenize "put \"Clicked\" into my.innerHTML")))
(assert= 6 (len tokens))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "put" (get (hs-tok tokens 0) "value"))
(assert= "string" (get (hs-tok tokens 1) "type"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "into" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "my" (get (hs-tok tokens 3) "value"))))
(deftest
"on click send custom(foo:\"fromBar\") to #d2"
(let
((tokens (hs-tokenize "on click send custom(foo:\"fromBar\") to #d2")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "on" (get (hs-tok tokens 0) "value"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "send" (get (hs-tok tokens 2) "value"))))
(deftest
"toggle between .foo and .bar"
(let
((tokens (hs-tokenize "toggle between .foo and .bar")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "toggle" (get (hs-tok tokens 0) "value"))
(assert= "keyword" (get (hs-tok tokens 1) "type"))
(assert= "between" (get (hs-tok tokens 1) "value"))
(assert= "class" (get (hs-tok tokens 2) "type"))
(assert= "foo" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "and" (get (hs-tok tokens 3) "value"))
(assert= "class" (get (hs-tok tokens 4) "type"))
(assert= "bar" (get (hs-tok tokens 4) "value"))))
(deftest
"if true put \"foo\" into me.innerHTML else put \"bar\" into me.innerHTML end"
(let
((tokens (hs-tokenize "if true put \"foo\" into me.innerHTML else put \"bar\" into me.innerHTML end")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "if" (get (hs-tok tokens 0) "value"))
(assert= "keyword" (get (hs-tok tokens 1) "type"))
(assert= "true" (get (hs-tok tokens 1) "value"))))
(deftest
"on click[buttons==0] log event"
(let
((tokens (hs-tokenize "on click[buttons==0] log event")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "ident" (get (hs-tok tokens 1) "type"))
(assert= "click" (get (hs-tok tokens 1) "value"))
(assert= "bracket-open" (get (hs-tok tokens 2) "type"))
(assert= "ident" (get (hs-tok tokens 3) "type"))
(assert= "buttons" (get (hs-tok tokens 3) "value"))
(assert= "op" (get (hs-tok tokens 4) "type"))
(assert= "==" (get (hs-tok tokens 4) "value"))))
(deftest
"wait 100ms then add .done"
(let
((tokens (hs-tokenize "wait 100ms then add .done")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "wait" (get (hs-tok tokens 0) "value"))
(assert= "number" (get (hs-tok tokens 1) "type"))
(assert= "100ms" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "then" (get (hs-tok tokens 2) "value"))
(assert= "keyword" (get (hs-tok tokens 3) "type"))
(assert= "add" (get (hs-tok tokens 3) "value"))
(assert= "class" (get (hs-tok tokens 4) "type"))
(assert= "done" (get (hs-tok tokens 4) "value"))))
(deftest
"increment @count then put it into me"
(let
((tokens (hs-tokenize "increment @count then put it into me")))
(assert= "keyword" (get (hs-tok tokens 0) "type"))
(assert= "increment" (get (hs-tok tokens 0) "value"))
(assert= "attr" (get (hs-tok tokens 1) "type"))
(assert= "count" (get (hs-tok tokens 1) "value"))
(assert= "keyword" (get (hs-tok tokens 4) "type"))
(assert= "it" (get (hs-tok tokens 4) "value"))))
(deftest
"on click from #bar add .clicked"
(let
((tokens (hs-tokenize "on click from #bar add .clicked")))
(assert= "keyword" (get (hs-tok tokens 2) "type"))
(assert= "from" (get (hs-tok tokens 2) "value"))
(assert= "id" (get (hs-tok tokens 3) "type"))
(assert= "bar" (get (hs-tok tokens 3) "value")))))
Reference in New Issue View Git Blame Copy Permalink