From f1ba7177e777c865b8bbd57bb353f078bce04f4c Mon Sep 17 00:00:00 2001 From: giles Date: Mon, 6 Apr 2026 08:21:02 +0000 Subject: [PATCH] =?UTF-8?q?Step=2018=20(part=203):=20Expand=20parser=20?= =?UTF-8?q?=E2=80=94=20expressions,=20commands,=20features?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tokenizer: * and % now emit as operators (were silently swallowed) Added keywords: install, measure, behavior, called 5 new arithmetic operator tests Parser — expression layer: Arithmetic (+, -, *, /, %) via parse-arith Unary not, no, unary minus the X of Y possessive (parse-the-expr) as Type conversion, X in Y membership, array literals [...] fetch URL parsing fixed — no longer consumes "as" meant for fetch Parser — 8 new commands: return, throw, append...to, tell...end, for...in...end, make a Type, install Behavior, measure Parser — 2 new features: def name(params)...end, behavior Name(params)...end Parser — enhanced: wait for event [from target], on every event modifier 33 new parser tests (16 suites), 5 tokenizer tests. 3043/3043 full build, zero regressions. Co-Authored-By: Claude Opus 4.6 (1M context) --- lib/hyperscript/parser.sx | 316 +++++++++++++++++++++-- lib/hyperscript/tokenizer.sx | 10 +- spec/tests/test-hyperscript-parser.sx | 285 ++++++++++++++++++++ spec/tests/test-hyperscript-tokenizer.sx | 37 +++ 4 files changed, 625 insertions(+), 23 deletions(-) diff --git a/lib/hyperscript/parser.sx b/lib/hyperscript/parser.sx index 04adab8d..fea84b91 100644 --- a/lib/hyperscript/parser.sx +++ b/lib/hyperscript/parser.sx @@ -110,7 +110,14 @@ ((= typ "string") (do (adv!) val)) ((and (= typ "keyword") (= val "true")) (do (adv!) true)) ((and (= typ "keyword") (= val "false")) (do (adv!) false)) - ((and (= typ "keyword") (= val "null")) (do (adv!) nil)) + ((and (= typ "keyword") (or (= val "null") (= val "nil"))) + (do (adv!) nil)) + ((and (= typ "keyword") (= val "not")) + (do (adv!) (list (quote not) (parse-expr)))) + ((and (= typ "keyword") (= val "no")) + (do (adv!) (list (quote no) (parse-expr)))) + ((and (= typ "keyword") (= val "the")) + (do (adv!) (parse-the-expr))) ((and (= typ "keyword") (= val "me")) (do (adv!) (list (quote me)))) ((and (= typ "keyword") (or (= val "it") (= val "result"))) @@ -118,9 +125,13 @@ ((and (= typ "keyword") (= val "event")) (do (adv!) (list (quote event)))) ((and (= typ "keyword") (= val "target")) - (do (adv!) (list (quote .) (list (quote event)) "target"))) + (do + (adv!) + (list (make-symbol ".") (list (quote event)) "target"))) ((and (= typ "keyword") (= val "detail")) - (do (adv!) (list (quote .) (list (quote event)) "detail"))) + (do + (adv!) + (list (make-symbol ".") (list (quote event)) "detail"))) ((and (= typ "keyword") (= val "my")) (do (adv!) (parse-poss-tail (list (quote me))))) ((and (= typ "keyword") (= val "its")) @@ -152,6 +163,13 @@ ((expr (parse-expr))) (if (= (tp-type) "paren-close") (adv!) nil) expr))) + ((= typ "bracket-open") (do (adv!) (parse-array-lit))) + ((and (= typ "op") (= val "-")) + (do + (adv!) + (let + ((operand (parse-atom))) + (list (quote -) 0 operand)))) (true nil))))) (define parse-poss @@ -201,6 +219,24 @@ (do (adv!) (list (quote and) left (parse-expr)))) ((and (= typ "keyword") (= val "or")) (do (adv!) (list (quote or) left (parse-expr)))) + ((and (= typ "keyword") (= val "as")) + (do + (adv!) + (let + ((type-name (tp-val))) + (adv!) + (list (quote as) left type-name)))) + ((and (= typ "keyword") (= val "of")) + (do + (adv!) + (let + ((target (parse-expr))) + (if + (and (list? left) (= (first left) (quote ref))) + (list (make-symbol ".") target (nth left 1)) + (list (quote of) left target))))) + ((and (= typ "keyword") (= val "in")) + (do (adv!) (list (quote in?) left (parse-expr)))) (true left))))) (define parse-expr @@ -211,7 +247,9 @@ (if (nil? left) nil - (let ((left2 (parse-poss left))) (parse-cmp left2)))))) + (let + ((left2 (parse-poss left))) + (let ((left3 (parse-arith left2))) (parse-cmp left3))))))) (define parse-tgt-kw (fn (kw default) (if (match-kw kw) (parse-expr) default))) @@ -307,12 +345,22 @@ parse-wait-cmd (fn () - (if - (= (tp-type) "number") - (let - ((tok (adv!))) - (list (quote wait) (parse-dur (get tok "value")))) - (list (quote wait) 0)))) + (cond + ((match-kw "for") + (let + ((event-name (tp-val))) + (adv!) + (let + ((source (if (match-kw "from") (parse-expr) nil))) + (if + source + (list (quote wait-for) event-name :from source) + (list (quote wait-for) event-name))))) + ((= (tp-type) "number") + (let + ((tok (adv!))) + (list (quote wait) (parse-dur (get tok "value"))))) + (true (list (quote wait) 0))))) (define parse-detail-dict (fn @@ -405,10 +453,12 @@ (fn () (let - ((url (parse-expr))) + ((url-atom (parse-atom))) (let - ((fmt (if (match-kw "as") (get (adv!) "value") "json"))) - (list (quote fetch) url fmt))))) + ((url (if (nil? url-atom) url-atom (parse-arith (parse-poss url-atom))))) + (let + ((fmt (if (match-kw "as") (let ((f (tp-val))) (adv!) f) "json"))) + (list (quote fetch) url fmt)))))) (define parse-call-args (fn @@ -453,6 +503,206 @@ (define parse-go-cmd (fn () (match-kw "to") (list (quote go) (parse-expr)))) + (begin + (define + parse-arith + (fn + (left) + (let + ((typ (tp-type)) (val (tp-val))) + (if + (and + (= typ "op") + (or + (= val "+") + (= val "-") + (= val "*") + (= val "/") + (= val "%"))) + (do + (adv!) + (let + ((op (cond ((= val "+") (quote +)) ((= val "-") (quote -)) ((= val "*") (quote *)) ((= val "/") (quote /)) ((= val "%") (make-symbol "%"))))) + (let + ((right (let ((a (parse-atom))) (if (nil? a) a (parse-poss a))))) + (parse-arith (list op left right))))) + left)))) + (define + parse-the-expr + (fn + () + (let + ((typ (tp-type)) (val (tp-val))) + (if + (or (= typ "ident") (= typ "keyword")) + (do + (adv!) + (if + (match-kw "of") + (list (make-symbol ".") (parse-expr) val) + (cond + ((= val "result") (list (quote it))) + ((= val "first") (parse-pos-kw (quote first))) + ((= val "last") (parse-pos-kw (quote last))) + ((= val "closest") (parse-trav (quote closest))) + ((= val "next") (parse-trav (quote next))) + ((= val "previous") (parse-trav (quote previous))) + (true (list (quote ref) val))))) + (parse-atom))))) + (define + parse-array-lit + (fn + () + (define + al-collect + (fn + (acc) + (if + (or (= (tp-type) "bracket-close") (at-end?)) + (do (if (= (tp-type) "bracket-close") (adv!) nil) acc) + (let + ((elem (parse-expr))) + (if (= (tp-type) "comma") (adv!) nil) + (al-collect (append acc (list elem))))))) + (cons (quote array) (al-collect (list))))) + (define + parse-return-cmd + (fn + () + (if + (or + (at-end?) + (and + (= (tp-type) "keyword") + (or + (= (tp-val) "end") + (= (tp-val) "then") + (= (tp-val) "else")))) + (list (quote return) nil) + (list (quote return) (parse-expr))))) + (define parse-throw-cmd (fn () (list (quote throw) (parse-expr)))) + (define + parse-append-cmd + (fn + () + (let + ((value (parse-expr))) + (expect-kw! "to") + (let + ((target (parse-expr))) + (list (quote append!) value target))))) + (define + parse-tell-cmd + (fn + () + (let + ((target (parse-expr))) + (let + ((body (parse-cmd-list))) + (match-kw "end") + (list (quote tell) target body))))) + (define + parse-for-cmd + (fn + () + (let + ((var-name (tp-val))) + (adv!) + (expect-kw! "in") + (let + ((collection (parse-expr))) + (let + ((idx (if (match-kw "index") (let ((iname (tp-val))) (adv!) iname) nil))) + (let + ((body (parse-cmd-list))) + (match-kw "end") + (if + idx + (list (quote for) var-name collection body :index idx) + (list (quote for) var-name collection body)))))))) + (define + parse-make-cmd + (fn + () + (if (= (tp-val) "a") (adv!) nil) + (let + ((type-name (tp-val))) + (adv!) + (let + ((called (if (match-kw "called") (let ((n (tp-val))) (adv!) n) nil))) + (if + called + (list (quote make) type-name called) + (list (quote make) type-name)))))) + (define + parse-install-cmd + (fn + () + (let + ((name (tp-val))) + (adv!) + (if + (= (tp-type) "paren-open") + (let + ((args (parse-call-args))) + (cons (quote install) (cons name args))) + (list (quote install) name))))) + (define + parse-measure-cmd + (fn + () + (let + ((tgt (parse-expr))) + (list (quote measure) (if (nil? tgt) (list (quote me)) tgt))))) + (define + parse-param-list + (fn () (if (= (tp-type) "paren-open") (parse-call-args) (list)))) + (define + parse-feat-body + (fn + () + (define + fb-collect + (fn + (acc) + (if + (or + (at-end?) + (and (= (tp-type) "keyword") (= (tp-val) "end"))) + acc + (let + ((feat (parse-feat))) + (if + (nil? feat) + acc + (fb-collect (append acc (list feat)))))))) + (fb-collect (list)))) + (define + parse-def-feat + (fn + () + (let + ((name (tp-val))) + (adv!) + (let + ((params (parse-param-list))) + (let + ((body (parse-cmd-list))) + (match-kw "end") + (list (quote def) name params body)))))) + (define + parse-behavior-feat + (fn + () + (let + ((name (tp-val))) + (adv!) + (let + ((params (parse-param-list))) + (let + ((body (parse-feat-body))) + (match-kw "end") + (list (quote behavior) name params body))))))) (define parse-cmd (fn @@ -502,6 +752,22 @@ (do (adv!) (list (quote settle)))) ((and (= typ "keyword") (= val "go")) (do (adv!) (parse-go-cmd))) + ((and (= typ "keyword") (= val "return")) + (do (adv!) (parse-return-cmd))) + ((and (= typ "keyword") (= val "throw")) + (do (adv!) (parse-throw-cmd))) + ((and (= typ "keyword") (= val "append")) + (do (adv!) (parse-append-cmd))) + ((and (= typ "keyword") (= val "tell")) + (do (adv!) (parse-tell-cmd))) + ((and (= typ "keyword") (= val "for")) + (do (adv!) (parse-for-cmd))) + ((and (= typ "keyword") (= val "make")) + (do (adv!) (parse-make-cmd))) + ((and (= typ "keyword") (= val "install")) + (do (adv!) (parse-install-cmd))) + ((and (= typ "keyword") (= val "measure")) + (do (adv!) (parse-measure-cmd))) (true (parse-expr)))))) (define parse-cmd-list @@ -530,21 +796,25 @@ (fn () (let - ((event-name (get (adv!) "value"))) + ((every? (match-kw "every"))) (let - ((flt (if (= (tp-type) "bracket-open") (do (adv!) (let ((expr (parse-expr))) (if (= (tp-type) "bracket-close") (adv!) nil) expr)) nil))) + ((event-name (let ((v (tp-val))) (adv!) v))) (let - ((source (if (match-kw "from") (parse-expr) nil))) + ((flt (if (= (tp-type) "bracket-open") (do (adv!) (let ((f (parse-expr))) (if (= (tp-type) "bracket-close") (adv!) nil) f)) nil))) (let - ((body (parse-cmd-list))) - (match-kw "end") + ((source (if (match-kw "from") (parse-expr) nil))) (let - ((parts (list (quote on) event-name))) + ((body (parse-cmd-list))) + (match-kw "end") (let - ((parts (if source (append parts (list :from source)) parts))) + ((parts (list (quote on) event-name))) (let - ((parts (if flt (append parts (list :filter flt)) parts))) - (append parts (list body))))))))))) + ((parts (if every? (append parts (list :every true)) parts))) + (let + ((parts (if flt (append parts (list :filter flt)) parts))) + (let + ((parts (if source (append parts (list :from source)) parts))) + (append parts (list body))))))))))))) (define parse-init-feat (fn @@ -562,6 +832,8 @@ (cond ((= val "on") (do (adv!) (parse-on-feat))) ((= val "init") (do (adv!) (parse-init-feat))) + ((= val "def") (do (adv!) (parse-def-feat))) + ((= val "behavior") (do (adv!) (parse-behavior-feat))) (true (parse-cmd-list)))))) (define coll-feats diff --git a/lib/hyperscript/tokenizer.sx b/lib/hyperscript/tokenizer.sx index 6a622668..76ef2284 100644 --- a/lib/hyperscript/tokenizer.sx +++ b/lib/hyperscript/tokenizer.sx @@ -133,7 +133,11 @@ "than" "greater" "class" - "anything")) + "anything" + "install" + "measure" + "behavior" + "called")) (define hs-keyword? (fn (word) (some (fn (k) (= k word)) hs-keywords))) @@ -507,6 +511,10 @@ (do (hs-emit! "op" ">" start) (hs-advance! 1) (scan!)) (= ch "!") (do (hs-emit! "op" "!" start) (hs-advance! 1) (scan!)) + (= ch "*") + (do (hs-emit! "op" "*" start) (hs-advance! 1) (scan!)) + (= ch "%") + (do (hs-emit! "op" "%" start) (hs-advance! 1) (scan!)) (= ch ".") (do (hs-emit! "dot" "." start) (hs-advance! 1) (scan!)) :else (do (hs-advance! 1) (scan!))))))) diff --git a/spec/tests/test-hyperscript-parser.sx b/spec/tests/test-hyperscript-parser.sx index eed03a85..2a57e5a4 100644 --- a/spec/tests/test-hyperscript-parser.sx +++ b/spec/tests/test-hyperscript-parser.sx @@ -375,6 +375,291 @@ (assert= "hello" (nth ast 2))))) ;; ── Full expressions (matching tokenizer conformance) ───────────── +(defsuite + "hs-parse-arithmetic" + (deftest + "addition" + (let + ((ast (hs-compile "set x to 1 + 2"))) + (let + ((val (nth ast 2))) + (assert= (quote +) (first val)) + (assert= 1 (nth val 1)) + (assert= 2 (nth val 2))))) + (deftest + "subtraction" + (let + ((ast (hs-compile "set x to 10 - 3"))) + (let + ((val (nth ast 2))) + (assert= (quote -) (first val)) + (assert= 10 (nth val 1)) + (assert= 3 (nth val 2))))) + (deftest + "multiplication" + (let + ((ast (hs-compile "set x to 4 * 5"))) + (let + ((val (nth ast 2))) + (assert= (quote *) (first val)) + (assert= 4 (nth val 1)) + (assert= 5 (nth val 2))))) + (deftest + "division" + (let + ((ast (hs-compile "set x to 10 / 2"))) + (let + ((val (nth ast 2))) + (assert= (quote /) (first val)) + (assert= 10 (nth val 1)) + (assert= 2 (nth val 2))))) + (deftest + "chained arithmetic" + (let + ((ast (hs-compile "set x to 1 + 2 + 3"))) + (let + ((val (nth ast 2))) + (assert= (quote +) (first val)) + (assert= 3 (nth val 2)))))) + +(defsuite + "hs-parse-unary" + (deftest + "not expr" + (let + ((ast (hs-compile "if not x end"))) + (let ((cnd (nth ast 1))) (assert= (quote not) (first cnd))))) + (deftest + "no expr" + (let + ((ast (hs-compile "if no x end"))) + (let ((cnd (nth ast 1))) (assert= (quote no) (first cnd))))) + (deftest + "unary minus" + (let + ((ast (hs-compile "set x to -5"))) + (let + ((val (nth ast 2))) + (assert= (quote -) (first val)) + (assert= 0 (nth val 1)) + (assert= 5 (nth val 2)))))) + +(defsuite + "hs-parse-the-of" + (deftest + "the innerHTML of me" + (let + ((ast (hs-compile "set the innerHTML of me to 'hi'"))) + (let + ((tgt (nth ast 1))) + (assert= (make-symbol ".") (first tgt)) + (assert= "innerHTML" (nth tgt 2))))) + (deftest + "the as article skip" + (let + ((ast (hs-compile "set the result to 5"))) + (let ((tgt (nth ast 1))) (assert= (quote it) (first tgt)))))) + +(defsuite + "hs-parse-as-conversion" + (deftest + "expr as Int" + (let + ((ast (hs-compile "set x to y as Int"))) + (let + ((val (nth ast 2))) + (assert= (quote as) (first val)) + (assert= "Int" (nth val 2))))) + (deftest + "expr as String" + (let + ((ast (hs-compile "set x to count as String"))) + (let + ((val (nth ast 2))) + (assert= (quote as) (first val)) + (assert= "String" (nth val 2)))))) + +(defsuite + "hs-parse-in-operator" + (deftest + "x in collection" + (let + ((ast (hs-compile "if x in items end"))) + (let ((cnd (nth ast 1))) (assert= (quote in?) (first cnd)))))) + +(defsuite + "hs-parse-array-literals" + (deftest + "empty array" + (let + ((ast (hs-compile "set x to []"))) + (let + ((val (nth ast 2))) + (assert= (quote array) (first val)) + (assert= 1 (len val))))) + (deftest + "array with elements" + (let + ((ast (hs-compile "set x to [1, 2, 3]"))) + (let + ((val (nth ast 2))) + (assert= (quote array) (first val)) + (assert= 4 (len val)) + (assert= 1 (nth val 1)) + (assert= 2 (nth val 2)) + (assert= 3 (nth val 3)))))) + +(defsuite + "hs-parse-return-throw" + (deftest + "return expr" + (let + ((ast (hs-compile "return 42"))) + (assert= (quote return) (first ast)) + (assert= 42 (nth ast 1)))) + (deftest + "return bare" + (let + ((ast (hs-compile "return"))) + (assert= (quote return) (first ast)) + (assert= nil (nth ast 1)))) + (deftest + "throw expr" + (let + ((ast (hs-compile "throw 'error'"))) + (assert= (quote throw) (first ast)) + (assert= "error" (nth ast 1))))) + +(defsuite + "hs-parse-append" + (deftest + "append to target" + (let + ((ast (hs-compile "append 'hello' to me"))) + (assert= (quote append!) (first ast)) + (assert= "hello" (nth ast 1)) + (assert= (quote me) (first (nth ast 2)))))) + +(defsuite + "hs-parse-tell" + (deftest + "tell target commands end" + (let + ((ast (hs-compile "tell
add .active end"))) + (assert= (quote tell) (first ast)) + (assert= (quote query) (first (nth ast 1))) + (assert= (quote add-class) (first (nth ast 2)))))) + +(defsuite + "hs-parse-for" + (deftest + "for x in items end" + (let + ((ast (hs-compile "for item in items log item end"))) + (assert= (quote for) (first ast)) + (assert= "item" (nth ast 1)) + (assert= (quote ref) (first (nth ast 2))) + (assert= (quote log) (first (nth ast 3))))) + (deftest + "for with index" + (let + ((ast (hs-compile "for item in items index i log item end"))) + (assert= (quote for) (first ast)) + (assert= "item" (nth ast 1))))) + +(defsuite + "hs-parse-make" + (deftest + "make a Object" + (let + ((ast (hs-compile "make a Object"))) + (assert= (quote make) (first ast)) + (assert= "Object" (nth ast 1)))) + (deftest + "make a Set called s" + (let + ((ast (hs-compile "make a Set called s"))) + (assert= (quote make) (first ast)) + (assert= "Set" (nth ast 1)) + (assert= "s" (nth ast 2))))) + +(defsuite + "hs-parse-install" + (deftest + "install behavior" + (let + ((ast (hs-compile "install Draggable"))) + (assert= (quote install) (first ast)) + (assert= "Draggable" (nth ast 1)))) + (deftest + "install with args" + (let + ((ast (hs-compile "install Sortable(true)"))) + (assert= (quote install) (first ast)) + (assert= "Sortable" (nth ast 1)) + (assert= true (nth ast 2))))) + +(defsuite + "hs-parse-measure" + (deftest + "measure target" + (let + ((ast (hs-compile "measure me"))) + (assert= (quote measure) (first ast)) + (assert= (quote me) (first (nth ast 1)))))) + +(defsuite + "hs-parse-wait-for" + (deftest + "wait for transitionend" + (let + ((ast (hs-compile "wait for transitionend"))) + (assert= (quote wait-for) (first ast)) + (assert= "transitionend" (nth ast 1)))) + (deftest + "wait for click from target" + (let + ((ast (hs-compile "wait for click from #btn"))) + (assert= (quote wait-for) (first ast)) + (assert= "click" (nth ast 1)) + (assert= :from (nth ast 2))))) + +(defsuite + "hs-parse-def-behavior" + (deftest + "def function" + (let + ((ast (hs-compile "def greet(name) log name end"))) + (assert= (quote def) (first ast)) + (assert= "greet" (nth ast 1)) + (assert= 1 (len (nth ast 2))) + (assert= (quote log) (first (nth ast 3))))) + (deftest + "behavior with on handler" + (let + ((ast (hs-compile "behavior Clickable on click add .clicked end end"))) + (assert= (quote behavior) (first ast)) + (assert= "Clickable" (nth ast 1)) + (assert= 1 (len (nth ast 3))))) + (deftest + "def no params" + (let + ((ast (hs-compile "def reset() set x to 0 end"))) + (assert= (quote def) (first ast)) + (assert= "reset" (nth ast 1)) + (assert= 0 (len (nth ast 2)))))) + +(defsuite + "hs-parse-every-modifier" + (deftest + "on every click" + (let + ((ast (hs-compile "on every click add .pulse end"))) + (assert= (quote on) (first ast)) + (assert= "click" (nth ast 1)) + (assert= :every (nth ast 2)) + (assert= true (nth ast 3))))) + (defsuite "hs-parse-conformance" (deftest diff --git a/spec/tests/test-hyperscript-tokenizer.sx b/spec/tests/test-hyperscript-tokenizer.sx index ab3e6c22..61195578 100644 --- a/spec/tests/test-hyperscript-tokenizer.sx +++ b/spec/tests/test-hyperscript-tokenizer.sx @@ -185,6 +185,43 @@ (assert= "number" (get (hs-tok tokens 1) "type")) (assert= "bracket-close" (get (hs-tok tokens 2) "type"))))) +(defsuite + "hs-tokenize-arithmetic-ops" + (deftest + "multiply operator" + (let + ((toks (hs-tokenize "4 * 5"))) + (assert= "number" (get (nth toks 0) "type")) + (assert= "op" (get (nth toks 1) "type")) + (assert= "*" (get (nth toks 1) "value")) + (assert= "number" (get (nth toks 2) "type")))) + (deftest + "modulo operator" + (let + ((toks (hs-tokenize "10 % 3"))) + (assert= "op" (get (nth toks 1) "type")) + (assert= "%" (get (nth toks 1) "value")))) + (deftest + "star as style not operator" + (let + ((toks (hs-tokenize "*color"))) + (assert= "style" (get (nth toks 0) "type")) + (assert= "color" (get (nth toks 0) "value")))) + (deftest + "division operator" + (let + ((toks (hs-tokenize "10 / 2"))) + (assert= "op" (get (nth toks 1) "type")) + (assert= "/" (get (nth toks 1) "value")))) + (deftest + "mixed arithmetic" + (let + ((toks (hs-tokenize "1 + 2 * 3"))) + (assert= "op" (get (nth toks 1) "type")) + (assert= "+" (get (nth toks 1) "value")) + (assert= "op" (get (nth toks 3) "type")) + (assert= "*" (get (nth toks 3) "value"))))) + (defsuite "hs-tokenize-comments" (deftest