giles
99753580b4
Salvaged from worktree-agent-* branches killed during sx-tree MCP outage: - lua: tokenizer + parser + phase-2 transpile (~157 tests) - prolog: tokenizer + parser + unification (72 tests, plan update lost to WIP) - forth: phase-1 reader/interpreter + phase-2 colon/VARIABLE (134 tests) - erlang: tokenizer + parser (114 tests) - haskell: tokenizer + parse tests (43 tests) Cherry-picked file contents only, not branch history, to avoid pulling in unrelated ocaml-vm merge commits that were in those branches' bases.
246 lines
7.7 KiB
Plaintext
246 lines
7.7 KiB
Plaintext
;; Erlang tokenizer tests
|
|
|
|
(define er-test-count 0)
|
|
(define er-test-pass 0)
|
|
(define er-test-fails (list))
|
|
|
|
(define tok-type (fn (t) (get t :type)))
|
|
(define tok-value (fn (t) (get t :value)))
|
|
|
|
(define tok-types (fn (src) (map tok-type (er-tokenize src))))
|
|
|
|
(define tok-values (fn (src) (map tok-value (er-tokenize src))))
|
|
|
|
(define
|
|
er-test
|
|
(fn
|
|
(name actual expected)
|
|
(set! er-test-count (+ er-test-count 1))
|
|
(if
|
|
(= actual expected)
|
|
(set! er-test-pass (+ er-test-pass 1))
|
|
(append! er-test-fails {:actual actual :expected expected :name name}))))
|
|
|
|
;; ── atoms ─────────────────────────────────────────────────────────
|
|
(er-test "atom: bare" (tok-values "foo") (list "foo" nil))
|
|
|
|
(er-test
|
|
"atom: snake_case"
|
|
(tok-values "hello_world")
|
|
(list "hello_world" nil))
|
|
|
|
(er-test
|
|
"atom: quoted"
|
|
(tok-values "'Hello World'")
|
|
(list "Hello World" nil))
|
|
|
|
(er-test
|
|
"atom: quoted with special chars"
|
|
(tok-values "'foo-bar'")
|
|
(list "foo-bar" nil))
|
|
|
|
(er-test "atom: with @" (tok-values "node@host") (list "node@host" nil))
|
|
|
|
(er-test
|
|
"atom: type is atom"
|
|
(tok-types "foo bar baz")
|
|
(list "atom" "atom" "atom" "eof"))
|
|
|
|
;; ── variables ─────────────────────────────────────────────────────
|
|
(er-test "var: uppercase" (tok-values "X") (list "X" nil))
|
|
|
|
(er-test "var: camelcase" (tok-values "FooBar") (list "FooBar" nil))
|
|
|
|
(er-test "var: underscore" (tok-values "_") (list "_" nil))
|
|
|
|
(er-test "var: _prefixed" (tok-values "_ignored") (list "_ignored" nil))
|
|
|
|
(er-test "var: type" (tok-types "X Y _") (list "var" "var" "var" "eof"))
|
|
|
|
;; ── integers ──────────────────────────────────────────────────────
|
|
(er-test "integer: zero" (tok-values "0") (list "0" nil))
|
|
|
|
(er-test "integer: positive" (tok-values "42") (list "42" nil))
|
|
|
|
(er-test "integer: big" (tok-values "12345678") (list "12345678" nil))
|
|
|
|
(er-test "integer: hex" (tok-values "16#FF") (list "16#FF" nil))
|
|
|
|
(er-test
|
|
"integer: type"
|
|
(tok-types "1 2 3")
|
|
(list "integer" "integer" "integer" "eof"))
|
|
|
|
(er-test "integer: char literal" (tok-types "$a") (list "integer" "eof"))
|
|
|
|
(er-test
|
|
"integer: char literal escape"
|
|
(tok-types "$\\n")
|
|
(list "integer" "eof"))
|
|
|
|
;; ── floats ────────────────────────────────────────────────────────
|
|
(er-test "float: simple" (tok-values "3.14") (list "3.14" nil))
|
|
|
|
(er-test "float: exponent" (tok-values "1.0e10") (list "1.0e10" nil))
|
|
|
|
(er-test "float: neg exponent" (tok-values "1.5e-3") (list "1.5e-3" nil))
|
|
|
|
(er-test "float: type" (tok-types "3.14") (list "float" "eof"))
|
|
|
|
;; ── strings ───────────────────────────────────────────────────────
|
|
(er-test "string: simple" (tok-values "\"hello\"") (list "hello" nil))
|
|
|
|
(er-test "string: empty" (tok-values "\"\"") (list "" nil))
|
|
|
|
(er-test "string: escape newline" (tok-values "\"a\\nb\"") (list "a\nb" nil))
|
|
|
|
(er-test "string: type" (tok-types "\"hello\"") (list "string" "eof"))
|
|
|
|
;; ── keywords ──────────────────────────────────────────────────────
|
|
(er-test "keyword: case" (tok-types "case") (list "keyword" "eof"))
|
|
|
|
(er-test
|
|
"keyword: of end when"
|
|
(tok-types "of end when")
|
|
(list "keyword" "keyword" "keyword" "eof"))
|
|
|
|
(er-test
|
|
"keyword: receive after"
|
|
(tok-types "receive after")
|
|
(list "keyword" "keyword" "eof"))
|
|
|
|
(er-test
|
|
"keyword: fun try catch"
|
|
(tok-types "fun try catch")
|
|
(list "keyword" "keyword" "keyword" "eof"))
|
|
|
|
(er-test
|
|
"keyword: andalso orelse not"
|
|
(tok-types "andalso orelse not")
|
|
(list "keyword" "keyword" "keyword" "eof"))
|
|
|
|
(er-test
|
|
"keyword: div rem"
|
|
(tok-types "div rem")
|
|
(list "keyword" "keyword" "eof"))
|
|
|
|
;; ── punct ─────────────────────────────────────────────────────────
|
|
(er-test "punct: parens" (tok-values "()") (list "(" ")" nil))
|
|
|
|
(er-test "punct: braces" (tok-values "{}") (list "{" "}" nil))
|
|
|
|
(er-test "punct: brackets" (tok-values "[]") (list "[" "]" nil))
|
|
|
|
(er-test
|
|
"punct: commas"
|
|
(tok-types "a,b")
|
|
(list "atom" "punct" "atom" "eof"))
|
|
|
|
(er-test
|
|
"punct: semicolon"
|
|
(tok-types "a;b")
|
|
(list "atom" "punct" "atom" "eof"))
|
|
|
|
(er-test "punct: period" (tok-types "a.") (list "atom" "punct" "eof"))
|
|
|
|
(er-test "punct: arrow" (tok-values "->") (list "->" nil))
|
|
|
|
(er-test "punct: backarrow" (tok-values "<-") (list "<-" nil))
|
|
|
|
(er-test "punct: binary brackets" (tok-values "<<>>") (list "<<" ">>" nil))
|
|
|
|
(er-test
|
|
"punct: cons bar"
|
|
(tok-values "[a|b]")
|
|
(list "[" "a" "|" "b" "]" nil))
|
|
|
|
(er-test "punct: double-bar (list comp)" (tok-values "||") (list "||" nil))
|
|
|
|
(er-test "punct: double-colon" (tok-values "::") (list "::" nil))
|
|
|
|
(er-test
|
|
"punct: module-colon"
|
|
(tok-values "lists:map")
|
|
(list "lists" ":" "map" nil))
|
|
|
|
;; ── operators ─────────────────────────────────────────────────────
|
|
(er-test
|
|
"op: plus minus times div"
|
|
(tok-values "+ - * /")
|
|
(list "+" "-" "*" "/" nil))
|
|
|
|
(er-test
|
|
"op: eq/neq"
|
|
(tok-values "== /= =:= =/=")
|
|
(list "==" "/=" "=:=" "=/=" nil))
|
|
|
|
(er-test "op: compare" (tok-values "< > =< >=") (list "<" ">" "=<" ">=" nil))
|
|
|
|
(er-test "op: list ops" (tok-values "++ --") (list "++" "--" nil))
|
|
|
|
(er-test "op: send" (tok-values "!") (list "!" nil))
|
|
|
|
(er-test "op: match" (tok-values "=") (list "=" nil))
|
|
|
|
;; ── comments ──────────────────────────────────────────────────────
|
|
(er-test
|
|
"comment: ignored"
|
|
(tok-values "x % this is a comment\ny")
|
|
(list "x" "y" nil))
|
|
|
|
(er-test
|
|
"comment: end-of-file"
|
|
(tok-values "x % comment to eof")
|
|
(list "x" nil))
|
|
|
|
;; ── combined ──────────────────────────────────────────────────────
|
|
(er-test
|
|
"combined: function head"
|
|
(tok-values "foo(X, Y) -> X + Y.")
|
|
(list "foo" "(" "X" "," "Y" ")" "->" "X" "+" "Y" "." nil))
|
|
|
|
(er-test
|
|
"combined: case expression"
|
|
(tok-values "case X of 1 -> ok; _ -> err end")
|
|
(list "case" "X" "of" "1" "->" "ok" ";" "_" "->" "err" "end" nil))
|
|
|
|
(er-test
|
|
"combined: tuple"
|
|
(tok-values "{ok, 42}")
|
|
(list "{" "ok" "," "42" "}" nil))
|
|
|
|
(er-test
|
|
"combined: list cons"
|
|
(tok-values "[H|T]")
|
|
(list "[" "H" "|" "T" "]" nil))
|
|
|
|
(er-test
|
|
"combined: receive"
|
|
(tok-values "receive X -> X end")
|
|
(list "receive" "X" "->" "X" "end" nil))
|
|
|
|
(er-test
|
|
"combined: guard"
|
|
(tok-values "when is_integer(X)")
|
|
(list "when" "is_integer" "(" "X" ")" nil))
|
|
|
|
(er-test
|
|
"combined: module attr"
|
|
(tok-values "-module(foo).")
|
|
(list "-" "module" "(" "foo" ")" "." nil))
|
|
|
|
(er-test
|
|
"combined: send"
|
|
(tok-values "Pid ! {self(), hello}")
|
|
(list "Pid" "!" "{" "self" "(" ")" "," "hello" "}" nil))
|
|
|
|
(er-test
|
|
"combined: whitespace skip"
|
|
(tok-values " a \n b \t c ")
|
|
(list "a" "b" "c" nil))
|
|
|
|
;; ── report ────────────────────────────────────────────────────────
|
|
(define
|
|
er-tokenize-test-summary
|
|
(str "tokenizer " er-test-pass "/" er-test-count))
|