giles
13d0ebcce8
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Has been cancelled
lib/common-lisp/reader.sx — CL tokenizer: symbols with package qualification (pkg:sym/pkg::sym), integers, floats, ratios, hex/ binary/octal (#xFF/#b1010/#o17), strings with escapes, #\ char literals (named + bare), reader macros (#' #( #: ,@), line and nested block comments. lib/common-lisp/tests/read.sx — 79 tests, all green. lib/common-lisp/test.sh — test runner (sx_server pipe protocol). Key SX gotcha: use str not concat for string building. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
181 lines
9.0 KiB
Plaintext
181 lines
9.0 KiB
Plaintext
;; Common Lisp tokenizer tests
|
|
|
|
(define cl-test-pass 0)
|
|
(define cl-test-fail 0)
|
|
(define cl-test-fails (list))
|
|
|
|
(define
|
|
cl-test
|
|
(fn
|
|
(name actual expected)
|
|
(if
|
|
(= actual expected)
|
|
(set! cl-test-pass (+ cl-test-pass 1))
|
|
(do
|
|
(set! cl-test-fail (+ cl-test-fail 1))
|
|
(append! cl-test-fails {:name name :expected expected :actual actual})))))
|
|
|
|
;; Helpers: extract types and values from token stream (drops eof)
|
|
(define
|
|
cl-tok-types
|
|
(fn
|
|
(src)
|
|
(map
|
|
(fn (t) (get t "type"))
|
|
(filter (fn (t) (not (= (get t "type") "eof"))) (cl-tokenize src)))))
|
|
|
|
(define
|
|
cl-tok-values
|
|
(fn
|
|
(src)
|
|
(map
|
|
(fn (t) (get t "value"))
|
|
(filter (fn (t) (not (= (get t "type") "eof"))) (cl-tokenize src)))))
|
|
|
|
(define
|
|
cl-tok-first
|
|
(fn (src) (nth (cl-tokenize src) 0)))
|
|
|
|
;; ── symbols ───────────────────────────────────────────────────────
|
|
|
|
(cl-test "symbol: bare lowercase" (cl-tok-values "foo") (list "FOO"))
|
|
(cl-test "symbol: uppercase" (cl-tok-values "BAR") (list "BAR"))
|
|
(cl-test "symbol: mixed case folded" (cl-tok-values "FooBar") (list "FOOBAR"))
|
|
(cl-test "symbol: with hyphen" (cl-tok-values "foo-bar") (list "FOO-BAR"))
|
|
(cl-test "symbol: with star" (cl-tok-values "*special*") (list "*SPECIAL*"))
|
|
(cl-test "symbol: with question" (cl-tok-values "null?") (list "NULL?"))
|
|
(cl-test "symbol: with exclamation" (cl-tok-values "set!") (list "SET!"))
|
|
(cl-test "symbol: plus sign alone" (cl-tok-values "+") (list "+"))
|
|
(cl-test "symbol: minus sign alone" (cl-tok-values "-") (list "-"))
|
|
(cl-test "symbol: type is symbol" (cl-tok-types "foo") (list "symbol"))
|
|
|
|
;; ── package-qualified symbols ─────────────────────────────────────
|
|
|
|
(cl-test "symbol: pkg:sym external" (cl-tok-values "cl:car") (list "CL:CAR"))
|
|
(cl-test "symbol: pkg::sym internal" (cl-tok-values "pkg::foo") (list "PKG::FOO"))
|
|
(cl-test "symbol: cl:car type" (cl-tok-types "cl:car") (list "symbol"))
|
|
|
|
;; ── keywords ──────────────────────────────────────────────────────
|
|
|
|
(cl-test "keyword: basic" (cl-tok-values ":foo") (list "FOO"))
|
|
(cl-test "keyword: type" (cl-tok-types ":foo") (list "keyword"))
|
|
(cl-test "keyword: upcase" (cl-tok-values ":hello-world") (list "HELLO-WORLD"))
|
|
(cl-test "keyword: multiple" (cl-tok-types ":a :b :c") (list "keyword" "keyword" "keyword"))
|
|
|
|
;; ── integers ──────────────────────────────────────────────────────
|
|
|
|
(cl-test "integer: zero" (cl-tok-values "0") (list "0"))
|
|
(cl-test "integer: positive" (cl-tok-values "42") (list "42"))
|
|
(cl-test "integer: negative" (cl-tok-values "-5") (list "-5"))
|
|
(cl-test "integer: positive-sign" (cl-tok-values "+3") (list "+3"))
|
|
(cl-test "integer: type" (cl-tok-types "42") (list "integer"))
|
|
(cl-test "integer: multi-digit" (cl-tok-values "12345678") (list "12345678"))
|
|
|
|
;; ── hex, binary, octal ───────────────────────────────────────────
|
|
|
|
(cl-test "hex: lowercase x" (cl-tok-values "#xFF") (list "#xFF"))
|
|
(cl-test "hex: uppercase X" (cl-tok-values "#XFF") (list "#XFF"))
|
|
(cl-test "hex: type" (cl-tok-types "#xFF") (list "integer"))
|
|
(cl-test "hex: zero" (cl-tok-values "#x0") (list "#x0"))
|
|
(cl-test "binary: #b" (cl-tok-values "#b1010") (list "#b1010"))
|
|
(cl-test "binary: type" (cl-tok-types "#b1010") (list "integer"))
|
|
(cl-test "octal: #o" (cl-tok-values "#o17") (list "#o17"))
|
|
(cl-test "octal: type" (cl-tok-types "#o17") (list "integer"))
|
|
|
|
;; ── floats ────────────────────────────────────────────────────────
|
|
|
|
(cl-test "float: basic" (cl-tok-values "3.14") (list "3.14"))
|
|
(cl-test "float: type" (cl-tok-types "3.14") (list "float"))
|
|
(cl-test "float: negative" (cl-tok-values "-2.5") (list "-2.5"))
|
|
(cl-test "float: exponent" (cl-tok-values "1.0e10") (list "1.0e10"))
|
|
(cl-test "float: neg exponent" (cl-tok-values "1.5e-3") (list "1.5e-3"))
|
|
(cl-test "float: leading dot" (cl-tok-values ".5") (list "0.5"))
|
|
(cl-test "float: exp only" (cl-tok-values "1e5") (list "1e5"))
|
|
|
|
;; ── ratios ────────────────────────────────────────────────────────
|
|
|
|
(cl-test "ratio: 1/3" (cl-tok-values "1/3") (list "1/3"))
|
|
(cl-test "ratio: type" (cl-tok-types "1/3") (list "ratio"))
|
|
(cl-test "ratio: 22/7" (cl-tok-values "22/7") (list "22/7"))
|
|
(cl-test "ratio: negative" (cl-tok-values "-1/2") (list "-1/2"))
|
|
|
|
;; ── strings ───────────────────────────────────────────────────────
|
|
|
|
(cl-test "string: empty" (cl-tok-values "\"\"") (list ""))
|
|
(cl-test "string: basic" (cl-tok-values "\"hello\"") (list "hello"))
|
|
(cl-test "string: type" (cl-tok-types "\"hello\"") (list "string"))
|
|
(cl-test "string: with space" (cl-tok-values "\"hello world\"") (list "hello world"))
|
|
(cl-test "string: escaped quote" (cl-tok-values "\"say \\\"hi\\\"\"") (list "say \"hi\""))
|
|
(cl-test "string: escaped backslash" (cl-tok-values "\"a\\\\b\"") (list "a\\b"))
|
|
(cl-test "string: newline escape" (cl-tok-values "\"a\\nb\"") (list "a\nb"))
|
|
(cl-test "string: tab escape" (cl-tok-values "\"a\\tb\"") (list "a\tb"))
|
|
|
|
;; ── characters ────────────────────────────────────────────────────
|
|
|
|
(cl-test "char: lowercase a" (cl-tok-values "#\\a") (list "a"))
|
|
(cl-test "char: uppercase A" (cl-tok-values "#\\A") (list "A"))
|
|
(cl-test "char: digit" (cl-tok-values "#\\1") (list "1"))
|
|
(cl-test "char: type" (cl-tok-types "#\\a") (list "char"))
|
|
(cl-test "char: Space" (cl-tok-values "#\\Space") (list " "))
|
|
(cl-test "char: Newline" (cl-tok-values "#\\Newline") (list "\n"))
|
|
(cl-test "char: Tab" (cl-tok-values "#\\Tab") (list "\t"))
|
|
(cl-test "char: Return" (cl-tok-values "#\\Return") (list "\r"))
|
|
|
|
;; ── reader macros ─────────────────────────────────────────────────
|
|
|
|
(cl-test "quote: type" (cl-tok-types "'x") (list "quote" "symbol"))
|
|
(cl-test "backquote: type" (cl-tok-types "`x") (list "backquote" "symbol"))
|
|
(cl-test "comma: type" (cl-tok-types ",x") (list "comma" "symbol"))
|
|
(cl-test "comma-at: type" (cl-tok-types ",@x") (list "comma-at" "symbol"))
|
|
(cl-test "hash-quote: type" (cl-tok-types "#'foo") (list "hash-quote" "symbol"))
|
|
(cl-test "hash-paren: type" (cl-tok-types "#(1 2)") (list "hash-paren" "integer" "integer" "rparen"))
|
|
|
|
;; ── uninterned ────────────────────────────────────────────────────
|
|
|
|
(cl-test "uninterned: type" (cl-tok-types "#:foo") (list "uninterned"))
|
|
(cl-test "uninterned: value upcase" (cl-tok-values "#:foo") (list "FOO"))
|
|
(cl-test "uninterned: compound" (cl-tok-values "#:my-sym") (list "MY-SYM"))
|
|
|
|
;; ── parens and structure ──────────────────────────────────────────
|
|
|
|
(cl-test "paren: empty list" (cl-tok-types "()") (list "lparen" "rparen"))
|
|
(cl-test "paren: nested" (cl-tok-types "((a))") (list "lparen" "lparen" "symbol" "rparen" "rparen"))
|
|
(cl-test "dot: standalone" (cl-tok-types "(a . b)") (list "lparen" "symbol" "dot" "symbol" "rparen"))
|
|
|
|
;; ── comments ──────────────────────────────────────────────────────
|
|
|
|
(cl-test "comment: line" (cl-tok-types "; comment\nfoo") (list "symbol"))
|
|
(cl-test "comment: inline" (cl-tok-values "foo ; bar\nbaz") (list "FOO" "BAZ"))
|
|
(cl-test "block-comment: basic" (cl-tok-types "#| hello |# foo") (list "symbol"))
|
|
(cl-test "block-comment: nested" (cl-tok-types "#| a #| b |# c |# x") (list "symbol"))
|
|
|
|
;; ── combined ──────────────────────────────────────────────────────
|
|
|
|
(cl-test
|
|
"combined: defun skeleton"
|
|
(cl-tok-types "(defun foo (x) x)")
|
|
(list "lparen" "symbol" "symbol" "lparen" "symbol" "rparen" "symbol" "rparen"))
|
|
|
|
(cl-test
|
|
"combined: let form"
|
|
(cl-tok-types "(let ((x 1)) x)")
|
|
(list
|
|
"lparen"
|
|
"symbol"
|
|
"lparen"
|
|
"lparen"
|
|
"symbol"
|
|
"integer"
|
|
"rparen"
|
|
"rparen"
|
|
"symbol"
|
|
"rparen"))
|
|
|
|
(cl-test
|
|
"combined: whitespace skip"
|
|
(cl-tok-values " foo bar baz ")
|
|
(list "FOO" "BAR" "BAZ"))
|
|
|
|
(cl-test "eof: present" (get (nth (cl-tokenize "") 0) "type") "eof")
|
|
(cl-test "eof: at end of tokens" (get (nth (cl-tokenize "x") 1) "type") "eof")
|