;; Common Lisp tokenizer tests

(define cl-test-pass 0)
(define cl-test-fail 0)
(define cl-test-fails (list))

(define
  cl-test
  (fn
    (name actual expected)
    (if
      (= actual expected)
      (set! cl-test-pass (+ cl-test-pass 1))
      (do
        (set! cl-test-fail (+ cl-test-fail 1))
        (append! cl-test-fails {:name name :expected expected :actual actual})))))

;; Helpers: extract types and values from token stream (drops eof)
(define
  cl-tok-types
  (fn
    (src)
    (map
      (fn (t) (get t "type"))
      (filter (fn (t) (not (= (get t "type") "eof"))) (cl-tokenize src)))))

(define
  cl-tok-values
  (fn
    (src)
    (map
      (fn (t) (get t "value"))
      (filter (fn (t) (not (= (get t "type") "eof"))) (cl-tokenize src)))))

(define
  cl-tok-first
  (fn (src) (nth (cl-tokenize src) 0)))

;; ── symbols ───────────────────────────────────────────────────────

(cl-test "symbol: bare lowercase" (cl-tok-values "foo") (list "FOO"))
(cl-test "symbol: uppercase" (cl-tok-values "BAR") (list "BAR"))
(cl-test "symbol: mixed case folded" (cl-tok-values "FooBar") (list "FOOBAR"))
(cl-test "symbol: with hyphen" (cl-tok-values "foo-bar") (list "FOO-BAR"))
(cl-test "symbol: with star" (cl-tok-values "*special*") (list "*SPECIAL*"))
(cl-test "symbol: with question" (cl-tok-values "null?") (list "NULL?"))
(cl-test "symbol: with exclamation" (cl-tok-values "set!") (list "SET!"))
(cl-test "symbol: plus sign alone" (cl-tok-values "+") (list "+"))
(cl-test "symbol: minus sign alone" (cl-tok-values "-") (list "-"))
(cl-test "symbol: type is symbol" (cl-tok-types "foo") (list "symbol"))

;; ── package-qualified symbols ─────────────────────────────────────

(cl-test "symbol: pkg:sym external" (cl-tok-values "cl:car") (list "CL:CAR"))
(cl-test "symbol: pkg::sym internal" (cl-tok-values "pkg::foo") (list "PKG::FOO"))
(cl-test "symbol: cl:car type" (cl-tok-types "cl:car") (list "symbol"))

;; ── keywords ──────────────────────────────────────────────────────

(cl-test "keyword: basic" (cl-tok-values ":foo") (list "FOO"))
(cl-test "keyword: type" (cl-tok-types ":foo") (list "keyword"))
(cl-test "keyword: upcase" (cl-tok-values ":hello-world") (list "HELLO-WORLD"))
(cl-test "keyword: multiple" (cl-tok-types ":a :b :c") (list "keyword" "keyword" "keyword"))

;; ── integers ──────────────────────────────────────────────────────

(cl-test "integer: zero" (cl-tok-values "0") (list "0"))
(cl-test "integer: positive" (cl-tok-values "42") (list "42"))
(cl-test "integer: negative" (cl-tok-values "-5") (list "-5"))
(cl-test "integer: positive-sign" (cl-tok-values "+3") (list "+3"))
(cl-test "integer: type" (cl-tok-types "42") (list "integer"))
(cl-test "integer: multi-digit" (cl-tok-values "12345678") (list "12345678"))

;; ── hex, binary, octal ───────────────────────────────────────────

(cl-test "hex: lowercase x" (cl-tok-values "#xFF") (list "#xFF"))
(cl-test "hex: uppercase X" (cl-tok-values "#XFF") (list "#XFF"))
(cl-test "hex: type" (cl-tok-types "#xFF") (list "integer"))
(cl-test "hex: zero" (cl-tok-values "#x0") (list "#x0"))
(cl-test "binary: #b" (cl-tok-values "#b1010") (list "#b1010"))
(cl-test "binary: type" (cl-tok-types "#b1010") (list "integer"))
(cl-test "octal: #o" (cl-tok-values "#o17") (list "#o17"))
(cl-test "octal: type" (cl-tok-types "#o17") (list "integer"))

;; ── floats ────────────────────────────────────────────────────────

(cl-test "float: basic" (cl-tok-values "3.14") (list "3.14"))
(cl-test "float: type" (cl-tok-types "3.14") (list "float"))
(cl-test "float: negative" (cl-tok-values "-2.5") (list "-2.5"))
(cl-test "float: exponent" (cl-tok-values "1.0e10") (list "1.0e10"))
(cl-test "float: neg exponent" (cl-tok-values "1.5e-3") (list "1.5e-3"))
(cl-test "float: leading dot" (cl-tok-values ".5") (list "0.5"))
(cl-test "float: exp only" (cl-tok-values "1e5") (list "1e5"))

;; ── ratios ────────────────────────────────────────────────────────

(cl-test "ratio: 1/3" (cl-tok-values "1/3") (list "1/3"))
(cl-test "ratio: type" (cl-tok-types "1/3") (list "ratio"))
(cl-test "ratio: 22/7" (cl-tok-values "22/7") (list "22/7"))
(cl-test "ratio: negative" (cl-tok-values "-1/2") (list "-1/2"))

;; ── strings ───────────────────────────────────────────────────────

(cl-test "string: empty" (cl-tok-values "\"\"") (list ""))
(cl-test "string: basic" (cl-tok-values "\"hello\"") (list "hello"))
(cl-test "string: type" (cl-tok-types "\"hello\"") (list "string"))
(cl-test "string: with space" (cl-tok-values "\"hello world\"") (list "hello world"))
(cl-test "string: escaped quote" (cl-tok-values "\"say \\\"hi\\\"\"") (list "say \"hi\""))
(cl-test "string: escaped backslash" (cl-tok-values "\"a\\\\b\"") (list "a\\b"))
(cl-test "string: newline escape" (cl-tok-values "\"a\\nb\"") (list "a\nb"))
(cl-test "string: tab escape" (cl-tok-values "\"a\\tb\"") (list "a\tb"))

;; ── characters ────────────────────────────────────────────────────

(cl-test "char: lowercase a" (cl-tok-values "#\\a") (list "a"))
(cl-test "char: uppercase A" (cl-tok-values "#\\A") (list "A"))
(cl-test "char: digit" (cl-tok-values "#\\1") (list "1"))
(cl-test "char: type" (cl-tok-types "#\\a") (list "char"))
(cl-test "char: Space" (cl-tok-values "#\\Space") (list " "))
(cl-test "char: Newline" (cl-tok-values "#\\Newline") (list "\n"))
(cl-test "char: Tab" (cl-tok-values "#\\Tab") (list "\t"))
(cl-test "char: Return" (cl-tok-values "#\\Return") (list "\r"))

;; ── reader macros ─────────────────────────────────────────────────

(cl-test "quote: type" (cl-tok-types "'x") (list "quote" "symbol"))
(cl-test "backquote: type" (cl-tok-types "`x") (list "backquote" "symbol"))
(cl-test "comma: type" (cl-tok-types ",x") (list "comma" "symbol"))
(cl-test "comma-at: type" (cl-tok-types ",@x") (list "comma-at" "symbol"))
(cl-test "hash-quote: type" (cl-tok-types "#'foo") (list "hash-quote" "symbol"))
(cl-test "hash-paren: type" (cl-tok-types "#(1 2)") (list "hash-paren" "integer" "integer" "rparen"))

;; ── uninterned ────────────────────────────────────────────────────

(cl-test "uninterned: type" (cl-tok-types "#:foo") (list "uninterned"))
(cl-test "uninterned: value upcase" (cl-tok-values "#:foo") (list "FOO"))
(cl-test "uninterned: compound" (cl-tok-values "#:my-sym") (list "MY-SYM"))

;; ── parens and structure ──────────────────────────────────────────

(cl-test "paren: empty list" (cl-tok-types "()") (list "lparen" "rparen"))
(cl-test "paren: nested" (cl-tok-types "((a))") (list "lparen" "lparen" "symbol" "rparen" "rparen"))
(cl-test "dot: standalone" (cl-tok-types "(a . b)") (list "lparen" "symbol" "dot" "symbol" "rparen"))

;; ── comments ──────────────────────────────────────────────────────

(cl-test "comment: line" (cl-tok-types "; comment\nfoo") (list "symbol"))
(cl-test "comment: inline" (cl-tok-values "foo ; bar\nbaz") (list "FOO" "BAZ"))
(cl-test "block-comment: basic" (cl-tok-types "#| hello |# foo") (list "symbol"))
(cl-test "block-comment: nested" (cl-tok-types "#| a #| b |# c |# x") (list "symbol"))

;; ── combined ──────────────────────────────────────────────────────

(cl-test
  "combined: defun skeleton"
  (cl-tok-types "(defun foo (x) x)")
  (list "lparen" "symbol" "symbol" "lparen" "symbol" "rparen" "symbol" "rparen"))

(cl-test
  "combined: let form"
  (cl-tok-types "(let ((x 1)) x)")
  (list
    "lparen"
    "symbol"
    "lparen"
    "lparen"
    "symbol"
    "integer"
    "rparen"
    "rparen"
    "symbol"
    "rparen"))

(cl-test
  "combined: whitespace skip"
  (cl-tok-values "  foo   bar   baz  ")
  (list "FOO" "BAR" "BAZ"))

(cl-test "eof: present" (get (nth (cl-tokenize "") 0) "type") "eof")
(cl-test "eof: at end of tokens" (get (nth (cl-tokenize "x") 1) "type") "eof")
