giles
96019e9fe8
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Has been cancelled
lib/ruby/tokenizer.sx — rb-tokenize: keywords, identifiers (@/@~/$/const), numbers (dec/hex/oct/bin/float), strings (dq with raw interpolation, sq), symbols, %w/%i, operators (all compound forms), punctuation, comments, line/col tracking. Plus test runner test.sh and 107 passing tests. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
550 lines
23 KiB
Plaintext
550 lines
23 KiB
Plaintext
;; Ruby tokenizer for Ruby 2.7 subset.
|
|
;; Token: {:type T :value V :line L :col C}
|
|
;;
|
|
;; Types: keyword ident ivar cvar gvar const
|
|
;; int float string symbol
|
|
;; op dot dotdot dotdotdot dcolon colon
|
|
;; lparen rparen lbracket rbracket lbrace rbrace
|
|
;; comma semi pipe newline words isymbols eof
|
|
|
|
;; ── Character code table ──────────────────────────────────────────
|
|
(define rb-ord-table
|
|
(let ((t (dict)) (i 0))
|
|
(define rb-build-table
|
|
(fn ()
|
|
(when (< i 128)
|
|
(do
|
|
(dict-set! t (char-from-code i) i)
|
|
(set! i (+ i 1))
|
|
(rb-build-table)))))
|
|
(rb-build-table)
|
|
t))
|
|
|
|
(define rb-ord (fn (c) (or (get rb-ord-table c) 0)))
|
|
|
|
;; ── Character predicates ──────────────────────────────────────────
|
|
(define rb-digit?
|
|
(fn (c) (and (string? c) (>= (rb-ord c) 48) (<= (rb-ord c) 57))))
|
|
|
|
(define rb-hex-digit?
|
|
(fn (c)
|
|
(and (string? c)
|
|
(or (and (>= (rb-ord c) 48) (<= (rb-ord c) 57))
|
|
(and (>= (rb-ord c) 97) (<= (rb-ord c) 102))
|
|
(and (>= (rb-ord c) 65) (<= (rb-ord c) 70))))))
|
|
|
|
(define rb-octal-digit?
|
|
(fn (c) (and (string? c) (>= (rb-ord c) 48) (<= (rb-ord c) 55))))
|
|
|
|
(define rb-binary-digit? (fn (c) (or (= c "0") (= c "1"))))
|
|
|
|
(define rb-lower?
|
|
(fn (c) (and (string? c) (>= (rb-ord c) 97) (<= (rb-ord c) 122))))
|
|
|
|
(define rb-upper?
|
|
(fn (c) (and (string? c) (>= (rb-ord c) 65) (<= (rb-ord c) 90))))
|
|
|
|
(define rb-ident-start?
|
|
(fn (c) (or (rb-lower? c) (rb-upper? c) (= c "_"))))
|
|
|
|
(define rb-ident-cont?
|
|
(fn (c) (or (rb-lower? c) (rb-upper? c) (rb-digit? c) (= c "_"))))
|
|
|
|
(define rb-space? (fn (c) (or (= c " ") (= c "\t") (= c "\r"))))
|
|
|
|
;; ── Reserved words ────────────────────────────────────────────────
|
|
(define rb-keywords
|
|
(list "__ENCODING__" "__LINE__" "__FILE__"
|
|
"BEGIN" "END"
|
|
"alias" "and"
|
|
"begin" "break"
|
|
"case" "class"
|
|
"def" "defined?" "do"
|
|
"else" "elsif" "end" "ensure"
|
|
"false" "for"
|
|
"if" "in"
|
|
"module"
|
|
"next" "nil" "not"
|
|
"or"
|
|
"redo" "rescue" "retry" "return"
|
|
"self" "super"
|
|
"then" "true"
|
|
"undef" "unless" "until"
|
|
"when" "while"
|
|
"yield"))
|
|
|
|
(define rb-keyword? (fn (w) (contains? rb-keywords w)))
|
|
|
|
;; ── Token constructor ─────────────────────────────────────────────
|
|
(define rb-make-token
|
|
(fn (type value line col) {:type type :value value :line line :col col}))
|
|
|
|
;; ── Radix number parser ───────────────────────────────────────────
|
|
(define rb-parse-radix
|
|
(fn (s radix)
|
|
(let ((n (len s)) (i 0) (acc 0))
|
|
(define rb-rad-loop
|
|
(fn ()
|
|
(when (< i n)
|
|
(do
|
|
(let ((c (substring s i (+ i 1))))
|
|
(cond
|
|
((and (>= (rb-ord c) 48) (<= (rb-ord c) 57))
|
|
(set! acc (+ (* acc radix) (- (rb-ord c) 48))))
|
|
((and (>= (rb-ord c) 97) (<= (rb-ord c) 102))
|
|
(set! acc (+ (* acc radix) (+ 10 (- (rb-ord c) 97)))))
|
|
((and (>= (rb-ord c) 65) (<= (rb-ord c) 70))
|
|
(set! acc (+ (* acc radix) (+ 10 (- (rb-ord c) 65)))))))
|
|
(set! i (+ i 1))
|
|
(rb-rad-loop)))))
|
|
(rb-rad-loop)
|
|
acc)))
|
|
|
|
;; ── Strip underscores from numeric literals ───────────────────────
|
|
(define rb-strip-underscores
|
|
(fn (s)
|
|
(let ((n (len s)) (i 0) (parts (list)))
|
|
(define rb-su-loop
|
|
(fn ()
|
|
(when (< i n)
|
|
(do
|
|
(let ((c (substring s i (+ i 1))))
|
|
(when (not (= c "_"))
|
|
(append! parts c)))
|
|
(set! i (+ i 1))
|
|
(rb-su-loop)))))
|
|
(rb-su-loop)
|
|
(join "" parts))))
|
|
|
|
;; ── Main tokenizer ────────────────────────────────────────────────
|
|
(define rb-tokenize
|
|
(fn (src)
|
|
(let ((tokens (list))
|
|
(pos 0)
|
|
(line 1)
|
|
(col 1)
|
|
(src-len (len src)))
|
|
(define rb-peek
|
|
(fn (offset)
|
|
(if (< (+ pos offset) src-len)
|
|
(substring src (+ pos offset) (+ pos offset 1))
|
|
nil)))
|
|
(define rb-cur (fn () (rb-peek 0)))
|
|
(define rb-advance!
|
|
(fn ()
|
|
(let ((c (rb-cur)))
|
|
(set! pos (+ pos 1))
|
|
(if (= c "\n")
|
|
(do (set! line (+ line 1)) (set! col 1))
|
|
(set! col (+ col 1))))))
|
|
(define rb-advance-n!
|
|
(fn (n)
|
|
(when (> n 0)
|
|
(do (rb-advance!) (rb-advance-n! (- n 1))))))
|
|
(define rb-push!
|
|
(fn (type value tok-line tok-col)
|
|
(append! tokens (rb-make-token type value tok-line tok-col))))
|
|
(define rb-read-while
|
|
(fn (pred)
|
|
(let ((start pos))
|
|
(define rb-rw-loop
|
|
(fn ()
|
|
(when (and (< pos src-len) (pred (rb-cur)))
|
|
(do (rb-advance!) (rb-rw-loop)))))
|
|
(rb-rw-loop)
|
|
(substring src start pos))))
|
|
(define rb-skip-line-comment!
|
|
(fn ()
|
|
(define rb-slc-loop
|
|
(fn ()
|
|
(when (and (< pos src-len) (not (= (rb-cur) "\n")))
|
|
(do (rb-advance!) (rb-slc-loop)))))
|
|
(rb-slc-loop)))
|
|
(define rb-read-escape
|
|
(fn ()
|
|
(rb-advance!)
|
|
(let ((c (rb-cur)))
|
|
(cond
|
|
((= c "n") (do (rb-advance!) "\n"))
|
|
((= c "t") (do (rb-advance!) "\t"))
|
|
((= c "r") (do (rb-advance!) "\r"))
|
|
((= c "\\") (do (rb-advance!) "\\"))
|
|
((= c "'") (do (rb-advance!) "'"))
|
|
((= c "\"") (do (rb-advance!) "\""))
|
|
((= c "a") (do (rb-advance!) (char-from-code 7)))
|
|
((= c "b") (do (rb-advance!) (char-from-code 8)))
|
|
((= c "f") (do (rb-advance!) (char-from-code 12)))
|
|
((= c "v") (do (rb-advance!) (char-from-code 11)))
|
|
((= c "e") (do (rb-advance!) (char-from-code 27)))
|
|
((= c "s") (do (rb-advance!) " "))
|
|
((= c "0") (do (rb-advance!) (char-from-code 0)))
|
|
(:else (do (rb-advance!) (str "\\" c)))))))
|
|
(define rb-read-sq-string
|
|
(fn ()
|
|
(let ((parts (list)))
|
|
(rb-advance!)
|
|
(define rb-sq-loop
|
|
(fn ()
|
|
(cond
|
|
((>= pos src-len) nil)
|
|
((= (rb-cur) "'") (rb-advance!))
|
|
((and (= (rb-cur) "\\")
|
|
(let ((n (rb-peek 1)))
|
|
(or (= n "\\") (= n "'"))))
|
|
(do
|
|
(rb-advance!)
|
|
(append! parts (rb-cur))
|
|
(rb-advance!)
|
|
(rb-sq-loop)))
|
|
(:else
|
|
(do
|
|
(append! parts (rb-cur))
|
|
(rb-advance!)
|
|
(rb-sq-loop))))))
|
|
(rb-sq-loop)
|
|
(join "" parts))))
|
|
(define rb-read-dq-string
|
|
(fn ()
|
|
(let ((parts (list)))
|
|
(rb-advance!)
|
|
(define rb-dq-loop
|
|
(fn ()
|
|
(cond
|
|
((>= pos src-len) nil)
|
|
((= (rb-cur) "\"") (rb-advance!))
|
|
((= (rb-cur) "\\")
|
|
(do
|
|
(append! parts (rb-read-escape))
|
|
(rb-dq-loop)))
|
|
((and (= (rb-cur) "#") (= (rb-peek 1) "{"))
|
|
(do
|
|
(append! parts "#{")
|
|
(rb-advance-n! 2)
|
|
(let ((depth 1))
|
|
(define rb-interp-inner
|
|
(fn ()
|
|
(when (and (< pos src-len) (> depth 0))
|
|
(do
|
|
(let ((c (rb-cur)))
|
|
(cond
|
|
((= c "{")
|
|
(do
|
|
(set! depth (+ depth 1))
|
|
(append! parts c)
|
|
(rb-advance!)))
|
|
((= c "}")
|
|
(do
|
|
(set! depth (- depth 1))
|
|
(when (> depth 0)
|
|
(do (append! parts c) (rb-advance!)))))
|
|
(:else
|
|
(do (append! parts c) (rb-advance!)))))
|
|
(rb-interp-inner)))))
|
|
(rb-interp-inner))
|
|
(when (= (rb-cur) "}")
|
|
(do (append! parts "}") (rb-advance!)))
|
|
(rb-dq-loop)))
|
|
(:else
|
|
(do
|
|
(append! parts (rb-cur))
|
|
(rb-advance!)
|
|
(rb-dq-loop))))))
|
|
(rb-dq-loop)
|
|
(join "" parts))))
|
|
(define rb-read-percent-words
|
|
(fn ()
|
|
(rb-advance-n! 2)
|
|
(let ((open-ch (rb-cur)))
|
|
(let ((close-ch
|
|
(cond
|
|
((= open-ch "[") "]")
|
|
((= open-ch "(") ")")
|
|
((= open-ch "{") "}")
|
|
((= open-ch "<") ">")
|
|
(:else open-ch))))
|
|
(rb-advance!)
|
|
(let ((items (list)))
|
|
(define rb-pw-skip
|
|
(fn ()
|
|
(when (and (< pos src-len) (or (rb-space? (rb-cur)) (= (rb-cur) "\n")))
|
|
(do (rb-advance!) (rb-pw-skip)))))
|
|
(define rb-pw-word
|
|
(fn (wparts)
|
|
(if (or (>= pos src-len)
|
|
(rb-space? (rb-cur))
|
|
(= (rb-cur) "\n")
|
|
(= (rb-cur) close-ch))
|
|
(append! items (join "" wparts))
|
|
(do
|
|
(append! wparts (rb-cur))
|
|
(rb-advance!)
|
|
(rb-pw-word wparts)))))
|
|
(define rb-pw-loop
|
|
(fn ()
|
|
(rb-pw-skip)
|
|
(when (and (< pos src-len) (not (= (rb-cur) close-ch)))
|
|
(do
|
|
(rb-pw-word (list))
|
|
(rb-pw-loop)))))
|
|
(rb-pw-loop)
|
|
(when (= (rb-cur) close-ch) (rb-advance!))
|
|
items)))))
|
|
(define rb-read-ident-word
|
|
(fn ()
|
|
(let ((start pos))
|
|
(rb-read-while rb-ident-cont?)
|
|
(when (and (= (rb-cur) "?") (not (= (rb-peek 1) "=")))
|
|
(rb-advance!))
|
|
(when (and (= (rb-cur) "!") (not (or (= (rb-peek 1) "=") (= (rb-peek 1) "~"))))
|
|
(rb-advance!))
|
|
(substring src start pos))))
|
|
(define rb-read-number!
|
|
(fn (tok-line tok-col)
|
|
(let ((start pos))
|
|
(cond
|
|
((and (= (rb-cur) "0") (let ((p (rb-peek 1))) (or (= p "b") (= p "B"))))
|
|
(do
|
|
(rb-advance-n! 2)
|
|
(let ((bin-str (rb-read-while rb-binary-digit?)))
|
|
(rb-push! "int" (rb-parse-radix bin-str 2) tok-line tok-col))))
|
|
((and (= (rb-cur) "0") (let ((p (rb-peek 1))) (or (= p "o") (= p "O"))))
|
|
(do
|
|
(rb-advance-n! 2)
|
|
(let ((oct-str (rb-read-while rb-octal-digit?)))
|
|
(rb-push! "int" (rb-parse-radix oct-str 8) tok-line tok-col))))
|
|
((and (= (rb-cur) "0") (let ((p (rb-peek 1))) (or (= p "x") (= p "X"))))
|
|
(do
|
|
(rb-advance-n! 2)
|
|
(let ((hex-str (rb-read-while rb-hex-digit?)))
|
|
(rb-push! "int" (rb-parse-radix hex-str 16) tok-line tok-col))))
|
|
(:else
|
|
(do
|
|
(rb-read-while (fn (c) (or (rb-digit? c) (= c "_"))))
|
|
(let ((is-float false))
|
|
(when (and (= (rb-cur) ".") (rb-digit? (rb-peek 1)))
|
|
(do
|
|
(set! is-float true)
|
|
(rb-advance!)
|
|
(rb-read-while (fn (c) (or (rb-digit? c) (= c "_"))))))
|
|
(when (or (= (rb-cur) "e") (= (rb-cur) "E"))
|
|
(do
|
|
(set! is-float true)
|
|
(rb-advance!)
|
|
(when (or (= (rb-cur) "+") (= (rb-cur) "-"))
|
|
(rb-advance!))
|
|
(rb-read-while rb-digit?)))
|
|
(let ((num-str (rb-strip-underscores (substring src start pos))))
|
|
(if is-float
|
|
(rb-push! "float" num-str tok-line tok-col)
|
|
(rb-push! "int" (parse-int num-str) tok-line tok-col))))))))))
|
|
(define rb-read-op!
|
|
(fn (tok-line tok-col)
|
|
(let ((c0 (rb-cur)) (c1 (rb-peek 1)) (c2 (rb-peek 2)))
|
|
(cond
|
|
((and (= c0 "<") (= c1 "=") (= c2 ">"))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "<=>" tok-line tok-col)))
|
|
((and (= c0 "=") (= c1 "=") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "===" tok-line tok-col)))
|
|
((and (= c0 "*") (= c1 "*") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "**=" tok-line tok-col)))
|
|
((and (= c0 "<") (= c1 "<") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "<<=" tok-line tok-col)))
|
|
((and (= c0 ">") (= c1 ">") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" ">>=" tok-line tok-col)))
|
|
((and (= c0 "&") (= c1 "&") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "&&=" tok-line tok-col)))
|
|
((and (= c0 "|") (= c1 "|") (= c2 "="))
|
|
(do (rb-advance-n! 3) (rb-push! "op" "||=" tok-line tok-col)))
|
|
((and (= c0 "*") (= c1 "*"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "**" tok-line tok-col)))
|
|
((and (= c0 "=") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "==" tok-line tok-col)))
|
|
((and (= c0 "!") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "!=" tok-line tok-col)))
|
|
((and (= c0 "<") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "<=" tok-line tok-col)))
|
|
((and (= c0 ">") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" ">=" tok-line tok-col)))
|
|
((and (= c0 "=") (= c1 "~"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "=~" tok-line tok-col)))
|
|
((and (= c0 "!") (= c1 "~"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "!~" tok-line tok-col)))
|
|
((and (= c0 "<") (= c1 "<"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "<<" tok-line tok-col)))
|
|
((and (= c0 ">") (= c1 ">"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" ">>" tok-line tok-col)))
|
|
((and (= c0 "&") (= c1 "&"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "&&" tok-line tok-col)))
|
|
((and (= c0 "|") (= c1 "|"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "||" tok-line tok-col)))
|
|
((and (= c0 "+") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "+=" tok-line tok-col)))
|
|
((and (= c0 "-") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "-=" tok-line tok-col)))
|
|
((and (= c0 "*") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "*=" tok-line tok-col)))
|
|
((and (= c0 "/") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "/=" tok-line tok-col)))
|
|
((and (= c0 "%") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "%=" tok-line tok-col)))
|
|
((and (= c0 "&") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "&=" tok-line tok-col)))
|
|
((and (= c0 "|") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "|=" tok-line tok-col)))
|
|
((and (= c0 "^") (= c1 "="))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "^=" tok-line tok-col)))
|
|
((and (= c0 "-") (= c1 ">"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "->" tok-line tok-col)))
|
|
((and (= c0 "=") (= c1 ">"))
|
|
(do (rb-advance-n! 2) (rb-push! "op" "=>" tok-line tok-col)))
|
|
((and (= c0 "|") (nil? c1))
|
|
(do (rb-advance!) (rb-push! "pipe" "|" tok-line tok-col)))
|
|
((= c0 "|")
|
|
(do (rb-advance!) (rb-push! "pipe" "|" tok-line tok-col)))
|
|
(:else
|
|
(do (rb-advance!) (rb-push! "op" c0 tok-line tok-col)))))))
|
|
(define rb-scan!
|
|
(fn ()
|
|
(cond
|
|
((>= pos src-len) nil)
|
|
((rb-space? (rb-cur)) (do (rb-advance!) (rb-scan!)))
|
|
((= (rb-cur) "#") (do (rb-skip-line-comment!) (rb-scan!)))
|
|
((= (rb-cur) "\n")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-advance!)
|
|
(rb-push! "newline" nil l c))
|
|
(rb-scan!)))
|
|
((rb-digit? (rb-cur))
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-read-number! l c))
|
|
(rb-scan!)))
|
|
((rb-ident-start? (rb-cur))
|
|
(do
|
|
(let ((l line) (c col))
|
|
(let ((w (rb-read-ident-word)))
|
|
(if (rb-keyword? w)
|
|
(rb-push! "keyword" w l c)
|
|
(if (rb-upper? (substring w 0 1))
|
|
(rb-push! "const" w l c)
|
|
(rb-push! "ident" w l c)))))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "@")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(if (= (rb-peek 1) "@")
|
|
(do
|
|
(rb-advance-n! 2)
|
|
(let ((name (rb-read-while rb-ident-cont?)))
|
|
(rb-push! "cvar" (str "@@" name) l c)))
|
|
(do
|
|
(rb-advance!)
|
|
(let ((name (rb-read-while rb-ident-cont?)))
|
|
(rb-push! "ivar" (str "@" name) l c)))))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "$")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-advance!)
|
|
(let ((name (rb-read-while rb-ident-cont?)))
|
|
(rb-push! "gvar" (str "$" name) l c)))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "\"")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-push! "string" (rb-read-dq-string) l c))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "'")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-push! "string" (rb-read-sq-string) l c))
|
|
(rb-scan!)))
|
|
((and (= (rb-cur) ":") (= (rb-peek 1) ":"))
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-advance-n! 2)
|
|
(rb-push! "dcolon" "::" l c))
|
|
(rb-scan!)))
|
|
((= (rb-cur) ":")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(rb-advance!)
|
|
(cond
|
|
((= (rb-cur) "\"")
|
|
(rb-push! "symbol" (rb-read-dq-string) l c))
|
|
((= (rb-cur) "'")
|
|
(rb-push! "symbol" (rb-read-sq-string) l c))
|
|
((rb-ident-start? (rb-cur))
|
|
(let ((name (rb-read-ident-word)))
|
|
(rb-push! "symbol" name l c)))
|
|
(:else
|
|
(rb-push! "colon" ":" l c))))
|
|
(rb-scan!)))
|
|
((and (= (rb-cur) "%")
|
|
(let ((p (rb-peek 1)))
|
|
(or (= p "w") (= p "W") (= p "i") (= p "I"))))
|
|
(do
|
|
(let ((l line) (c col))
|
|
(let ((kind (rb-peek 1)))
|
|
(let ((items (rb-read-percent-words)))
|
|
(if (or (= kind "i") (= kind "I"))
|
|
(rb-push! "isymbols" items l c)
|
|
(rb-push! "words" items l c)))))
|
|
(rb-scan!)))
|
|
((= (rb-cur) ".")
|
|
(do
|
|
(let ((l line) (c col))
|
|
(cond
|
|
((and (= (rb-peek 1) ".") (= (rb-peek 2) "."))
|
|
(do (rb-advance-n! 3) (rb-push! "dotdotdot" "..." l c)))
|
|
((= (rb-peek 1) ".")
|
|
(do (rb-advance-n! 2) (rb-push! "dotdot" ".." l c)))
|
|
(:else
|
|
(do (rb-advance!) (rb-push! "dot" "." l c)))))
|
|
(rb-scan!)))
|
|
((= (rb-cur) ",")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "comma" "," l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) ";")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "semi" ";" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "(")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "lparen" "(" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) ")")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "rparen" ")" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "[")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "lbracket" "[" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "]")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "rbracket" "]" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "{")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "lbrace" "{" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((= (rb-cur) "}")
|
|
(do
|
|
(let ((l line) (c col)) (rb-push! "rbrace" "}" l c) (rb-advance!))
|
|
(rb-scan!)))
|
|
((or (= (rb-cur) "+") (= (rb-cur) "-") (= (rb-cur) "*")
|
|
(= (rb-cur) "/") (= (rb-cur) "%") (= (rb-cur) "=")
|
|
(= (rb-cur) "!") (= (rb-cur) "<") (= (rb-cur) ">")
|
|
(= (rb-cur) "&") (= (rb-cur) "^") (= (rb-cur) "~")
|
|
(= (rb-cur) "|"))
|
|
(do
|
|
(let ((l line) (c col)) (rb-read-op! l c))
|
|
(rb-scan!)))
|
|
(:else (do (rb-advance!) (rb-scan!))))))
|
|
(rb-scan!)
|
|
(rb-push! "eof" nil line col)
|
|
tokens)))
|