giles
9bd6bbb7e7
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 54s
Tokens {:type :value :pos} for atoms, vars, numbers, strings, punct, ops.
Operators :- ?- <= >= != < > = + - * /. Comments % and /* */.
140 lines
4.2 KiB
Plaintext
140 lines
4.2 KiB
Plaintext
;; lib/datalog/tests/tokenize.sx — tokenizer unit tests
|
|
;;
|
|
;; Run via: bash lib/datalog/conformance.sh
|
|
;; Or: (load "lib/datalog/tokenizer.sx") (load "lib/datalog/tests/tokenize.sx")
|
|
;; (dl-tokenize-tests-run!)
|
|
|
|
(define dl-tk-pass 0)
|
|
(define dl-tk-fail 0)
|
|
(define dl-tk-failures (list))
|
|
|
|
(define
|
|
dl-tk-test!
|
|
(fn
|
|
(name got expected)
|
|
(if
|
|
(= got expected)
|
|
(set! dl-tk-pass (+ dl-tk-pass 1))
|
|
(do
|
|
(set! dl-tk-fail (+ dl-tk-fail 1))
|
|
(append!
|
|
dl-tk-failures
|
|
(str name "\n expected: " expected "\n got: " got))))))
|
|
|
|
(define dl-tk-types (fn (toks) (map (fn (t) (get t :type)) toks)))
|
|
(define dl-tk-values (fn (toks) (map (fn (t) (get t :value)) toks)))
|
|
|
|
(define
|
|
dl-tk-run-all!
|
|
(fn
|
|
()
|
|
(do
|
|
(dl-tk-test! "empty" (dl-tk-types (dl-tokenize "")) (list "eof"))
|
|
(dl-tk-test!
|
|
"atom dot"
|
|
(dl-tk-types (dl-tokenize "foo."))
|
|
(list "atom" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"atom dot value"
|
|
(dl-tk-values (dl-tokenize "foo."))
|
|
(list "foo" "." nil))
|
|
(dl-tk-test!
|
|
"var"
|
|
(dl-tk-types (dl-tokenize "X."))
|
|
(list "var" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"underscore var"
|
|
(dl-tk-types (dl-tokenize "_x."))
|
|
(list "var" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"integer"
|
|
(dl-tk-values (dl-tokenize "42"))
|
|
(list 42 nil))
|
|
(dl-tk-test!
|
|
"decimal"
|
|
(dl-tk-values (dl-tokenize "3.14"))
|
|
(list 3.14 nil))
|
|
(dl-tk-test!
|
|
"string"
|
|
(dl-tk-values (dl-tokenize "\"hello\""))
|
|
(list "hello" nil))
|
|
(dl-tk-test!
|
|
"quoted atom"
|
|
(dl-tk-types (dl-tokenize "'two words'"))
|
|
(list "atom" "eof"))
|
|
(dl-tk-test!
|
|
"quoted atom value"
|
|
(dl-tk-values (dl-tokenize "'two words'"))
|
|
(list "two words" nil))
|
|
(dl-tk-test! ":-" (dl-tk-values (dl-tokenize ":-")) (list ":-" nil))
|
|
(dl-tk-test! "?-" (dl-tk-values (dl-tokenize "?-")) (list "?-" nil))
|
|
(dl-tk-test! "<=" (dl-tk-values (dl-tokenize "<=")) (list "<=" nil))
|
|
(dl-tk-test! ">=" (dl-tk-values (dl-tokenize ">=")) (list ">=" nil))
|
|
(dl-tk-test! "!=" (dl-tk-values (dl-tokenize "!=")) (list "!=" nil))
|
|
(dl-tk-test!
|
|
"single op values"
|
|
(dl-tk-values (dl-tokenize "< > = + - * /"))
|
|
(list "<" ">" "=" "+" "-" "*" "/" nil))
|
|
(dl-tk-test!
|
|
"single op types"
|
|
(dl-tk-types (dl-tokenize "< > = + - * /"))
|
|
(list "op" "op" "op" "op" "op" "op" "op" "eof"))
|
|
(dl-tk-test!
|
|
"punct"
|
|
(dl-tk-values (dl-tokenize "( ) , ."))
|
|
(list "(" ")" "," "." nil))
|
|
(dl-tk-test!
|
|
"fact tokens"
|
|
(dl-tk-types (dl-tokenize "parent(tom, bob)."))
|
|
(list "atom" "punct" "atom" "punct" "atom" "punct" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"rule shape"
|
|
(dl-tk-types (dl-tokenize "p(X) :- q(X)."))
|
|
(list
|
|
"atom"
|
|
"punct"
|
|
"var"
|
|
"punct"
|
|
"op"
|
|
"atom"
|
|
"punct"
|
|
"var"
|
|
"punct"
|
|
"punct"
|
|
"eof"))
|
|
(dl-tk-test!
|
|
"comparison literal"
|
|
(dl-tk-values (dl-tokenize "<(X, 5)"))
|
|
(list "<" "(" "X" "," 5 ")" nil))
|
|
(dl-tk-test!
|
|
"is form"
|
|
(dl-tk-values (dl-tokenize "is(Y, +(X, 1))"))
|
|
(list "is" "(" "Y" "," "+" "(" "X" "," 1 ")" ")" nil))
|
|
(dl-tk-test!
|
|
"line comment"
|
|
(dl-tk-types (dl-tokenize "% comment line\nfoo."))
|
|
(list "atom" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"block comment"
|
|
(dl-tk-types (dl-tokenize "/* a\nb */ x."))
|
|
(list "atom" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"whitespace"
|
|
(dl-tk-types (dl-tokenize " foo ,\t bar ."))
|
|
(list "atom" "punct" "atom" "punct" "eof"))
|
|
(dl-tk-test!
|
|
"positions"
|
|
(map (fn (t) (get t :pos)) (dl-tokenize "foo bar"))
|
|
(list 0 4 7)))))
|
|
|
|
(define
|
|
dl-tokenize-tests-run!
|
|
(fn
|
|
()
|
|
(do
|
|
(set! dl-tk-pass 0)
|
|
(set! dl-tk-fail 0)
|
|
(set! dl-tk-failures (list))
|
|
(dl-tk-run-all!)
|
|
{:failures dl-tk-failures :total (+ dl-tk-pass dl-tk-fail) :passed dl-tk-pass :failed dl-tk-fail})))
|