go: lex.sx — keywords, ident/int/string/rune lits, comments, ops, ASI + 78 tests [consumes-lex]

First Go-on-SX iteration. Tokenizer consumes lib/guest/lex.sx character-class
predicates. Automatic semicolon insertion per Go spec § Semicolons fires on
newline, EOF, and block comments containing a newline, after
ident/int/string/rune/{break,continue,fallthrough,return}/{++,--,),],}}.

Scoreboard + conformance.sh wired; lex 78/78. Plan Phase 1 sub-items
checked; floats/raw-strings/hex-ints still ⬜.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

lib/go/tests/lex.sx

@@ -0,0 +1,204 @@
+;; Go tokenizer tests.
+
+(define go-test-count 0)
+(define go-test-pass 0)
+(define go-test-fails (list))
+
+(define gtok-type (fn (t) (get t :type)))
+(define gtok-value (fn (t) (get t :value)))
+(define tok-types (fn (src) (map gtok-type (go-tokenize src))))
+(define tok-values (fn (src) (map gtok-value (go-tokenize src))))
+
+(define
+  go-test
+  (fn
+    (name actual expected)
+    (set! go-test-count (+ go-test-count 1))
+    (if
+      (= actual expected)
+      (set! go-test-pass (+ go-test-pass 1))
+      (append! go-test-fails {:name name :expected expected :actual actual}))))
+
+;; ── empty / whitespace ────────────────────────────────────────────
+(go-test "empty source" (tok-types "") (list "eof"))
+(go-test "spaces only" (tok-types "   ") (list "eof"))
+(go-test "tabs only" (tok-types "\t\t") (list "eof"))
+(go-test
+  "newline only — no prior token, no ASI"
+  (tok-types "\n")
+  (list "eof"))
+
+;; ── identifiers ───────────────────────────────────────────────────
+(go-test "ident: simple" (tok-values "foo") (list "foo" "\n" nil))
+(go-test
+  "ident: underscore prefix"
+  (tok-values "_bar")
+  (list "_bar" "\n" nil))
+(go-test "ident: mixed case" (tok-values "fooBar") (list "fooBar" "\n" nil))
+(go-test "ident: with digits" (tok-values "x123") (list "x123" "\n" nil))
+(go-test "ident: type tag" (tok-types "foo") (list "ident" "semi" "eof"))
+
+;; ── keywords (all 25) ─────────────────────────────────────────────
+(go-test "kw: break" (tok-types "break") (list "keyword" "semi" "eof"))
+(go-test "kw: case" (tok-types "case") (list "keyword" "eof"))
+(go-test "kw: chan" (tok-types "chan") (list "keyword" "eof"))
+(go-test "kw: const" (tok-types "const") (list "keyword" "eof"))
+(go-test "kw: continue" (tok-types "continue") (list "keyword" "semi" "eof"))
+(go-test "kw: default" (tok-types "default") (list "keyword" "eof"))
+(go-test "kw: defer" (tok-types "defer") (list "keyword" "eof"))
+(go-test "kw: else" (tok-types "else") (list "keyword" "eof"))
+(go-test
+  "kw: fallthrough"
+  (tok-types "fallthrough")
+  (list "keyword" "semi" "eof"))
+(go-test "kw: for" (tok-types "for") (list "keyword" "eof"))
+(go-test "kw: func" (tok-types "func") (list "keyword" "eof"))
+(go-test "kw: go" (tok-types "go") (list "keyword" "eof"))
+(go-test "kw: goto" (tok-types "goto") (list "keyword" "eof"))
+(go-test "kw: if" (tok-types "if") (list "keyword" "eof"))
+(go-test "kw: import" (tok-types "import") (list "keyword" "eof"))
+(go-test "kw: interface" (tok-types "interface") (list "keyword" "eof"))
+(go-test "kw: map" (tok-types "map") (list "keyword" "eof"))
+(go-test "kw: package" (tok-types "package") (list "keyword" "eof"))
+(go-test "kw: range" (tok-types "range") (list "keyword" "eof"))
+(go-test "kw: return" (tok-types "return") (list "keyword" "semi" "eof"))
+(go-test "kw: select" (tok-types "select") (list "keyword" "eof"))
+(go-test "kw: struct" (tok-types "struct") (list "keyword" "eof"))
+(go-test "kw: switch" (tok-types "switch") (list "keyword" "eof"))
+(go-test "kw: type" (tok-types "type") (list "keyword" "eof"))
+(go-test "kw: var" (tok-types "var") (list "keyword" "eof"))
+
+;; ── integer literals ──────────────────────────────────────────────
+(go-test "int: zero" (tok-values "0") (list "0" "\n" nil))
+(go-test "int: small" (tok-values "42") (list "42" "\n" nil))
+(go-test "int: bigger" (tok-values "123456") (list "123456" "\n" nil))
+(go-test "int: type" (tok-types "42") (list "int" "semi" "eof"))
+
+;; ── string literals ───────────────────────────────────────────────
+(go-test "string: empty" (tok-values "\"\"") (list "" "\n" nil))
+(go-test "string: hello" (tok-values "\"hello\"") (list "hello" "\n" nil))
+(go-test
+  "string: with space"
+  (tok-values "\"hi there\"")
+  (list "hi there" "\n" nil))
+(go-test "string: escape n" (tok-values "\"a\\nb\"") (list "a\nb" "\n" nil))
+(go-test "string: escape quote" (tok-values "\"a\\\"b\"") (list "a\"b" "\n" nil))
+(go-test
+  "string: escape backslash"
+  (tok-values "\"a\\\\b\"")
+  (list "a\\b" "\n" nil))
+(go-test "string: type" (tok-types "\"x\"") (list "string" "semi" "eof"))
+
+;; ── rune literals ─────────────────────────────────────────────────
+(go-test "rune: simple" (tok-values "'a'") (list "a" "\n" nil))
+(go-test "rune: escape" (tok-values "'\\n'") (list "\n" "\n" nil))
+(go-test "rune: type" (tok-types "'a'") (list "rune" "semi" "eof"))
+
+;; ── comments ──────────────────────────────────────────────────────
+(go-test "line comment" (tok-types "// ignored") (list "eof"))
+(go-test "line comment then code" (tok-values "// hi\nx") (list "x" "\n" nil))
+(go-test "block comment" (tok-types "/* a b c */") (list "eof"))
+(go-test
+  "block comment inline"
+  (tok-values "x /* mid */ y")
+  (list "x" "y" "\n" nil))
+(go-test
+  "block comment with newline — ASI"
+  (tok-types "x /* multi\nline */ y")
+  (list "ident" "semi" "ident" "semi" "eof"))
+
+;; ── operators & punctuation ───────────────────────────────────────
+(go-test
+  "ops: arithmetic"
+  (tok-values "+ - * / %")
+  (list "+" "-" "*" "/" "%" nil))
+(go-test
+  "ops: comparison"
+  (tok-values "== != < > <= >=")
+  (list "==" "!=" "<" ">" "<=" ">=" nil))
+(go-test "ops: logical" (tok-values "&& || !") (list "&&" "||" "!" nil))
+(go-test
+  "ops: assign forms"
+  (tok-values "= := += -=")
+  (list "=" ":=" "+=" "-=" nil))
+(go-test "ops: channel arrow" (tok-values "<- chan") (list "<-" "chan" nil))
+(go-test "ops: incdec ASI" (tok-types "++ --") (list "op" "op" "semi" "eof"))
+(go-test "ops: ellipsis" (tok-values "...") (list "..." nil))
+(go-test
+  "punct: all brackets"
+  (tok-values "( ) { } [ ]")
+  (list "(" ")" "{" "}" "[" "]" "\n" nil))
+(go-test
+  "punct: comma colon dot"
+  (tok-values ", : .")
+  (list "," ":" "." nil))
+
+;; ── automatic semicolon insertion (Go spec § Semicolons) ──────────
+(go-test
+  "ASI: after ident at newline"
+  (tok-types "x\ny")
+  (list "ident" "semi" "ident" "semi" "eof"))
+(go-test "ASI: after int" (tok-types "42\n") (list "int" "semi" "eof"))
+(go-test
+  "ASI: after string"
+  (tok-types "\"hi\"\n")
+  (list "string" "semi" "eof"))
+(go-test "ASI: after rune" (tok-types "'a'\n") (list "rune" "semi" "eof"))
+(go-test
+  "ASI: after )"
+  (tok-types "f()\n")
+  (list "ident" "op" "op" "semi" "eof"))
+(go-test
+  "ASI: after ]"
+  (tok-types "x[0]\n")
+  (list "ident" "op" "int" "op" "semi" "eof"))
+(go-test "ASI: after }" (tok-types "{}\n") (list "op" "op" "semi" "eof"))
+(go-test "ASI: after ++" (tok-types "i++\n") (list "ident" "op" "semi" "eof"))
+(go-test
+  "ASI: NOT after +"
+  (tok-types "x +\ny")
+  (list "ident" "op" "ident" "semi" "eof"))
+(go-test
+  "ASI: NOT after ("
+  (tok-types "f(\nx)")
+  (list "ident" "op" "ident" "op" "semi" "eof"))
+(go-test
+  "ASI: blank lines collapse — single semi only"
+  (tok-types "x\n\n\ny")
+  (list "ident" "semi" "ident" "semi" "eof"))
+(go-test
+  "ASI: at EOF after ident"
+  (tok-types "x")
+  (list "ident" "semi" "eof"))
+(go-test
+  "ASI: explicit semi"
+  (tok-types "x;y")
+  (list "ident" "semi" "ident" "semi" "eof"))
+
+;; ── short program ─────────────────────────────────────────────────
+(go-test
+  "short-decl: x := 42 (types)"
+  (tok-types "x := 42")
+  (list "ident" "op" "int" "semi" "eof"))
+(go-test
+  "short-decl: x := 42 (values)"
+  (tok-values "x := 42")
+  (list "x" ":=" "42" "\n" nil))
+(go-test
+  "func decl shape"
+  (tok-types "func foo() int { return 0 }")
+  (list
+    "keyword"
+    "ident"
+    "op"
+    "op"
+    "ident"
+    "op"
+    "keyword"
+    "int"
+    "op"
+    "semi"
+    "eof"))
+
+;; ── report ────────────────────────────────────────────────────────
+(define go-lex-test-summary (str "lex " go-test-pass "/" go-test-count))