go: lex.sx — keywords, ident/int/string/rune lits, comments, ops, ASI + 78 tests [consumes-lex]

First Go-on-SX iteration. Tokenizer consumes lib/guest/lex.sx character-class
predicates. Automatic semicolon insertion per Go spec § Semicolons fires on
newline, EOF, and block comments containing a newline, after
ident/int/string/rune/{break,continue,fallthrough,return}/{++,--,),],}}.

Scoreboard + conformance.sh wired; lex 78/78. Plan Phase 1 sub-items
checked; floats/raw-strings/hex-ints still ⬜.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

lib/go/conformance.sh

@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Go-on-SX conformance runner.
+#
+# Loads every Go-on-SX test suite via the epoch protocol, collects
+# pass/fail counts, and writes lib/go/scoreboard.json + .md.
+#
+# Usage:
+#   bash lib/go/conformance.sh           # run all suites
+#   bash lib/go/conformance.sh -v        # verbose per-suite
+
+set -uo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
+if [ ! -x "$SX_SERVER" ]; then
+  SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
+fi
+if [ ! -x "$SX_SERVER" ]; then
+  echo "ERROR: sx_server.exe not found." >&2
+  exit 1
+fi
+
+VERBOSE="${1:-}"
+TMPFILE=$(mktemp)
+OUTFILE=$(mktemp)
+trap "rm -f $TMPFILE $OUTFILE" EXIT
+
+# Each suite: name | pass-counter | total-counter
+SUITES=(
+  "lex|go-test-pass|go-test-count"
+)
+
+cat > "$TMPFILE" <<'EPOCHS'
+(epoch 1)
+(load "lib/guest/lex.sx")
+(load "lib/go/lex.sx")
+(load "lib/go/tests/lex.sx")
+EPOCHS
+
+idx=0
+for entry in "${SUITES[@]}"; do
+  name="${entry%%|*}"
+  pass_var=$(echo "$entry" | awk -F'|' '{print $2}')
+  total_var=$(echo "$entry" | awk -F'|' '{print $3}')
+  epoch=$((100 + idx))
+  echo "(epoch $epoch)" >> "$TMPFILE"
+  echo "(eval \"(list $pass_var $total_var)\")" >> "$TMPFILE"
+  idx=$((idx + 1))
+done
+
+"$SX_SERVER" < "$TMPFILE" > "$OUTFILE" 2>&1
+
+parse_pair() {
+  local epoch="$1"
+  local line
+  line=$(grep -A1 "^(ok-len $epoch " "$OUTFILE" | tail -1)
+  echo "$line" | sed -E 's/[()]//g'
+}
+
+TOTAL_PASS=0
+TOTAL_COUNT=0
+JSON_SUITES=""
+MD_ROWS=""
+
+idx=0
+for entry in "${SUITES[@]}"; do
+  name="${entry%%|*}"
+  epoch=$((100 + idx))
+  pair=$(parse_pair "$epoch")
+  pass=$(echo "$pair" | awk '{print $1}')
+  count=$(echo "$pair" | awk '{print $2}')
+  if [ -z "$pass" ] || [ -z "$count" ]; then
+    pass=0
+    count=0
+  fi
+  TOTAL_PASS=$((TOTAL_PASS + pass))
+  TOTAL_COUNT=$((TOTAL_COUNT + count))
+  status="ok"
+  marker="✅"
+  if [ "$pass" != "$count" ]; then
+    status="fail"
+    marker="❌"
+  fi
+  if [ "$VERBOSE" = "-v" ]; then
+    printf "  %-12s %s/%s\n" "$name" "$pass" "$count"
+  fi
+  if [ -n "$JSON_SUITES" ]; then JSON_SUITES+=","; fi
+  JSON_SUITES+=$'\n    '
+  JSON_SUITES+="{\"name\":\"$name\",\"pass\":$pass,\"total\":$count,\"status\":\"$status\"}"
+  MD_ROWS+="| $marker | $name | $pass | $count |"$'\n'
+  idx=$((idx + 1))
+done
+
+printf '\nGo-on-SX conformance: %d / %d\n' "$TOTAL_PASS" "$TOTAL_COUNT"
+
+cat > lib/go/scoreboard.json <<JSON
+{
+  "language": "go",
+  "total_pass": $TOTAL_PASS,
+  "total": $TOTAL_COUNT,
+  "suites": [$JSON_SUITES,
+    {"name":"parse","pass":0,"total":0,"status":"pending"},
+    {"name":"types","pass":0,"total":0,"status":"pending"},
+    {"name":"eval","pass":0,"total":0,"status":"pending"},
+    {"name":"runtime","pass":0,"total":0,"status":"pending"},
+    {"name":"stdlib","pass":0,"total":0,"status":"pending"},
+    {"name":"e2e","pass":0,"total":0,"status":"pending"}
+  ]
+}
+JSON
+
+cat > lib/go/scoreboard.md <<MD
+# Go-on-SX Scoreboard
+
+**Total: ${TOTAL_PASS} / ${TOTAL_COUNT} tests passing**
+
+|  | Suite | Pass | Total |
+|---|---|---|---|
+$MD_ROWS| ⬜ | parse | 0 | 0 |
+| ⬜ | types | 0 | 0 |
+| ⬜ | eval | 0 | 0 |
+| ⬜ | runtime | 0 | 0 |
+| ⬜ | stdlib | 0 | 0 |
+| ⬜ | e2e | 0 | 0 |
+
+Generated by \`lib/go/conformance.sh\`.
+MD
+
+if [ "$TOTAL_PASS" -eq "$TOTAL_COUNT" ]; then
+  exit 0
+else
+  exit 1
+fi

lib/go/lex.sx

@@ -0,0 +1,371 @@
+;; lib/go/lex.sx — Go tokenizer with automatic semicolon insertion.
+;;
+;; Consumes lib/guest/lex.sx character-class predicates.
+;;
+;; Tokens: {:type T :value V :pos P}
+;; Types:
+;;   "ident"   — identifiers (foo, _bar, mixedCase)
+;;   "keyword" — one of the 25 Go keywords
+;;   "int"     — integer literals (decimal only this iteration)
+;;   "string"  — interpreted string literals "..."
+;;   "rune"    — rune literals 'x' (single char + simple escapes)
+;;   "op"      — operators & punctuation; :value is the literal text
+;;   "semi"    — explicit ';' or auto-inserted (Go spec § Semicolons)
+;;   "eof"     — end-of-input sentinel
+;;
+;; ASI (Go spec § Semicolons): a newline (or EOF, or a block comment
+;; containing a newline) emits a ";semi" if the previous emitted token's
+;; type is ident/int/string/rune, or its value is one of
+;;   {break, continue, fallthrough, return, ++, --, ), ], }}.
+;;
+;; All scanner locals are gl- prefixed: SX host primitives (peek/emit/etc.)
+;; silently shadow guest-language defines. See feedback_sx_bind_clash.
+
+(define
+  go-keywords
+  (list
+    "break"
+    "case"
+    "chan"
+    "const"
+    "continue"
+    "default"
+    "defer"
+    "else"
+    "fallthrough"
+    "for"
+    "func"
+    "go"
+    "goto"
+    "if"
+    "import"
+    "interface"
+    "map"
+    "package"
+    "range"
+    "return"
+    "select"
+    "struct"
+    "switch"
+    "type"
+    "var"))
+
+(define go-keyword? (fn (s) (some (fn (k) (= k s)) go-keywords)))
+
+(define go-asi-keywords (list "break" "continue" "fallthrough" "return"))
+
+(define go-asi-ops (list "++" "--" ")" "]" "}"))
+
+(define
+  go-asi-trigger?
+  (fn
+    (tok)
+    (if
+      (= tok nil)
+      false
+      (let
+        ((ty (get tok :type)) (v (get tok :value)))
+        (or
+          (= ty "ident")
+          (= ty "int")
+          (= ty "string")
+          (= ty "rune")
+          (and (= ty "keyword") (some (fn (k) (= k v)) go-asi-keywords))
+          (and (= ty "op") (some (fn (o) (= o v)) go-asi-ops)))))))
+
+(define
+  go-tokenize
+  (fn
+    (src)
+    (let
+      ((tokens (list)) (pos 0) (src-len (len src)))
+      (define
+        gl-peek
+        (fn
+          (offset)
+          (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
+      (define gl-cur (fn () (gl-peek 0)))
+      (define gl-advance! (fn (n) (set! pos (+ pos n))))
+      (define
+        gl-last
+        (fn
+          ()
+          (if
+            (= (len tokens) 0)
+            nil
+            (nth tokens (- (len tokens) 1)))))
+      (define gl-emit! (fn (type value start) (append! tokens {:type type :value value :pos start})))
+      (define
+        gl-maybe-asi!
+        (fn
+          (at)
+          (when (go-asi-trigger? (gl-last)) (gl-emit! "semi" "\n" at))))
+      (define
+        gl-skip-line!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (not (= (gl-cur) "\n")))
+            (gl-advance! 1)
+            (gl-skip-line!))))
+      (define
+        gl-skip-block!
+        (fn
+          (saw-nl)
+          (cond
+            (>= pos src-len)
+            saw-nl
+            (and (= (gl-cur) "*") (= (gl-peek 1) "/"))
+            (do (gl-advance! 2) saw-nl)
+            :else (let
+              ((is-nl (= (gl-cur) "\n")))
+              (gl-advance! 1)
+              (gl-skip-block! (or saw-nl is-nl))))))
+      (define
+        gl-read-ident!
+        (fn
+          (start)
+          (when
+            (and (< pos src-len) (lex-ident-char? (gl-cur)))
+            (gl-advance! 1)
+            (gl-read-ident! start))
+          (slice src start pos)))
+      (define
+        gl-read-digits!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (lex-digit? (gl-cur)))
+            (gl-advance! 1)
+            (gl-read-digits!))))
+      (define
+        gl-read-string!
+        (fn
+          ()
+          (gl-advance! 1)
+          (let
+            ((chars (list)))
+            (define
+              gl-string-loop
+              (fn
+                ()
+                (cond
+                  (>= pos src-len)
+                  nil
+                  (= (gl-cur) "\"")
+                  (gl-advance! 1)
+                  (= (gl-cur) "\\")
+                  (do
+                    (gl-advance! 1)
+                    (when
+                      (< pos src-len)
+                      (let
+                        ((ch (gl-cur)))
+                        (cond
+                          (= ch "n")
+                          (append! chars "\n")
+                          (= ch "t")
+                          (append! chars "\t")
+                          (= ch "r")
+                          (append! chars "\r")
+                          (= ch "\\")
+                          (append! chars "\\")
+                          (= ch "\"")
+                          (append! chars "\"")
+                          (= ch "'")
+                          (append! chars "'")
+                          :else (append! chars ch))
+                        (gl-advance! 1)))
+                    (gl-string-loop))
+                  :else (do
+                    (append! chars (gl-cur))
+                    (gl-advance! 1)
+                    (gl-string-loop)))))
+            (gl-string-loop)
+            (join "" chars))))
+      (define
+        gl-read-rune!
+        (fn
+          ()
+          (gl-advance! 1)
+          (let
+            ((chars (list)))
+            (cond
+              (and (< pos src-len) (= (gl-cur) "\\"))
+              (do
+                (gl-advance! 1)
+                (when
+                  (< pos src-len)
+                  (let
+                    ((ch (gl-cur)))
+                    (cond
+                      (= ch "n")
+                      (append! chars "\n")
+                      (= ch "t")
+                      (append! chars "\t")
+                      (= ch "r")
+                      (append! chars "\r")
+                      (= ch "\\")
+                      (append! chars "\\")
+                      (= ch "'")
+                      (append! chars "'")
+                      (= ch "\"")
+                      (append! chars "\"")
+                      :else (append! chars ch))
+                    (gl-advance! 1))))
+              (< pos src-len)
+              (do (append! chars (gl-cur)) (gl-advance! 1)))
+            (when
+              (and (< pos src-len) (= (gl-cur) "'"))
+              (gl-advance! 1))
+            (join "" chars))))
+      (define
+        gl-match-op
+        (fn
+          ()
+          (let
+            ((c0 (gl-cur))
+              (c1 (gl-peek 1))
+              (c2 (gl-peek 2)))
+            (cond
+              (and (= c0 "<") (= c1 "<") (= c2 "="))
+              "<<="
+              (and (= c0 ">") (= c1 ">") (= c2 "="))
+              ">>="
+              (and (= c0 "&") (= c1 "^") (= c2 "="))
+              "&^="
+              (and (= c0 ".") (= c1 ".") (= c2 "."))
+              "..."
+              (and (= c0 "=") (= c1 "="))
+              "=="
+              (and (= c0 "!") (= c1 "="))
+              "!="
+              (and (= c0 "<") (= c1 "="))
+              "<="
+              (and (= c0 ">") (= c1 "="))
+              ">="
+              (and (= c0 "&") (= c1 "&"))
+              "&&"
+              (and (= c0 "|") (= c1 "|"))
+              "||"
+              (and (= c0 "+") (= c1 "+"))
+              "++"
+              (and (= c0 "-") (= c1 "-"))
+              "--"
+              (and (= c0 "<") (= c1 "<"))
+              "<<"
+              (and (= c0 ">") (= c1 ">"))
+              ">>"
+              (and (= c0 "+") (= c1 "="))
+              "+="
+              (and (= c0 "-") (= c1 "="))
+              "-="
+              (and (= c0 "*") (= c1 "="))
+              "*="
+              (and (= c0 "/") (= c1 "="))
+              "/="
+              (and (= c0 "%") (= c1 "="))
+              "%="
+              (and (= c0 "&") (= c1 "="))
+              "&="
+              (and (= c0 "|") (= c1 "="))
+              "|="
+              (and (= c0 "^") (= c1 "="))
+              "^="
+              (and (= c0 ":") (= c1 "="))
+              ":="
+              (and (= c0 "<") (= c1 "-"))
+              "<-"
+              (and (= c0 "&") (= c1 "^"))
+              "&^"
+              (or
+                (= c0 "+")
+                (= c0 "-")
+                (= c0 "*")
+                (= c0 "/")
+                (= c0 "%")
+                (= c0 "&")
+                (= c0 "|")
+                (= c0 "^")
+                (= c0 "<")
+                (= c0 ">")
+                (= c0 "=")
+                (= c0 "!")
+                (= c0 "(")
+                (= c0 ")")
+                (= c0 "{")
+                (= c0 "}")
+                (= c0 "[")
+                (= c0 "]")
+                (= c0 ",")
+                (= c0 ".")
+                (= c0 ":"))
+              c0
+              :else nil))))
+      (define
+        gl-scan!
+        (fn
+          ()
+          (cond
+            (>= pos src-len)
+            nil
+            (= (gl-cur) "\n")
+            (do (gl-maybe-asi! pos) (gl-advance! 1) (gl-scan!))
+            (lex-space? (gl-cur))
+            (do (gl-advance! 1) (gl-scan!))
+            (and (= (gl-cur) "/") (= (gl-peek 1) "/"))
+            (do (gl-advance! 2) (gl-skip-line!) (gl-scan!))
+            (and (= (gl-cur) "/") (= (gl-peek 1) "*"))
+            (do
+              (gl-advance! 2)
+              (let
+                ((saw-nl (gl-skip-block! false)))
+                (when saw-nl (gl-maybe-asi! pos)))
+              (gl-scan!))
+            (= (gl-cur) ";")
+            (do
+              (gl-emit! "semi" ";" pos)
+              (gl-advance! 1)
+              (gl-scan!))
+            (lex-ident-start? (gl-cur))
+            (do
+              (let
+                ((start pos))
+                (gl-read-ident! start)
+                (let
+                  ((word (slice src start pos)))
+                  (gl-emit!
+                    (if (go-keyword? word) "keyword" "ident")
+                    word
+                    start)))
+              (gl-scan!))
+            (lex-digit? (gl-cur))
+            (do
+              (let
+                ((start pos))
+                (gl-read-digits!)
+                (gl-emit! "int" (slice src start pos) start))
+              (gl-scan!))
+            (= (gl-cur) "\"")
+            (let
+              ((start pos) (v (gl-read-string!)))
+              (gl-emit! "string" v start)
+              (gl-scan!))
+            (= (gl-cur) "'")
+            (let
+              ((start pos) (v (gl-read-rune!)))
+              (gl-emit! "rune" v start)
+              (gl-scan!))
+            :else (let
+              ((op (gl-match-op)))
+              (cond
+                op
+                (do
+                  (gl-emit! "op" op pos)
+                  (gl-advance! (len op))
+                  (gl-scan!))
+                :else (do (gl-advance! 1) (gl-scan!)))))))
+      (gl-scan!)
+      (gl-maybe-asi! pos)
+      (gl-emit! "eof" nil pos)
+      tokens)))

lib/go/scoreboard.json

@@ -0,0 +1,14 @@
+{
+  "language": "go",
+  "total_pass": 78,
+  "total": 78,
+  "suites": [
+    {"name":"lex","pass":78,"total":78,"status":"ok"},
+    {"name":"parse","pass":0,"total":0,"status":"pending"},
+    {"name":"types","pass":0,"total":0,"status":"pending"},
+    {"name":"eval","pass":0,"total":0,"status":"pending"},
+    {"name":"runtime","pass":0,"total":0,"status":"pending"},
+    {"name":"stdlib","pass":0,"total":0,"status":"pending"},
+    {"name":"e2e","pass":0,"total":0,"status":"pending"}
+  ]
+}

lib/go/scoreboard.md

@@ -0,0 +1,15 @@
+# Go-on-SX Scoreboard
+
+**Total: 78 / 78 tests passing**
+
+|  | Suite | Pass | Total |
+|---|---|---|---|
+| ✅ | lex | 78 | 78 |
+| ⬜ | parse | 0 | 0 |
+| ⬜ | types | 0 | 0 |
+| ⬜ | eval | 0 | 0 |
+| ⬜ | runtime | 0 | 0 |
+| ⬜ | stdlib | 0 | 0 |
+| ⬜ | e2e | 0 | 0 |
+
+Generated by `lib/go/conformance.sh`.

lib/go/tests/lex.sx

@@ -0,0 +1,204 @@
+;; Go tokenizer tests.
+
+(define go-test-count 0)
+(define go-test-pass 0)
+(define go-test-fails (list))
+
+(define gtok-type (fn (t) (get t :type)))
+(define gtok-value (fn (t) (get t :value)))
+(define tok-types (fn (src) (map gtok-type (go-tokenize src))))
+(define tok-values (fn (src) (map gtok-value (go-tokenize src))))
+
+(define
+  go-test
+  (fn
+    (name actual expected)
+    (set! go-test-count (+ go-test-count 1))
+    (if
+      (= actual expected)
+      (set! go-test-pass (+ go-test-pass 1))
+      (append! go-test-fails {:name name :expected expected :actual actual}))))
+
+;; ── empty / whitespace ────────────────────────────────────────────
+(go-test "empty source" (tok-types "") (list "eof"))
+(go-test "spaces only" (tok-types "   ") (list "eof"))
+(go-test "tabs only" (tok-types "\t\t") (list "eof"))
+(go-test
+  "newline only — no prior token, no ASI"
+  (tok-types "\n")
+  (list "eof"))
+
+;; ── identifiers ───────────────────────────────────────────────────
+(go-test "ident: simple" (tok-values "foo") (list "foo" "\n" nil))
+(go-test
+  "ident: underscore prefix"
+  (tok-values "_bar")
+  (list "_bar" "\n" nil))
+(go-test "ident: mixed case" (tok-values "fooBar") (list "fooBar" "\n" nil))
+(go-test "ident: with digits" (tok-values "x123") (list "x123" "\n" nil))
+(go-test "ident: type tag" (tok-types "foo") (list "ident" "semi" "eof"))
+
+;; ── keywords (all 25) ─────────────────────────────────────────────
+(go-test "kw: break" (tok-types "break") (list "keyword" "semi" "eof"))
+(go-test "kw: case" (tok-types "case") (list "keyword" "eof"))
+(go-test "kw: chan" (tok-types "chan") (list "keyword" "eof"))
+(go-test "kw: const" (tok-types "const") (list "keyword" "eof"))
+(go-test "kw: continue" (tok-types "continue") (list "keyword" "semi" "eof"))
+(go-test "kw: default" (tok-types "default") (list "keyword" "eof"))
+(go-test "kw: defer" (tok-types "defer") (list "keyword" "eof"))
+(go-test "kw: else" (tok-types "else") (list "keyword" "eof"))
+(go-test
+  "kw: fallthrough"
+  (tok-types "fallthrough")
+  (list "keyword" "semi" "eof"))
+(go-test "kw: for" (tok-types "for") (list "keyword" "eof"))
+(go-test "kw: func" (tok-types "func") (list "keyword" "eof"))
+(go-test "kw: go" (tok-types "go") (list "keyword" "eof"))
+(go-test "kw: goto" (tok-types "goto") (list "keyword" "eof"))
+(go-test "kw: if" (tok-types "if") (list "keyword" "eof"))
+(go-test "kw: import" (tok-types "import") (list "keyword" "eof"))
+(go-test "kw: interface" (tok-types "interface") (list "keyword" "eof"))
+(go-test "kw: map" (tok-types "map") (list "keyword" "eof"))
+(go-test "kw: package" (tok-types "package") (list "keyword" "eof"))
+(go-test "kw: range" (tok-types "range") (list "keyword" "eof"))
+(go-test "kw: return" (tok-types "return") (list "keyword" "semi" "eof"))
+(go-test "kw: select" (tok-types "select") (list "keyword" "eof"))
+(go-test "kw: struct" (tok-types "struct") (list "keyword" "eof"))
+(go-test "kw: switch" (tok-types "switch") (list "keyword" "eof"))
+(go-test "kw: type" (tok-types "type") (list "keyword" "eof"))
+(go-test "kw: var" (tok-types "var") (list "keyword" "eof"))
+
+;; ── integer literals ──────────────────────────────────────────────
+(go-test "int: zero" (tok-values "0") (list "0" "\n" nil))
+(go-test "int: small" (tok-values "42") (list "42" "\n" nil))
+(go-test "int: bigger" (tok-values "123456") (list "123456" "\n" nil))
+(go-test "int: type" (tok-types "42") (list "int" "semi" "eof"))
+
+;; ── string literals ───────────────────────────────────────────────
+(go-test "string: empty" (tok-values "\"\"") (list "" "\n" nil))
+(go-test "string: hello" (tok-values "\"hello\"") (list "hello" "\n" nil))
+(go-test
+  "string: with space"
+  (tok-values "\"hi there\"")
+  (list "hi there" "\n" nil))
+(go-test "string: escape n" (tok-values "\"a\\nb\"") (list "a\nb" "\n" nil))
+(go-test "string: escape quote" (tok-values "\"a\\\"b\"") (list "a\"b" "\n" nil))
+(go-test
+  "string: escape backslash"
+  (tok-values "\"a\\\\b\"")
+  (list "a\\b" "\n" nil))
+(go-test "string: type" (tok-types "\"x\"") (list "string" "semi" "eof"))
+
+;; ── rune literals ─────────────────────────────────────────────────
+(go-test "rune: simple" (tok-values "'a'") (list "a" "\n" nil))
+(go-test "rune: escape" (tok-values "'\\n'") (list "\n" "\n" nil))
+(go-test "rune: type" (tok-types "'a'") (list "rune" "semi" "eof"))
+
+;; ── comments ──────────────────────────────────────────────────────
+(go-test "line comment" (tok-types "// ignored") (list "eof"))
+(go-test "line comment then code" (tok-values "// hi\nx") (list "x" "\n" nil))
+(go-test "block comment" (tok-types "/* a b c */") (list "eof"))
+(go-test
+  "block comment inline"
+  (tok-values "x /* mid */ y")
+  (list "x" "y" "\n" nil))
+(go-test
+  "block comment with newline — ASI"
+  (tok-types "x /* multi\nline */ y")
+  (list "ident" "semi" "ident" "semi" "eof"))
+
+;; ── operators & punctuation ───────────────────────────────────────
+(go-test
+  "ops: arithmetic"
+  (tok-values "+ - * / %")
+  (list "+" "-" "*" "/" "%" nil))
+(go-test
+  "ops: comparison"
+  (tok-values "== != < > <= >=")
+  (list "==" "!=" "<" ">" "<=" ">=" nil))
+(go-test "ops: logical" (tok-values "&& || !") (list "&&" "||" "!" nil))
+(go-test
+  "ops: assign forms"
+  (tok-values "= := += -=")
+  (list "=" ":=" "+=" "-=" nil))
+(go-test "ops: channel arrow" (tok-values "<- chan") (list "<-" "chan" nil))
+(go-test "ops: incdec ASI" (tok-types "++ --") (list "op" "op" "semi" "eof"))
+(go-test "ops: ellipsis" (tok-values "...") (list "..." nil))
+(go-test
+  "punct: all brackets"
+  (tok-values "( ) { } [ ]")
+  (list "(" ")" "{" "}" "[" "]" "\n" nil))
+(go-test
+  "punct: comma colon dot"
+  (tok-values ", : .")
+  (list "," ":" "." nil))
+
+;; ── automatic semicolon insertion (Go spec § Semicolons) ──────────
+(go-test
+  "ASI: after ident at newline"
+  (tok-types "x\ny")
+  (list "ident" "semi" "ident" "semi" "eof"))
+(go-test "ASI: after int" (tok-types "42\n") (list "int" "semi" "eof"))
+(go-test
+  "ASI: after string"
+  (tok-types "\"hi\"\n")
+  (list "string" "semi" "eof"))
+(go-test "ASI: after rune" (tok-types "'a'\n") (list "rune" "semi" "eof"))
+(go-test
+  "ASI: after )"
+  (tok-types "f()\n")
+  (list "ident" "op" "op" "semi" "eof"))
+(go-test
+  "ASI: after ]"
+  (tok-types "x[0]\n")
+  (list "ident" "op" "int" "op" "semi" "eof"))
+(go-test "ASI: after }" (tok-types "{}\n") (list "op" "op" "semi" "eof"))
+(go-test "ASI: after ++" (tok-types "i++\n") (list "ident" "op" "semi" "eof"))
+(go-test
+  "ASI: NOT after +"
+  (tok-types "x +\ny")
+  (list "ident" "op" "ident" "semi" "eof"))
+(go-test
+  "ASI: NOT after ("
+  (tok-types "f(\nx)")
+  (list "ident" "op" "ident" "op" "semi" "eof"))
+(go-test
+  "ASI: blank lines collapse — single semi only"
+  (tok-types "x\n\n\ny")
+  (list "ident" "semi" "ident" "semi" "eof"))
+(go-test
+  "ASI: at EOF after ident"
+  (tok-types "x")
+  (list "ident" "semi" "eof"))
+(go-test
+  "ASI: explicit semi"
+  (tok-types "x;y")
+  (list "ident" "semi" "ident" "semi" "eof"))
+
+;; ── short program ─────────────────────────────────────────────────
+(go-test
+  "short-decl: x := 42 (types)"
+  (tok-types "x := 42")
+  (list "ident" "op" "int" "semi" "eof"))
+(go-test
+  "short-decl: x := 42 (values)"
+  (tok-values "x := 42")
+  (list "x" ":=" "42" "\n" nil))
+(go-test
+  "func decl shape"
+  (tok-types "func foo() int { return 0 }")
+  (list
+    "keyword"
+    "ident"
+    "op"
+    "op"
+    "ident"
+    "op"
+    "keyword"
+    "int"
+    "op"
+    "semi"
+    "eof"))
+
+;; ── report ────────────────────────────────────────────────────────
+(define go-lex-test-summary (str "lex " go-test-pass "/" go-test-count))

plans/go-on-sx.md

@@ -131,16 +131,21 @@ Loop-style. Each phase: implement → test → commit → tick `[ ]` → append
 Progress-log line → push `origin/loops/go`.
 
 ### Phase 1 — Tokenizer (`lib/go/lex.sx`) ⬜
-- Consume `lib/guest/core/lex.sx`. Tag the chisel note `consumes-lex`.
+- [x] Scaffold + scoreboard + conformance runner (consumes lib/guest/lex.sx)
-- Keywords (25), operators + punctuation (47 distinct), identifiers,
+- [x] Identifiers + 25 keywords
-  literals (int / float / imaginary / rune / string with raw + interpreted
+- [x] Decimal integer literals
-  variants), comments.
+- [x] Interpreted string literals `"..."` with `\n \t \r \\ \" \'` escapes
-- **Automatic semicolon insertion** — the one tricky bit. Newline becomes
+- [x] Rune literals `'x'` (single char + simple escapes)
-  `;` after identifier/literal/`)`/`]`/`}` per Go spec § Semicolons. Build
+- [x] Line + block comments (block w/ newline triggers ASI)
-  it into the tokenizer, not the parser.
+- [x] Common operator/punct set incl. `:= <- ++ -- == != <= >= && || ...`
-- Tests: golden-token streams for every keyword/operator/literal kind +
+- [x] **Automatic semicolon insertion** (Go spec § Semicolons) — newline,
-  ASI edge cases.
+      EOF, and block-comment-with-newline trigger `;` after
-- **Acceptance:** lex/ suite at 50+ tests.
+      ident/int/string/rune/{break,continue,fallthrough,return}/{++,--,),],}}.
+- [ ] Float / imaginary literals
+- [ ] Raw string literals `` `...` ``
+- [ ] Hex/octal/binary integer literals (0x… 0o… 0b…) + underscores
+- [ ] Full operator set audit (47 distinct per Go spec)
+- **Acceptance:** lex/ suite at 50+ tests. Current: 78/78.
 
 ### Phase 2 — Parser (`lib/go/parse.sx`) ⬜
 - Consume `lib/guest/core/pratt.sx` + `lib/guest/core/ast.sx`. Chisel notes
@@ -399,6 +404,11 @@ _(none yet)_
 
 _Newest first. Append one dated entry per commit._
 
+- 2026-05-26 — Phase 1 first slice: `lib/go/lex.sx` tokenizer consuming
+  `lib/guest/lex.sx` predicates. 25 keywords, ident/int/string/rune lits,
+  line+block comments, common operators, automatic semicolon insertion per
+  Go spec § Semicolons (newline / EOF / block-comment-with-newline triggers).
+  Scoreboard + conformance.sh wired. 78/78 tests. `[consumes-lex]`.
 - 2026-05-26 — Plan rewritten to integrate the lib/guest framework
   (chiselling discipline, sister plans for scheduler + bidirectional
   types, type-checker phase added, conformance scoreboard model adopted).