diff --git a/sx/sx/nav-data.sx b/sx/sx/nav-data.sx index e8a480a..d70cd6c 100644 --- a/sx/sx/nav-data.sx +++ b/sx/sx/nav-data.sx @@ -103,7 +103,11 @@ (define plans-nav-items (list (dict :label "Isomorphic Architecture" :href "/plans/isomorphic-architecture" - :summary "Making the server/client boundary a sliding window — per-page bundles, smart expansion, SPA routing, client IO, streaming suspense."))) + :summary "Making the server/client boundary a sliding window — per-page bundles, smart expansion, SPA routing, client IO, streaming suspense.") + (dict :label "Reader Macros" :href "/plans/reader-macros" + :summary "Extensible parse-time transformations via # dispatch — datum comments, raw strings, and quote shorthand.") + (dict :label "SX-Activity" :href "/plans/sx-activity" + :summary "ActivityPub federation with SX as wire format — IPFS-backed component registry, content-addressed media, Bitcoin-anchored provenance."))) (define bootstrappers-nav-items (list (dict :label "Overview" :href "/bootstrappers/") diff --git a/sx/sx/plans.sx b/sx/sx/plans.sx index 1c0b0d7..65afda3 100644 --- a/sx/sx/plans.sx +++ b/sx/sx/plans.sx @@ -20,6 +20,489 @@ (p :class "text-sm text-stone-500 mt-1" (get item "summary"))))) plans-nav-items))))) +;; --------------------------------------------------------------------------- +;; Reader Macros +;; --------------------------------------------------------------------------- + +(defcomp ~plan-reader-macros-content () + (~doc-page :title "Reader Macros" + + (~doc-section :title "Context" :id "context" + (p "SX has three hardcoded reader transformations: " (code "`") " → " (code "(quasiquote ...)") ", " (code ",") " → " (code "(unquote ...)") ", " (code ",@") " → " (code "(splice-unquote ...)") ". These are baked into the parser with no extensibility. The " (code "~") " prefix for components and " (code "&") " for param modifiers are just symbol characters, handled at eval time.") + (p "Reader macros add parse-time transformations triggered by a dispatch character. Motivating use case: a " (code "~md") " component that uses heredoc syntax for markdown source instead of string literals. More broadly: datum comments, raw strings, and custom literal syntax.")) + + (~doc-section :title "Design" :id "design" + + (~doc-subsection :title "Dispatch Character: #" + (p "Lisp tradition. " (code "#") " is NOT in " (code "ident-start") " or " (code "ident-char") " — completely free. Pattern:") + (~doc-code :code (highlight "#;expr → (read and discard expr, return next)\n#|...| → raw string literal\n#'expr → (quote expr)" "lisp"))) + + (~doc-subsection :title "#; — Datum comment" + (p "Scheme/Racket standard. Reads and discards the next expression. Preserves balanced parens.") + (~doc-code :code (highlight "(list 1 #;(this is commented out) 2 3) → (list 1 2 3)" "lisp"))) + + (~doc-subsection :title "#|...| — Raw string" + (p "No escape processing. Everything between " (code "#|") " and " (code "|") " is literal. Enables inline markdown, regex patterns, code examples.") + (~doc-code :code (highlight "(~md #|## Title\n\nSome **bold** text with \"quotes\" and \\backslashes.|)" "lisp"))) + + (~doc-subsection :title "#' — Quote shorthand" + (p "Currently no single-char quote (" (code "`") " is quasiquote).") + (~doc-code :code (highlight "#'my-function → (quote my-function)" "lisp"))) + + (~doc-subsection :title "No user-defined reader macros (yet)" + (p "Would require multi-pass parsing or boot-phase registration. The three built-ins cover practical needs. Extensible dispatch can come later without breaking anything."))) + + ;; ----------------------------------------------------------------------- + ;; Implementation + ;; ----------------------------------------------------------------------- + + (~doc-section :title "Implementation" :id "implementation" + + (~doc-subsection :title "1. Spec: parser.sx" + (p "Add " (code "#") " dispatch to " (code "read-expr") " (after the " (code ",") "/" (code ",@") " case, before number). Add " (code "read-raw-string") " helper function.") + (~doc-code :code (highlight ";; Reader macro dispatch\n(= ch \"#\")\n (do (set! pos (inc pos))\n (if (>= pos len-src)\n (error \"Unexpected end of input after #\")\n (let ((dispatch-ch (nth source pos)))\n (cond\n ;; #; — datum comment: read and discard next expr\n (= dispatch-ch \";\")\n (do (set! pos (inc pos))\n (read-expr) ;; read and discard\n (read-expr)) ;; return the NEXT expr\n\n ;; #| — raw string\n (= dispatch-ch \"|\")\n (do (set! pos (inc pos))\n (read-raw-string))\n\n ;; #' — quote shorthand\n (= dispatch-ch \"'\")\n (do (set! pos (inc pos))\n (list (make-symbol \"quote\") (read-expr)))\n\n :else\n (error (str \"Unknown reader macro: #\" dispatch-ch))))))" "lisp")) + (p "The " (code "read-raw-string") " helper:") + (~doc-code :code (highlight "(define read-raw-string\n (fn ()\n (let ((buf \"\"))\n (define raw-loop\n (fn ()\n (if (>= pos len-src)\n (error \"Unterminated raw string\")\n (let ((ch (nth source pos)))\n (if (= ch \"|\")\n (do (set! pos (inc pos)) nil) ;; done\n (do (set! buf (str buf ch))\n (set! pos (inc pos))\n (raw-loop)))))))\n (raw-loop)\n buf)))" "lisp"))) + + (~doc-subsection :title "2. Python: parser.py" + (p "Add " (code "#") " dispatch to " (code "_parse_expr()") " (after " (code ",") "/" (code ",@") " handling ~line 252). Add " (code "_read_raw_string()") " method to Tokenizer.") + (~doc-code :code (highlight "# In _parse_expr(), after the comma/splice-unquote block:\nif raw == \"#\":\n tok._advance(1) # consume the #\n if tok.pos >= len(tok.text):\n raise ParseError(\"Unexpected end of input after #\",\n tok.pos, tok.line, tok.col)\n dispatch = tok.text[tok.pos]\n if dispatch == \";\":\n tok._advance(1)\n _parse_expr(tok) # read and discard\n return _parse_expr(tok) # return next\n if dispatch == \"|\":\n tok._advance(1)\n return tok._read_raw_string()\n if dispatch == \"'\":\n tok._advance(1)\n return [Symbol(\"quote\"), _parse_expr(tok)]\n raise ParseError(f\"Unknown reader macro: #{dispatch}\",\n tok.pos, tok.line, tok.col)" "python")) + (p "The " (code "_read_raw_string()") " method on Tokenizer:") + (~doc-code :code (highlight "def _read_raw_string(self) -> str:\n buf = []\n while self.pos < len(self.text):\n ch = self.text[self.pos]\n if ch == \"|\":\n self._advance(1)\n return \"\".join(buf)\n buf.append(ch)\n self._advance(1)\n raise ParseError(\"Unterminated raw string\",\n self.pos, self.line, self.col)" "python"))) + + (~doc-subsection :title "3. JS: auto-transpiled" + (p "JS parser comes from bootstrap of parser.sx — spec change handles it automatically.")) + + (~doc-subsection :title "4. Rebootstrap both targets" + (p "Run " (code "bootstrap_js.py") " and " (code "bootstrap_py.py") " to regenerate " (code "sx-ref.js") " and " (code "sx_ref.py") " from the updated parser.sx spec.")) + + (~doc-subsection :title "5. Grammar update" + (p "Add reader macro syntax to the grammar comment at the top of parser.sx:") + (~doc-code :code (highlight ";; reader → '#;' expr (datum comment)\n;; | '#|' raw-chars '|' (raw string)\n;; | \"#'\" expr (quote shorthand)" "lisp")))) + + ;; ----------------------------------------------------------------------- + ;; Files + ;; ----------------------------------------------------------------------- + + (~doc-section :title "Files" :id "files" + (div :class "overflow-x-auto rounded border border-stone-200" + (table :class "w-full text-left text-sm" + (thead (tr :class "border-b border-stone-200 bg-stone-100" + (th :class "px-3 py-2 font-medium text-stone-600" "File") + (th :class "px-3 py-2 font-medium text-stone-600" "Change"))) + (tbody + (tr :class "border-b border-stone-100" + (td :class "px-3 py-2 font-mono text-sm text-violet-700" "shared/sx/ref/parser.sx") + (td :class "px-3 py-2 text-stone-700" "# dispatch in read-expr, read-raw-string helper, grammar comment")) + (tr :class "border-b border-stone-100" + (td :class "px-3 py-2 font-mono text-sm text-violet-700" "shared/sx/parser.py") + (td :class "px-3 py-2 text-stone-700" "# dispatch in _parse_expr(), _read_raw_string()")) + (tr :class "border-b border-stone-100" + (td :class "px-3 py-2 font-mono text-sm text-violet-700" "shared/sx/ref/sx_ref.py") + (td :class "px-3 py-2 text-stone-700" "Rebootstrap")) + (tr :class "border-b border-stone-100" + (td :class "px-3 py-2 font-mono text-sm text-violet-700" "shared/static/scripts/sx-ref.js") + (td :class "px-3 py-2 text-stone-700" "Rebootstrap")))))) + + ;; ----------------------------------------------------------------------- + ;; Verification + ;; ----------------------------------------------------------------------- + + (~doc-section :title "Verification" :id "verification" + + (~doc-subsection :title "Parse tests" + (ul :class "list-disc pl-5 text-stone-700 space-y-1 font-mono text-sm" + (li "#;(ignored) 42 → 42") + (li "(list 1 #;2 3) → (list 1 3)") + (li "#|hello \"world\" \\n| → string: hello \"world\" \\n (literal, no escaping)") + (li "#|multi\\nline| → string with actual newline") + (li "#'foo → (quote foo)") + (li "# at EOF → error") + (li "#x unknown → error"))) + + (~doc-subsection :title "Regression" + (ul :class "list-disc pl-5 text-stone-700 space-y-1" + (li "All existing parser tests pass after changes") + (li "Rebootstrapped JS and Python pass their test suites") + (li "JS parity: SX.parse('#|hello|') returns [\"hello\"]")))))) + +;; --------------------------------------------------------------------------- +;; SX-Activity: Federated SX over ActivityPub +;; --------------------------------------------------------------------------- + +(defcomp ~plan-sx-activity-content () + (~doc-page :title "SX-Activity" + + (~doc-section :title "Context" :id "context" + (p "ActivityPub federates social actions using JSON-LD over HTTP. It works, but JSON-LD is verbose, requires context resolution, and carries no computation — it's inert data that every consumer must interpret from scratch. Meanwhile SX already has: a compact wire format, a safe evaluable language, content-addressed DAG execution (artdag), IPFS storage with CIDs, and OpenTimestamps Bitcoin anchoring.") + (p "SX-Activity replaces JSON-LD with s-expressions as the federation wire format. Activities become " (strong "evaluable programs") " rather than inert data. Components travel with content, stored on IPFS, resolved by content address. Media is content-addressed. Provenance is anchored in the Bitcoin chain. The result: a federation protocol where receiving a post means receiving the ability to " (em "render") " it — not just the data, but the UI.") + (p "The key insight: " (strong "if the wire format is already an evaluable language, federation becomes code distribution.") " A Create activity carrying a Note isn't just data — it's a renderable document with its own component definitions, stylesheets, and media references, all content-addressed and independently verifiable.")) + + (~doc-section :title "Current State" :id "current-state" + (ul :class "space-y-2 text-stone-700 list-disc pl-5" + (li (strong "ActivityPub: ") "Full implementation — virtual per-app actors, HTTP signatures, webfinger, inbox/outbox, followers/following, delivery with idempotent logging.") + (li (strong "Activity bus: ") "Unified event bus with NOTIFY/LISTEN wakeup, at-least-once delivery, handler registry keyed by (activity_type, object_type).") + (li (strong "Content addressing: ") "artdag nodes use SHA3-256 hashing. Cache layer tracks IPFS CIDs. IPFSPin model tracks pinned content across domains.") + (li (strong "Bitcoin anchoring: ") "APAnchor model — Merkle tree of activities, OpenTimestamps proof CID, Bitcoin txid. Infrastructure exists but isn't wired to all activity types.") + (li (strong "SX wire format: ") "Server serializes to SX source via _aser, client parses and renders. Component caching via localStorage + content hashes.") + (li (strong "Boundary enforcement: ") "SX_BOUNDARY_STRICT=1 validates all primitives at registration. Pure components can't do IO — safe to load from untrusted sources."))) + + ;; ----------------------------------------------------------------------- + ;; Phase 1: SX Wire Format for Activities + ;; ----------------------------------------------------------------------- + + (~doc-section :title "Phase 1: SX Wire Format for Activities" :id "phase-1" + + (div :class "rounded border border-violet-200 bg-violet-50 p-4 mb-4" + (p :class "text-violet-900 font-medium" "What it enables") + (p :class "text-violet-800" "Activities expressed as s-expressions instead of JSON-LD. Same semantics as ActivityStreams, but compact, parseable, and directly evaluable. Dual-format support for backward compatibility with existing AP servers.")) + + (~doc-subsection :title "The Problem" + (p "JSON-LD activities are verbose and require context resolution:") + (~doc-code :code (highlight "{\"@context\": \"https://www.w3.org/ns/activitystreams\",\n \"type\": \"Create\",\n \"actor\": \"https://example.com/users/alice\",\n \"object\": {\n \"type\": \"Note\",\n \"content\": \"
Hello world
\",\n \"attributedTo\": \"https://example.com/users/alice\"\n }}" "json")) + (p "Every consumer parses JSON, resolves @context, extracts fields, then builds their own UI around the raw data. The content is HTML embedded in a JSON string — two formats nested, neither evaluable.")) + + (~doc-subsection :title "SX Activity Format" + (p "The same activity as SX:") + (~doc-code :code (highlight "(Create\n :actor \"https://example.com/users/alice\"\n :published \"2026-03-06T12:00:00Z\"\n :object (Note\n :attributed-to \"https://example.com/users/alice\"\n :content (p \"Hello world\")\n :media-type \"text/sx\"))" "lisp")) + (p "The content isn't a string containing markup — it " (em "is") " markup. The receiving server can evaluate it directly. The Note's content is a renderable SX expression.")) + + (~doc-subsection :title "Approach" + (div :class "space-y-4" + (div + (h4 :class "font-semibold text-stone-700" "1. Activity vocabulary in SX") + (p "Map ActivityStreams types to SX symbols. Activities are lists with a type head and keyword properties:") + (~doc-code :code (highlight ";; Core activity types\n(Create :actor ... :object ...)\n(Update :actor ... :object ...)\n(Delete :actor ... :object ...)\n(Follow :actor ... :object ...)\n(Like :actor ... :object ...)\n(Announce :actor ... :object ...)\n\n;; Object types\n(Note :content ... :attributed-to ...)\n(Article :name ... :content ... :summary ...)\n(Image :url ... :media-type ... :cid ...)\n(Collection :total-items ... :items ...)" "lisp"))) + + (div + (h4 :class "font-semibold text-stone-700" "2. Content negotiation") + (p "Inbox accepts both formats. " (code "Accept: text/sx") " gets SX, " (code "Accept: application/activity+json") " gets JSON-LD. Outbox serves both. SX-native servers negotiate SX; legacy Mastodon/Pleroma servers get JSON-LD as today.")) + + (div + (h4 :class "font-semibold text-stone-700" "3. Bidirectional translation") + (p "Lossless mapping between JSON-LD and SX activity formats. Translate at the boundary — internal processing always uses SX. The existing " (code "APActivity") " model gains an " (code "sx_source") " column storing the canonical SX representation.")) + + (div + (h4 :class "font-semibold text-stone-700" "4. HTTP Signatures over SX") + (p "Same RSA signature mechanism. Digest header computed over the SX body. Existing keypair infrastructure unchanged.")))) + + (~doc-subsection :title "Verification" + (ul :class "list-disc pl-5 text-stone-700 space-y-1" + (li "Round-trip: SX → JSON-LD → SX produces identical output") + (li "Legacy AP servers receive valid JSON-LD (Mastodon can display the post)") + (li "SX-native servers receive evaluable SX (client can render directly)") + (li "HTTP signatures verify over SX bodies")))) + + ;; ----------------------------------------------------------------------- + ;; Phase 2: Content-Addressed Components on IPFS + ;; ----------------------------------------------------------------------- + + (~doc-section :title "Phase 2: Content-Addressed Components on IPFS" :id "phase-2" + + (div :class "rounded border border-violet-200 bg-violet-50 p-4 mb-4" + (p :class "text-violet-900 font-medium" "What it enables") + (p :class "text-violet-800" "Component definitions stored on IPFS, referenced by CID. Any server can publish components. Any browser can fetch them. No central registry — content addressing IS the registry.")) + + (~doc-subsection :title "The Insight" + (p "SX components are pure functions — they take data and return markup. They can't do IO (boundary enforcement guarantees this). That means they're " (strong "safe to load from any source") ". And if they're content-addressed, the CID " (em "is") " the identity — you don't need to trust the source, you just verify the hash.") + (p "Currently, component definitions travel with each page via " (code "