From 858275dff9952ed5a2abc41baa8c2f1e1741f63d Mon Sep 17 00:00:00 2001 From: giles Date: Tue, 24 Mar 2026 23:41:28 +0000 Subject: [PATCH 1/6] sx-pub: plan page, dev infrastructure, JS bundle rebuild - Add sx-pub plan page under Applications with full protocol spec: wire format, endpoints, storage, flows, AP critique, phases - Add nav entry and page function for /sx/(applications.(sx-pub)) - Add docker-compose.dev-pub.yml (sx_docs image + DB + IPFS + Redis) - Add dev-pub.sh launch script (pub.sx-web.org via Caddy) - Rebuild sx-browser.js with var fixes for SES compatibility Co-Authored-By: Claude Opus 4.6 (1M context) --- dev-pub.sh | 30 ++++ docker-compose.dev-pub.yml | 112 ++++++++++++ shared/static/scripts/sx-browser.js | 14 +- sx/sx/nav-data.sx | 3 +- sx/sx/page-functions.sx | 7 + sx/sx/plans/sx-pub.sx | 257 ++++++++++++++++++++++++++++ 6 files changed, 415 insertions(+), 8 deletions(-) create mode 100755 dev-pub.sh create mode 100644 docker-compose.dev-pub.yml create mode 100644 sx/sx/plans/sx-pub.sx diff --git a/dev-pub.sh b/dev-pub.sh new file mode 100755 index 00000000..47074458 --- /dev/null +++ b/dev-pub.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Dev mode for sx-pub (SX-based ActivityPub) +# Bind-mounted source + auto-reload on externalnet +# Browse to pub.sx-web.org +# +# Usage: +# ./dev-pub.sh # Start sx-pub dev +# ./dev-pub.sh down # Stop +# ./dev-pub.sh logs # Tail logs +# ./dev-pub.sh --build # Rebuild image then start + +COMPOSE="docker compose -p sx-pub -f docker-compose.dev-pub.yml" + +case "${1:-up}" in + down) + $COMPOSE down + ;; + logs) + $COMPOSE logs -f sx_pub + ;; + *) + BUILD_FLAG="" + if [[ "${1:-}" == "--build" ]]; then + BUILD_FLAG="--build" + fi + $COMPOSE up $BUILD_FLAG + ;; +esac diff --git a/docker-compose.dev-pub.yml b/docker-compose.dev-pub.yml new file mode 100644 index 00000000..f2867030 --- /dev/null +++ b/docker-compose.dev-pub.yml @@ -0,0 +1,112 @@ +# Dev mode for sx-pub (SX-based ActivityPub) +# Starts as sx_docs clone — AP protocol built in SX from scratch +# Accessible at pub.sx-web.org via Caddy on externalnet +# Own DB + pgbouncer + IPFS node + +services: + sx_pub: + image: registry.rose-ash.com:5000/sx_docs:latest + environment: + SX_STANDALONE: "true" + SECRET_KEY: "${SECRET_KEY:-pub-dev-secret}" + REDIS_URL: redis://redis:6379/0 + DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:5432/sx_pub + WORKERS: "1" + ENVIRONMENT: development + RELOAD: "true" + SX_USE_REF: "1" + SX_USE_OCAML: "1" + SX_OCAML_BIN: "/app/bin/sx_server" + SX_BOUNDARY_STRICT: "1" + SX_DEV: "1" + OCAMLRUNPARAM: "b" + IPFS_API: http://ipfs:5001 + ports: + - "8014:8000" + volumes: + - /root/sx-pub/_config/dev-sh-config.yaml:/app/config/app-config.yaml:ro + - ./shared:/app/shared + - ./sx/app.py:/app/app.py + - ./sx/sxc:/app/sxc + - ./sx/bp:/app/bp + - ./sx/services:/app/services + - ./sx/content:/app/content + - ./sx/sx:/app/sx + - ./sx/path_setup.py:/app/path_setup.py + - ./sx/entrypoint.sh:/usr/local/bin/entrypoint.sh + - ./sx/__init__.py:/app/__init__.py:ro + # Spec + web SX files + - ./spec:/app/spec:ro + - ./web:/app/web:ro + # OCaml SX kernel binary + - ./hosts/ocaml/_build/default/bin/sx_server.exe:/app/bin/sx_server:ro + # sibling models for cross-domain SQLAlchemy imports + - ./blog/__init__.py:/app/blog/__init__.py:ro + - ./blog/models:/app/blog/models:ro + - ./market/__init__.py:/app/market/__init__.py:ro + - ./market/models:/app/market/models:ro + - ./cart/__init__.py:/app/cart/__init__.py:ro + - ./cart/models:/app/cart/models:ro + - ./events/__init__.py:/app/events/__init__.py:ro + - ./events/models:/app/events/models:ro + - ./federation/__init__.py:/app/federation/__init__.py:ro + - ./federation/models:/app/federation/models:ro + - ./account/__init__.py:/app/account/__init__.py:ro + - ./account/models:/app/account/models:ro + - ./relations/__init__.py:/app/relations/__init__.py:ro + - ./relations/models:/app/relations/models:ro + - ./likes/__init__.py:/app/likes/__init__.py:ro + - ./likes/models:/app/likes/models:ro + - ./orders/__init__.py:/app/orders/__init__.py:ro + - ./orders/models:/app/orders/models:ro + depends_on: + - pgbouncer + - redis + - ipfs + networks: + - externalnet + - default + restart: unless-stopped + + db: + image: postgres:16 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: change-me + POSTGRES_DB: sx_pub + volumes: + - db_data:/var/lib/postgresql/data + restart: unless-stopped + + pgbouncer: + image: edoburu/pgbouncer:latest + environment: + DB_HOST: db + DB_PORT: "5432" + DB_USER: postgres + DB_PASSWORD: change-me + POOL_MODE: transaction + DEFAULT_POOL_SIZE: "10" + MAX_CLIENT_CONN: "100" + AUTH_TYPE: plain + depends_on: + - db + restart: unless-stopped + + ipfs: + image: ipfs/kubo:latest + volumes: + - ipfs_data:/data/ipfs + restart: unless-stopped + + redis: + image: redis:7-alpine + restart: unless-stopped + +volumes: + db_data: + ipfs_data: + +networks: + externalnet: + external: true diff --git a/shared/static/scripts/sx-browser.js b/shared/static/scripts/sx-browser.js index 18357a24..48e9b420 100644 --- a/shared/static/scripts/sx-browser.js +++ b/shared/static/scripts/sx-browser.js @@ -14,7 +14,7 @@ // ========================================================================= var NIL = Object.freeze({ _nil: true, toString: function() { return "nil"; } }); - var SX_VERSION = "2026-03-24T22:31:01Z"; + var SX_VERSION = "2026-03-24T22:34:15Z"; function isNil(x) { return x === NIL || x === null || x === undefined; } function isSxTruthy(x) { return x !== false && !isNil(x); } @@ -4769,7 +4769,7 @@ PRIMITIVES["bind-sse-swap"] = bindSseSwap; var eventName = slice(name, 6); return (isSxTruthy(!isSxTruthy(isProcessed(el, (String("on:") + String(eventName))))) ? (markProcessed(el, (String("on:") + String(eventName))), (function() { var exprs = sxParse(body); - return domOn(el, eventName, function(e) { return (function() { + return domListen(el, eventName, function(e) { return (function() { var handlerEnv = envExtend({}); envBind(handlerEnv, "event", e); envBind(handlerEnv, "this", el); @@ -4826,7 +4826,7 @@ PRIMITIVES["process-one"] = processOne; var els = domQueryAll(sxOr(root, domBody()), "[data-sx-emit]"); return forEach(function(el) { return (isSxTruthy(!isSxTruthy(isProcessed(el, "emit"))) ? (markProcessed(el, "emit"), (function() { var eventName = domGetAttr(el, "data-sx-emit"); - return (isSxTruthy(eventName) ? domOn(el, "click", function(e) { return (function() { + return (isSxTruthy(eventName) ? domListen(el, "click", function(e) { return (function() { var detailJson = domGetAttr(el, "data-sx-emit-detail"); var detail = (isSxTruthy(detailJson) ? jsonParse(detailJson) : {}); return domDispatch(el, eventName, detail); @@ -5973,12 +5973,12 @@ PRIMITIVES["clear-stores"] = clearStores; PRIMITIVES["emit-event"] = emitEvent; // on-event - var onEvent = function(el, eventName, handler) { return domOn(el, eventName, handler); }; + var onEvent = function(el, eventName, handler) { return domListen(el, eventName, handler); }; PRIMITIVES["on-event"] = onEvent; // bridge-event var bridgeEvent = function(el, eventName, targetSignal, transformFn) { return effect(function() { return (function() { - var remove = domOn(el, eventName, function(e) { return (function() { + var remove = domListen(el, eventName, function(e) { return (function() { var detail = eventDetail(e); var newVal = (isSxTruthy(transformFn) ? cekCall(transformFn, [detail]) : detail); return reset_b(targetSignal, newVal); @@ -6327,8 +6327,8 @@ PRIMITIVES["resource"] = resource; // If lambda takes 0 params, call without event arg (convenience for on-click handlers) var wrapped = isLambda(handler) ? (lambdaParams(handler).length === 0 - ? function(e) { try { cekCall(handler, NIL); } catch(err) { console.error("[sx-ref] domListen handler error:", name, err); } } - : function(e) { try { cekCall(handler, [e]); } catch(err) { console.error("[sx-ref] domListen handler error:", name, err); } }) + ? function(e) { try { cekCall(handler, NIL); } catch(err) { console.error("[sx-ref] domListen handler error:", name, err); } finally { runPostRenderHooks(); } } + : function(e) { try { cekCall(handler, [e]); } catch(err) { console.error("[sx-ref] domListen handler error:", name, err); } finally { runPostRenderHooks(); } }) : handler; if (name === "click") logInfo("domListen: click on <" + (el.tagName||"?").toLowerCase() + "> text=" + (el.textContent||"").substring(0,20) + " isLambda=" + isLambda(handler)); var passiveEvents = { touchstart: 1, touchmove: 1, wheel: 1, scroll: 1 }; diff --git a/sx/sx/nav-data.sx b/sx/sx/nav-data.sx index f3e0af0c..a88937c0 100644 --- a/sx/sx/nav-data.sx +++ b/sx/sx/nav-data.sx @@ -443,7 +443,8 @@ :children (list {:label "SX URLs" :href "/sx/(applications.(sx-urls))"} {:label "CSSX" :href "/sx/(applications.(cssx))" :children cssx-nav-items} - {:label "Protocols" :href "/sx/(applications.(protocol))" :children protocols-nav-items})} + {:label "Protocols" :href "/sx/(applications.(protocol))" :children protocols-nav-items} + {:label "sx-pub" :href "/sx/(applications.(sx-pub))"})} {:label "Etc" :href "/sx/(etc)" :children (list {:label "Essays" :href "/sx/(etc.(essay))" :children essays-nav-items} diff --git a/sx/sx/page-functions.sx b/sx/sx/page-functions.sx index b507a8c4..ac7ac1f3 100644 --- a/sx/sx/page-functions.sx +++ b/sx/sx/page-functions.sx @@ -461,6 +461,13 @@ (define protocol (make-page-fn "~protocols/wire-format-content" "~protocols/" nil "-content")) +;; sx-pub (under applications) +(define sx-pub + (fn (slug) + (if (nil? slug) + '(~sx-pub/overview-content) + nil))) + ;; Essays (under etc) ;; Convention: ~essays/{slug}/essay-{slug} (define essay diff --git a/sx/sx/plans/sx-pub.sx b/sx/sx/plans/sx-pub.sx new file mode 100644 index 00000000..76c561d8 --- /dev/null +++ b/sx/sx/plans/sx-pub.sx @@ -0,0 +1,257 @@ +;; --------------------------------------------------------------------------- +;; sx-pub: Federated SX Publishing Protocol +;; --------------------------------------------------------------------------- + +(defcomp ~sx-pub/overview-content () + (~docs/page :title "sx-pub" + + (~docs/section :title "What It Is" :id "what" + (p "A federated content publishing protocol built entirely in SX. No JSON. Content pinned to IPFS, referenced by CID. Blockchain-anchored provenance.") + (p "One actor — " (code "sx") " — publishes SX specs, platform libraries, components, and documentation as content-addressed packages. Other sx-pub servers can follow, mirror, and refer to content by CID. Human-readable paths map to immutable IPFS addresses.") + (div :class "rounded border border-violet-200 bg-violet-50 p-4 mt-4" + (p :class "text-violet-900 font-medium" "The key insight") + (p :class "text-violet-800" "IPFS is the database. SX is the wire format. defhandlers are the endpoints. The wire format is the programming language is the component system is the package manager."))) + + ;; ----------------------------------------------------------------------- + ;; Why ActivityPub Failed — and Why LISP Fixes It + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Why ActivityPub Failed" :id "why" + (p "ActivityPub's problems aren't bugs — they're consequences of a fundamental design error: " (strong "the wire format is dead data") ". JSON-LD activities describe what happened but can't " (em "do") " anything. Every consumer must independently interpret, render, and extend. The fediverse is N incompatible applications showing the same inert data differently.") + + (~docs/subsection :title "1. The Rendering Gap" + (p "An ActivityPub Note looks like this:") + (~docs/code :code "{\"type\": \"Note\",\n \"content\": \"

Hello world

\",\n \"attributedTo\": \"https://example.com/users/alice\"}") + (p "The content is an HTML string embedded in a JSON string. Two formats nested, neither evaluable. Every client — Mastodon, Pleroma, Misskey, Lemmy — parses this independently and wraps it in their own UI. A poll renders differently everywhere. An event looks different everywhere. There is no shared component model because the protocol has no concept of components.") + (p "In sx-pub, the content " (em "is") " the component:") + (~docs/code :code "(Note\n :attributed-to \"https://pub.sx-web.org/pub/actor\"\n :content (p \"Hello \" (strong \"world\")))") + (p "No HTML-in-JSON. No parsing step. The receiver evaluates the s-expression and gets rendered output. The content carries its own rendering semantics.")) + + (~docs/subsection :title "2. Inert Data vs. Evaluable Programs" + (p "A JSON-LD activity is a description. It says \"Alice created a Note with this HTML.\" The receiving server must then decide: how do I display a Note? What if it has attachments? What about custom emoji? What about a poll? Every new feature requires every implementation to add rendering code.") + (p "An SX activity is a " (em "program") ". It evaluates to its own UI. A poll isn't a JSON blob that each client renders differently — it's a " (code "(defcomp ~poll ...)") " that evaluates identically everywhere because the CEK machine has deterministic semantics. " (strong "The wire format is the renderer."))) + + (~docs/subsection :title "3. Extensions Are Impossible" + (p "ActivityPub extensions (FEPs) add new JSON fields that most implementations ignore. Want to federate a recipe, a calendar event, a code snippet with syntax highlighting? Write a spec, wait years for implementations to support it, accept that most never will.") + (p "In sx-pub, extensions are just components. Publish " (code "(defcomp ~recipe ...)") " to IPFS. Anyone who follows you gets the component. Their server can evaluate it immediately — " (strong "no implementation changes needed") ". The extension mechanism is the same as the content mechanism: pin an s-expression, reference it by CID.")) + + (~docs/subsection :title "4. No Content Integrity" + (p "ActivityPub has no content addressing. A server can silently edit a federated post. HTTP signatures verify the " (em "transport") " but not the " (em "content") ". There's no way to prove that what you see is what was originally published.") + (p "sx-pub content is IPFS-pinned and referenced by CID (content hash). The CID " (em "is") " the identity. Mutate a single character and the CID changes. Merkle trees of CIDs get anchored to Bitcoin via OpenTimestamps — cryptographic proof of what was published and when.")) + + (~docs/subsection :title "5. The N-Client Problem" + (p "Because AP content is inert, every client must build its own renderer, interaction model, and extension system. The result: Mastodon is one application, Pleroma is another, Misskey is another. They can exchange data, but the " (em "experience") " is entirely determined by the client. Rich content degrades to plain text when the client doesn't understand it.") + (p "sx-pub eliminates this entirely. Content carries its rendering logic. A server that receives " (code "(defcomp ~interactive-chart ...)") " can render it correctly without any prior knowledge of charts — because the component is self-contained, sandboxed (boundary enforcement prevents IO in pure components), and deterministic. " (strong "Every server renders the same content the same way."))) + + (~docs/subsection :title "6. No Dependency System" + (p "AP has no concept of shared code. If ten ActivityPub projects all need a Markdown parser, ten teams write ten Markdown parsers. There's no mechanism to publish reusable logic that travels between servers.") + (p "sx-pub's " (code ":requires") " field on published documents declares CID dependencies. A component that uses a chart library says " (code ":requires (list \"bafy...chart-lib\")") ". Pin the dependency tree and everything works offline. " (strong "This is a package manager built into the federation protocol.") " No npm, no crates.io, no central registry — just content-addressed s-expressions on IPFS.")) + + (~docs/subsection :title "7. The @context Illusion" + (p "JSON-LD @context was supposed to provide semantic interoperability. In practice it's a constant source of bugs. Implementations disagree on which contexts to support, which extensions to recognise, how to resolve conflicts. The W3C ActivityStreams context document is a single point of failure that the entire fediverse depends on.") + (p "SX needs no context resolution. " (code "(Publish :actor \"...\" :object (SxDocument :cid \"...\"))") " is self-describing. The s-expression " (em "is") " the meaning. Symbols are symbols. Keywords are keywords. No indirection, no remote JSON documents, no semantic web machinery.")) + + (div :class "rounded border border-rose-200 bg-rose-50 p-4 mt-6" + (p :class "text-rose-900 font-medium" "The fundamental difference") + (p :class "text-rose-800" "ActivityPub federates " (em "descriptions") " of content. Consumers must independently figure out how to present them. sx-pub federates " (em "programs") " — self-contained, evaluable, deterministic, content-addressed. The gap between \"data\" and \"application\" disappears because in LISP, they were never separate to begin with."))) + + ;; ----------------------------------------------------------------------- + ;; Wire Format + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Wire Format" :id "wire-format" + (p "Everything is s-expressions. No JSON-LD, no @context resolution, no nested HTML-in-JSON strings.") + + (~docs/subsection :title "Actor" + (~docs/code :code "(SxActor\n :id \"https://pub.sx-web.org/pub/actor\"\n :type \"SxPublisher\"\n :name \"sx\"\n :summary \"SX language — specs, platforms, components\"\n :public-key-pem \"-----BEGIN PUBLIC KEY-----\\n...\"\n :inbox \"/pub/inbox\"\n :outbox \"/pub/outbox\"\n :followers \"/pub/followers\"\n :following \"/pub/following\"\n :collections (list\n (SxCollection :name \"core-specs\" :href \"/pub/core-specs\")\n (SxCollection :name \"platforms\" :href \"/pub/platforms\")\n (SxCollection :name \"components\" :href \"/pub/components\")))")) + + (~docs/subsection :title "Publish Activity" + (p "When content is published, it's pinned to IPFS and announced to followers:") + (~docs/code :code "(Publish\n :actor \"https://pub.sx-web.org/pub/actor\"\n :published \"2026-03-24T12:00:00Z\"\n :object (SxDocument\n :name \"evaluator\"\n :path \"/pub/core-specs/evaluator\"\n :cid \"bafy...eval123\"\n :content-type \"text/sx\"\n :size 48000\n :hash \"sha3-256:abc123...\"\n :requires (list \"bafy...parser\" \"bafy...primitives\")\n :summary \"CEK machine evaluator — frames, utils, step function\"))") + (p "The " (code ":requires") " field declares CID dependencies — a component that imports the parser declares that dependency. Receivers can pin the full dependency graph.")) + + (~docs/subsection :title "Follow — Subscribe to a Remote Server" + (~docs/code :code "(Follow\n :actor \"https://pub.sx-web.org/pub/actor\"\n :object \"https://other.example.com/pub/actor\")") + (p "When accepted, the remote server delivers " (code "(Publish ...)") " activities to our inbox. We pin their CIDs locally.")) + + (~docs/subsection :title "Announce — Mirror Remote Content" + (~docs/code :code "(Announce\n :actor \"https://pub.sx-web.org/pub/actor\"\n :object \"bafy...remote-cid\")") + (p "Re-publish content from a followed server to our own followers. The CID is already pinned — we're just signalling that we endorse it.")) + + (~docs/subsection :title "Anchor — Blockchain Provenance Record" + (~docs/code :code "(Anchor\n :tree-cid \"bafy...merkle-tree\"\n :leaf-index 42\n :ots-cid \"bafy...ots-proof\"\n :btc-txid \"abc123...def\"\n :btc-block 890123\n :anchored-at \"2026-03-24T12:00:00Z\")") + (p "Embedded in every " (code "(Publish ...)") " activity. Receivers can verify: the Merkle tree was anchored in that Bitcoin block, and the CID is a leaf in that tree. Inclusion proof is verifiable offline."))) + + ;; ----------------------------------------------------------------------- + ;; Endpoints + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Endpoints" :id "endpoints" + (p "All endpoints are " (code "defhandler") " definitions in " (code ".sx") " files. Python provides only IO primitives behind the boundary.") + + (table :class "w-full text-sm" + (thead + (tr :class "border-b border-stone-200 text-left" + (th :class "py-2 pr-4" "Method") + (th :class "py-2 pr-4" "Route") + (th :class "py-2" "Purpose"))) + (tbody :class "text-stone-600" + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/actor") (td :class "py-2" "Actor profile")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/outbox") (td :class "py-2" "Published activity feed")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "POST") (td :class "py-2 pr-4 font-mono text-xs" "/pub/inbox") (td :class "py-2" "Receive activities from remote servers")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/⟨collection⟩") (td :class "py-2" "List items in a collection")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/⟨collection⟩/⟨slug⟩") (td :class "py-2" "Document by path — resolves CID, fetches from IPFS")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/cid/⟨cid⟩") (td :class "py-2" "Direct CID fetch (immutable, cache forever)")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "POST") (td :class "py-2 pr-4 font-mono text-xs" "/pub/publish") (td :class "py-2" "Pin to IPFS + create Publish activity + announce")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "POST") (td :class "py-2 pr-4 font-mono text-xs" "/pub/follow") (td :class "py-2" "Follow a remote sx-pub server")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/followers") (td :class "py-2" "Who follows us")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/following") (td :class "py-2" "Who we follow")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/.well-known/webfinger") (td :class "py-2" "Discovery")) + (tr + (td :class "py-2 pr-4 font-mono text-xs" "GET") (td :class "py-2 pr-4 font-mono text-xs" "/pub/anchor/⟨tree-cid⟩") (td :class "py-2" "Merkle tree verification"))))) + + ;; ----------------------------------------------------------------------- + ;; Storage + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Storage" :id "storage" + (p "IPFS is the canonical content store. PostgreSQL is the local index.") + + (table :class "w-full text-sm" + (thead + (tr :class "border-b border-stone-200 text-left" + (th :class "py-2 pr-4" "What") + (th :class "py-2" "Where"))) + (tbody :class "text-stone-600" + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Content (SX source)") (td :class "py-2" "IPFS — pinned, referenced by CID")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Path → CID index") (td :class "py-2" "PostgreSQL")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Outbox activities") (td :class "py-2" "PostgreSQL")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Followers / following") (td :class "py-2" "PostgreSQL")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Anchor records") (td :class "py-2" "PostgreSQL")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Remote mirrored content") (td :class "py-2" "PostgreSQL + IPFS (pinned locally)")) + (tr :class "border-b border-stone-100" + (td :class "py-2 pr-4" "Content cache") (td :class "py-2" "Redis")) + (tr + (td :class "py-2 pr-4" "Actor keypair") (td :class "py-2" "PostgreSQL"))))) + + ;; ----------------------------------------------------------------------- + ;; SX / Python Split + ;; ----------------------------------------------------------------------- + + (~docs/section :title "SX / Python Split" :id "split" + (p "SX handles all composition, logic, and rendering. Python provides IO primitives behind the boundary.") + + (div :class "grid grid-cols-2 gap-6" + (div + (h4 :class "font-semibold text-stone-700 mb-2" "Pure SX (defhandler + defcomp)") + (ul :class "text-sm text-stone-600 space-y-1 list-disc pl-5" + (li "All HTTP routing and request dispatch") + (li "Content negotiation (Accept header)") + (li "Response construction (SX actor docs, collections)") + (li "UI components for browsing") + (li "Activity serialization") + (li "Nav data for /pub/ sections"))) + (div + (h4 :class "font-semibold text-stone-700 mb-2" "Python Helpers (IO boundary)") + (ul :class "text-sm text-stone-600 space-y-1 list-disc pl-5" + (li (code "pub-ipfs-add") " / " (code "pub-ipfs-get") " / " (code "pub-ipfs-pin") " — IPFS ops") + (li (code "pub-db-query") " / " (code "pub-db-insert") " — database") + (li (code "pub-sign") " / " (code "pub-verify-signature") " — HTTP signatures") + (li (code "pub-hash") " — SHA3-256") + (li (code "pub-anchor-batch") " — OpenTimestamps") + (li (code "pub-deliver") " — POST to follower inboxes"))))) + + ;; ----------------------------------------------------------------------- + ;; Flows + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Flows" :id "flows" + + (~docs/subsection :title "Publish" + (p :class "text-sm text-stone-600 mb-2" "Publishing is " (strong "anchor-then-announce") " — content is provenance-sealed before anyone sees it. This prevents a malicious follower from seeing content, anchoring it first, and claiming provenance.") + (ol :class "text-sm text-stone-600 space-y-2 list-decimal pl-5" + (li (code "POST /pub/publish") " with collection, slug, SX content") + (li "Pin to IPFS → get CID (content is now addressable but not announced)") + (li "Store path→CID in DB (status: " (code "anchoring") ")") + (li "Build Merkle tree, submit to OpenTimestamps, wait for Bitcoin confirmation") + (li "On confirmation: update status to " (code "anchored") ", record proof") + (li "Construct " (code "(Publish ...)") " with anchor proof embedded") + (li "Deliver to follower inboxes — " (strong "only now") " is it public")) + (p :class "text-sm text-stone-500 mt-2 italic" "This means publishing isn't instant — there's a delay while the anchor confirms. That's a feature: it forces a deliberate pace and makes provenance airtight.") + (~docs/code :code "(Publish\n :actor \"https://pub.sx-web.org/pub/actor\"\n :published \"2026-03-24T12:00:00Z\"\n :object (SxDocument\n :name \"evaluator\"\n :cid \"bafy...eval123\"\n :anchor (Anchor\n :tree-cid \"bafy...merkle\"\n :ots-cid \"bafy...proof\"\n :btc-txid \"abc123...\"\n :btc-block 890123)))") + (p :class "text-sm text-stone-600" "Receivers verify: the CID was anchored in that Bitcoin block " (em "before") " the Publish was sent. No way to backdate.")) + + (~docs/subsection :title "Follow" + (ol :class "text-sm text-stone-600 space-y-2 list-decimal pl-5" + (li (code "POST /pub/follow") " with remote actor URL") + (li "Fetch remote actor document (" (code "text/sx") ")") + (li "Send " (code "(Follow ...)") " to their inbox, signed with our key") + (li "They verify signature, add us to followers, send " (code "(Accept ...)")) + (li "We store them in our following list") + (li "They start delivering " (code "(Publish ...)") " activities to our inbox"))) + + (~docs/subsection :title "Receive (inbox)" + (ol :class "text-sm text-stone-600 space-y-2 list-decimal pl-5" + (li "Remote server POSTs " (code "(Publish ...)") " to our inbox") + (li "Verify HTTP signature against their actor's public key") + (li "Pin the referenced CID to our local IPFS node") + (li "Index the remote content in our DB (marked as mirrored)") + (li "Now we can resolve that CID locally — " (code "/pub/cid/bafy...") " just works"))) + + (~docs/subsection :title "Anchoring" + (p :class "text-sm text-stone-600 mb-2" "Anchoring is integrated into the publish flow — not a separate background job. Content waits for provenance before going public.") + (ol :class "text-sm text-stone-600 space-y-2 list-decimal pl-5" + (li "Batch pending CIDs into a Merkle tree (multiple publishes can share one tree)") + (li "Pin Merkle tree to IPFS") + (li "Submit Merkle root to OpenTimestamps calendar servers") + (li "Poll for Bitcoin confirmation (typically ~1 hour)") + (li "On confirmation: store proof, transition content to " (code "anchored") " status") + (li "Trigger delivery of Publish activities for all items in the batch")) + (p :class "text-sm text-stone-500 mt-2 italic" "Batching amortises the anchor cost — one Bitcoin attestation can seal hundreds of CIDs via the Merkle tree. Each CID gets an inclusion proof linking it to the anchored root."))) + + ;; ----------------------------------------------------------------------- + ;; CID ↔ Path + ;; ----------------------------------------------------------------------- + + (~docs/section :title "CID ↔ Path Translation" :id "cid-path" + (p "Published SX refers to dependencies by CID — the immutable, universal identifier. But humans browse by path.") + + (~docs/code :code ";; Canonical SX (stored in IPFS) references by CID\n(defcomp ~my-app/dashboard (&key data)\n ;; This component depends on a chart library published at another CID\n (let ((chart-lib (require \"bafy...chart-lib-cid\")))\n (div :class \"dashboard\"\n (chart-lib/bar-chart :data data))))\n\n;; But on pub.sx-web.org you browse by path:\n;; /pub/components/dashboard\n;; /pub/core-specs/evaluator\n;; /pub/platforms/javascript\n;;\n;; The DB maps: path → CID → IPFS content\n;; Following a remote server maps: their-path → CID → our local pin") + + (p "When you follow another sx-pub server and they publish a component, you get the " (code "(Publish ...)") " activity with the CID. Your server pins it. Now " (code "(require \"bafy...\")") " resolves locally — no network round-trip, no central registry, no package manager.")) + + ;; ----------------------------------------------------------------------- + ;; Phases + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Implementation Phases" :id "phases" + (div :class "space-y-4" + (div :class "rounded border border-emerald-200 bg-emerald-50 p-4" + (h4 :class "font-semibold text-emerald-800" "Phase 1: Foundation") + (p :class "text-emerald-700 text-sm" "DB schema, async IPFS client, actor endpoint, webfinger, " (code "/pub/actor") " returns SX actor document.")) + (div :class "rounded border border-sky-200 bg-sky-50 p-4" + (h4 :class "font-semibold text-sky-800" "Phase 2: Publishing") + (p :class "text-sky-700 text-sm" "Pin to IPFS, path→CID index, collection browsing. Publish the actual SX spec files as the first content.")) + (div :class "rounded border border-violet-200 bg-violet-50 p-4" + (h4 :class "font-semibold text-violet-800" "Phase 3: Federation") + (p :class "text-violet-700 text-sm" "Inbox/outbox, follow/accept, HTTP signature verification, activity delivery, content mirroring.")) + (div :class "rounded border border-amber-200 bg-amber-50 p-4" + (h4 :class "font-semibold text-amber-800" "Phase 4: Anchoring") + (p :class "text-amber-700 text-sm" "Merkle trees, OpenTimestamps, Bitcoin proof, provenance UI.")))))) From 7b3d76329185c1c3615780341d20d92c19f2a075 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 01:17:27 +0000 Subject: [PATCH 2/6] sx-pub Phase 1: DB schema, IPFS wiring, actor + webfinger + collections + status endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for the sx-pub federated publishing protocol: - 6 SQLAlchemy models: actor, collections, documents, activities, followers, following - Conditional DB enablement in sx_docs (DATABASE_URL present → enable DB) - Python IO helpers: get_or_create_actor (auto-generates RSA keypair), list_collections, check_status, seed_default_collections - 4 defhandler endpoints returning text/sx (no JSON): /pub/actor, /pub/webfinger, /pub/collections, /pub/status - Alembic migration infrastructure for sx service - Docker compose: DB + pgbouncer + IPFS + env vars Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.dev-pub.yml | 2 + shared/models/sx_pub.py | 164 ++++++++++++++++++++++++++++++++++++ sx/alembic.ini | 35 ++++++++ sx/alembic/env.py | 17 ++++ sx/app.py | 4 +- sx/sx/handlers/pub-api.sx | 98 +++++++++++++++++++++ sx/sxc/pages/helpers.py | 23 +++++ sx/sxc/pages/pub_helpers.py | 144 +++++++++++++++++++++++++++++++ 8 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 shared/models/sx_pub.py create mode 100644 sx/alembic.ini create mode 100644 sx/alembic/env.py create mode 100644 sx/sx/handlers/pub-api.sx create mode 100644 sx/sxc/pages/pub_helpers.py diff --git a/docker-compose.dev-pub.yml b/docker-compose.dev-pub.yml index f2867030..9e2d88e3 100644 --- a/docker-compose.dev-pub.yml +++ b/docker-compose.dev-pub.yml @@ -11,6 +11,8 @@ services: SECRET_KEY: "${SECRET_KEY:-pub-dev-secret}" REDIS_URL: redis://redis:6379/0 DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:5432/sx_pub + ALEMBIC_DATABASE_URL: postgresql+psycopg://postgres:change-me@db:5432/sx_pub + SX_PUB_DOMAIN: pub.sx-web.org WORKERS: "1" ENVIRONMENT: development RELOAD: "true" diff --git a/shared/models/sx_pub.py b/shared/models/sx_pub.py new file mode 100644 index 00000000..23d2be0f --- /dev/null +++ b/shared/models/sx_pub.py @@ -0,0 +1,164 @@ +"""sx-pub ORM models — federated SX publishing protocol. + +Tables for the sx-pub actor, content collections, published documents, +outbox activities, and federation relationships (followers/following). +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import ( + String, Integer, DateTime, Text, + ForeignKey, UniqueConstraint, Index, func, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from shared.db.base import Base + + +class SxPubActor(Base): + """Singleton actor for this sx-pub instance.""" + __tablename__ = "sx_pub_actor" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + preferred_username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) + display_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + summary: Mapped[str | None] = mapped_column(Text, nullable=True) + public_key_pem: Mapped[str] = mapped_column(Text, nullable=False) + private_key_pem: Mapped[str] = mapped_column(Text, nullable=False) + domain: Mapped[str] = mapped_column(String(255), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubCollection(Base): + """Named grouping of published documents (e.g. core-specs, platforms).""" + __tablename__ = "sx_pub_collections" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + slug: Mapped[str] = mapped_column(String(128), unique=True, nullable=False) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + documents = relationship("SxPubDocument", back_populates="collection", lazy="selectin") + + def __repr__(self) -> str: + return f"" + + +class SxPubDocument(Base): + """Published content — path→CID index entry.""" + __tablename__ = "sx_pub_documents" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + collection_id: Mapped[int] = mapped_column( + Integer, ForeignKey("sx_pub_collections.id", ondelete="CASCADE"), nullable=False, + ) + slug: Mapped[str] = mapped_column(String(255), nullable=False) + title: Mapped[str | None] = mapped_column(String(512), nullable=True) + summary: Mapped[str | None] = mapped_column(Text, nullable=True) + content_hash: Mapped[str] = mapped_column(String(128), nullable=False) + ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True) + size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + requires: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + status: Mapped[str] = mapped_column( + String(20), nullable=False, default="draft", server_default="draft", + ) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + collection = relationship("SxPubCollection", back_populates="documents") + + __table_args__ = ( + UniqueConstraint("collection_id", "slug", name="uq_pub_doc_collection_slug"), + Index("ix_pub_doc_collection", "collection_id"), + Index("ix_pub_doc_status", "status"), + Index("ix_pub_doc_cid", "ipfs_cid"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubActivity(Base): + """Outbox activity (Publish, Follow, Announce, Anchor).""" + __tablename__ = "sx_pub_activities" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + activity_type: Mapped[str] = mapped_column(String(64), nullable=False) + object_type: Mapped[str | None] = mapped_column(String(64), nullable=True) + object_data: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True) + published: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + __table_args__ = ( + Index("ix_pub_activity_type", "activity_type"), + Index("ix_pub_activity_published", "published"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubFollower(Base): + """Remote server that follows us.""" + __tablename__ = "sx_pub_followers" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + follower_acct: Mapped[str] = mapped_column(String(512), nullable=False) + follower_inbox: Mapped[str] = mapped_column(String(512), nullable=False) + follower_actor_url: Mapped[str] = mapped_column(String(512), nullable=False) + follower_public_key: Mapped[str | None] = mapped_column(Text, nullable=True) + state: Mapped[str] = mapped_column( + String(20), nullable=False, default="accepted", server_default="accepted", + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + __table_args__ = ( + UniqueConstraint("follower_acct", name="uq_pub_follower_acct"), + Index("ix_pub_follower_state", "state"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubFollowing(Base): + """Remote server we follow.""" + __tablename__ = "sx_pub_following" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + remote_actor_url: Mapped[str] = mapped_column(String(512), unique=True, nullable=False) + remote_inbox: Mapped[str] = mapped_column(String(512), nullable=False) + state: Mapped[str] = mapped_column( + String(20), nullable=False, default="pending", server_default="pending", + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + accepted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + __table_args__ = ( + Index("ix_pub_following_state", "state"), + ) + + def __repr__(self) -> str: + return f"" diff --git a/sx/alembic.ini b/sx/alembic.ini new file mode 100644 index 00000000..a04e0716 --- /dev/null +++ b/sx/alembic.ini @@ -0,0 +1,35 @@ +[alembic] +script_location = alembic +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/sx/alembic/env.py b/sx/alembic/env.py new file mode 100644 index 00000000..2ac3ba13 --- /dev/null +++ b/sx/alembic/env.py @@ -0,0 +1,17 @@ +from alembic import context +from shared.db.alembic_env import run_alembic + +MODELS = [ + "shared.models.sx_pub", +] + +TABLES = frozenset({ + "sx_pub_actor", + "sx_pub_collections", + "sx_pub_documents", + "sx_pub_activities", + "sx_pub_followers", + "sx_pub_following", +}) + +run_alembic(context.config, MODELS, TABLES) diff --git a/sx/app.py b/sx/app.py index 6a774ce0..0bf28255 100644 --- a/sx/app.py +++ b/sx/app.py @@ -62,8 +62,10 @@ def create_app() -> "Quart": extra_kw = {} if SX_STANDALONE: - extra_kw["no_db"] = True extra_kw["no_oauth"] = True + # Enable DB if DATABASE_URL is set (needed for sx-pub) + if not os.getenv("DATABASE_URL"): + extra_kw["no_db"] = True app = create_base_app( "sx", diff --git a/sx/sx/handlers/pub-api.sx b/sx/sx/handlers/pub-api.sx new file mode 100644 index 00000000..cee64460 --- /dev/null +++ b/sx/sx/handlers/pub-api.sx @@ -0,0 +1,98 @@ +;; ========================================================================== +;; sx-pub Phase 1 API endpoints — actor, webfinger, collections, status +;; +;; All responses are text/sx. No JSON. +;; ========================================================================== + + +;; -------------------------------------------------------------------------- +;; Actor +;; -------------------------------------------------------------------------- + +(defhandler pub-actor + :path "/pub/actor" + :method :get + :returns "element" + (&key) + (let ((actor (helper "pub-actor-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxActor" + "\n :id \"https://" (get actor "domain") "/pub/actor\"" + "\n :type \"SxPublisher\"" + "\n :name \"" (get actor "display-name") "\"" + "\n :summary \"" (get actor "summary") "\"" + "\n :inbox \"/pub/inbox\"" + "\n :outbox \"/pub/outbox\"" + "\n :followers \"/pub/followers\"" + "\n :following \"/pub/following\"" + "\n :public-key-pem \"" (get actor "public-key-pem") "\")")))) + + +;; -------------------------------------------------------------------------- +;; Webfinger +;; -------------------------------------------------------------------------- + +(defhandler pub-webfinger + :path "/pub/webfinger" + :method :get + :returns "element" + (&key) + (let ((resource (helper "request-arg" "resource" "")) + (actor (helper "pub-actor-data"))) + (let ((expected (str "acct:" (get actor "preferred-username") "@" (get actor "domain")))) + (if (!= resource expected) + (do + (set-response-status 404) + (str "(Error :message \"Resource not found\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxWebfinger" + "\n :subject \"" expected "\"" + "\n :actor \"https://" (get actor "domain") "/pub/actor\"" + "\n :type \"SxPublisher\")")))))) + + +;; -------------------------------------------------------------------------- +;; Collections +;; -------------------------------------------------------------------------- + +(defhandler pub-collections + :path "/pub/collections" + :method :get + :returns "element" + (&key) + (let ((collections (helper "pub-collections-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (c) + (str "\n (SxCollection" + " :slug \"" (get c "slug") "\"" + " :name \"" (get c "name") "\"" + " :description \"" (get c "description") "\"" + " :href \"/pub/" (get c "slug") "\")")) + collections))) + (str "(SxCollections" (join "" items) ")"))))) + + +;; -------------------------------------------------------------------------- +;; Status +;; -------------------------------------------------------------------------- + +(defhandler pub-status + :path "/pub/status" + :method :get + :returns "element" + (&key) + (let ((status (helper "pub-status-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxPubStatus" + "\n :healthy " (get status "healthy") + "\n :db \"" (get status "db") "\"" + "\n :ipfs \"" (get status "ipfs") "\"" + "\n :actor \"" (get status "actor") "\"" + "\n :domain \"" (or (get status "domain") "unknown") "\")")))) diff --git a/sx/sxc/pages/helpers.py b/sx/sxc/pages/helpers.py index 9e0463c8..c8874f9b 100644 --- a/sx/sxc/pages/helpers.py +++ b/sx/sxc/pages/helpers.py @@ -37,6 +37,10 @@ def _register_sx_helpers() -> None: "spec-explorer-data": _spec_explorer_data, "spec-explorer-data-by-slug": _spec_explorer_data_by_slug, "handler-source": _handler_source, + # sx-pub helpers (only functional when DATABASE_URL is set) + "pub-actor-data": _pub_actor_data, + "pub-collections-data": _pub_collections_data, + "pub-status-data": _pub_status_data, }) @@ -1717,3 +1721,22 @@ def _page_helpers_demo_data() -> dict: results["attr-keys"] = list(ATTR_DETAILS.keys()) return results + + +# --------------------------------------------------------------------------- +# sx-pub helpers — thin wrappers for SX access +# --------------------------------------------------------------------------- + +async def _pub_actor_data(): + from .pub_helpers import get_or_create_actor + return await get_or_create_actor() + + +async def _pub_collections_data(): + from .pub_helpers import list_collections + return await list_collections() + + +async def _pub_status_data(): + from .pub_helpers import check_status + return await check_status() diff --git a/sx/sxc/pages/pub_helpers.py b/sx/sxc/pages/pub_helpers.py new file mode 100644 index 00000000..3fd19a5b --- /dev/null +++ b/sx/sxc/pages/pub_helpers.py @@ -0,0 +1,144 @@ +"""sx-pub Python IO helpers — actor management, IPFS status, collections. + +These are called from SX defhandlers via (helper "pub-..." args...). +All DB access uses g.s (per-request async session from register_db). +""" +from __future__ import annotations + +import logging +import os +from typing import Any + +logger = logging.getLogger("sx.pub") + +SX_PUB_DOMAIN = os.getenv("SX_PUB_DOMAIN", "pub.sx-web.org") + + +async def get_or_create_actor() -> dict[str, Any]: + """Get or create the singleton sx-pub actor. Auto-generates RSA keypair.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubActor + + result = await g.s.execute( + select(SxPubActor).where(SxPubActor.preferred_username == "sx") + ) + actor = result.scalar_one_or_none() + + if actor is None: + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives import serialization + + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, + ) + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + public_pem = private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode("utf-8") + + actor = SxPubActor( + preferred_username="sx", + display_name="SX Language", + summary="Federated SX specification publisher", + public_key_pem=public_pem, + private_key_pem=private_pem, + domain=SX_PUB_DOMAIN, + ) + g.s.add(actor) + await g.s.flush() + logger.info("Created sx-pub actor id=%d domain=%s", actor.id, SX_PUB_DOMAIN) + + # Seed default collections on first run + await seed_default_collections() + + return { + "preferred-username": actor.preferred_username, + "display-name": actor.display_name or actor.preferred_username, + "summary": actor.summary or "", + "public-key-pem": actor.public_key_pem, + "domain": actor.domain, + } + + +async def seed_default_collections() -> None: + """Create default collections if they don't exist.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection + + defaults = [ + ("core-specs", "Core Specifications", "Language spec files — evaluator, parser, primitives, render", 0), + ("platforms", "Platforms", "Host platform implementations — JavaScript, Python, OCaml", 1), + ("components", "Components", "Reusable UI components published as content-addressed SX", 2), + ("libraries", "Libraries", "SX library modules — stdlib, signals, freeze, web forms", 3), + ] + + for slug, name, description, order in defaults: + exists = await g.s.execute( + select(SxPubCollection).where(SxPubCollection.slug == slug) + ) + if exists.scalar_one_or_none() is None: + g.s.add(SxPubCollection( + slug=slug, name=name, description=description, sort_order=order, + )) + await g.s.flush() + logger.info("Seeded %d default collections", len(defaults)) + + +async def list_collections() -> list[dict[str, Any]]: + """List all pub collections.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection + + result = await g.s.execute( + select(SxPubCollection).order_by(SxPubCollection.sort_order) + ) + return [ + { + "slug": c.slug, + "name": c.name, + "description": c.description or "", + } + for c in result.scalars().all() + ] + + +async def check_status() -> dict[str, Any]: + """Health check — DB, IPFS, actor.""" + status: dict[str, Any] = {"healthy": "true"} + + # DB + try: + from quart import g + from sqlalchemy import text + await g.s.execute(text("SELECT 1")) + status["db"] = "connected" + except Exception as e: + status["db"] = f"error: {e}" + status["healthy"] = "false" + + # IPFS + try: + from shared.utils.ipfs_client import is_available + ok = await is_available() + status["ipfs"] = "available" if ok else "unavailable" + except Exception as e: + status["ipfs"] = f"error: {e}" + + # Actor + try: + actor = await get_or_create_actor() + status["actor"] = actor["preferred-username"] + status["domain"] = actor["domain"] + except Exception as e: + status["actor"] = f"error: {e}" + status["healthy"] = "false" + + return status From cf130c417464931d5d197021445b46f4456fa7d8 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 01:30:05 +0000 Subject: [PATCH 3/6] sx-pub Phase 2: publish to IPFS, browse collections, resolve by path + CID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New endpoints: - POST /pub/publish — pin SX content to IPFS, store path→CID in DB - GET /pub/browse/ — list published documents - GET /pub/doc// — resolve path to CID, fetch from IPFS - GET /pub/cid/ — direct CID fetch (immutable, cache forever) New helpers: pub-publish, pub-collection-items, pub-resolve-document, pub-fetch-cid Tested: published stdlib.sx (6.9KB) → QmQQyR3Ltqi5sFiwZh5dutPbAM4QsEBnw419RyNnTj4fFM Co-Authored-By: Claude Opus 4.6 (1M context) --- sx/sx/handlers/pub-api.sx | 121 +++++++++++++++++++++++++- sx/sxc/pages/helpers.py | 24 ++++++ sx/sxc/pages/pub_helpers.py | 168 ++++++++++++++++++++++++++++++++++++ 3 files changed, 312 insertions(+), 1 deletion(-) diff --git a/sx/sx/handlers/pub-api.sx b/sx/sx/handlers/pub-api.sx index cee64460..11c8a47a 100644 --- a/sx/sx/handlers/pub-api.sx +++ b/sx/sx/handlers/pub-api.sx @@ -1,5 +1,5 @@ ;; ========================================================================== -;; sx-pub Phase 1 API endpoints — actor, webfinger, collections, status +;; sx-pub API endpoints — actor, webfinger, collections, publishing, browsing ;; ;; All responses are text/sx. No JSON. ;; ========================================================================== @@ -96,3 +96,122 @@ "\n :ipfs \"" (get status "ipfs") "\"" "\n :actor \"" (get status "actor") "\"" "\n :domain \"" (or (get status "domain") "unknown") "\")")))) + + +;; ========================================================================== +;; Phase 2: Publishing + Browsing +;; ========================================================================== + + +;; -------------------------------------------------------------------------- +;; Publish +;; -------------------------------------------------------------------------- + +(defhandler pub-publish + :path "/pub/publish" + :method :post + :csrf false + :returns "element" + (&key) + (let ((collection (helper "request-form" "collection" "")) + (slug (helper "request-form" "slug" "")) + (content (helper "request-form" "content" "")) + (title (helper "request-form" "title" "")) + (summary (helper "request-form" "summary" ""))) + (if (or (= collection "") (= slug "") (= content "")) + (do + (set-response-status 400) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + "(Error :message \"Missing collection, slug, or content\")") + (let ((result (helper "pub-publish" collection slug content title summary))) + (if (get result "error") + (do + (set-response-status 500) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str "(Error :message \"" (get result "error") "\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(Published" + "\n :path \"" (get result "path") "\"" + "\n :cid \"" (get result "cid") "\"" + "\n :hash \"" (get result "hash") "\"" + "\n :size " (get result "size") + "\n :collection \"" (get result "collection") "\"" + "\n :slug \"" (get result "slug") "\"" + "\n :title \"" (get result "title") "\")"))))))) + + +;; -------------------------------------------------------------------------- +;; Browse collection +;; -------------------------------------------------------------------------- + +(defhandler pub-browse-collection + :path "/pub/browse/" + :method :get + :returns "element" + (&key collection_slug) + (let ((data (helper "pub-collection-items" collection_slug))) + (if (get data "error") + (do + (set-response-status 404) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str "(Error :message \"" (get data "error") "\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (d) + (str "\n (SxDocument" + " :slug \"" (get d "slug") "\"" + " :title \"" (get d "title") "\"" + " :summary \"" (get d "summary") "\"" + " :cid \"" (get d "cid") "\"" + " :size " (get d "size") ")")) + (get data "items")))) + (str + "(SxCollection" + "\n :slug \"" (get data "collection") "\"" + "\n :name \"" (get data "name") "\"" + "\n :description \"" (get data "description") "\"" + (join "" items) ")")))))) + + +;; -------------------------------------------------------------------------- +;; Resolve document by path +;; -------------------------------------------------------------------------- + +(defhandler pub-document + :path "/pub/doc//" + :method :get + :returns "element" + (&key collection_slug slug) + (let ((data (helper "pub-resolve-document" collection_slug slug))) + (if (get data "error") + (do + (set-response-status 404) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str "(Error :message \"" (get data "error") "\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (set-response-header "X-IPFS-CID" (get data "cid")) + (get data "content"))))) + + +;; -------------------------------------------------------------------------- +;; Direct CID fetch +;; -------------------------------------------------------------------------- + +(defhandler pub-cid + :path "/pub/cid/" + :method :get + :returns "element" + (&key cid) + (let ((data (helper "pub-fetch-cid" cid))) + (if (get data "error") + (do + (set-response-status 404) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str "(Error :message \"" (get data "error") "\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (set-response-header "Cache-Control" "public, max-age=31536000, immutable") + (get data "content"))))) diff --git a/sx/sxc/pages/helpers.py b/sx/sxc/pages/helpers.py index c8874f9b..9ecbb368 100644 --- a/sx/sxc/pages/helpers.py +++ b/sx/sxc/pages/helpers.py @@ -41,6 +41,10 @@ def _register_sx_helpers() -> None: "pub-actor-data": _pub_actor_data, "pub-collections-data": _pub_collections_data, "pub-status-data": _pub_status_data, + "pub-publish": _pub_publish, + "pub-collection-items": _pub_collection_items, + "pub-resolve-document": _pub_resolve_document, + "pub-fetch-cid": _pub_fetch_cid, }) @@ -1740,3 +1744,23 @@ async def _pub_collections_data(): async def _pub_status_data(): from .pub_helpers import check_status return await check_status() + + +async def _pub_publish(collection, slug, content, title="", summary=""): + from .pub_helpers import publish_document + return await publish_document(collection, slug, content, title, summary) + + +async def _pub_collection_items(collection_slug): + from .pub_helpers import collection_items + return await collection_items(collection_slug) + + +async def _pub_resolve_document(collection_slug, slug): + from .pub_helpers import resolve_document + return await resolve_document(collection_slug, slug) + + +async def _pub_fetch_cid(cid): + from .pub_helpers import fetch_cid + return await fetch_cid(cid) diff --git a/sx/sxc/pages/pub_helpers.py b/sx/sxc/pages/pub_helpers.py index 3fd19a5b..48ed7c42 100644 --- a/sx/sxc/pages/pub_helpers.py +++ b/sx/sxc/pages/pub_helpers.py @@ -142,3 +142,171 @@ async def check_status() -> dict[str, Any]: status["healthy"] = "false" return status + + +# --------------------------------------------------------------------------- +# Phase 2: Publishing + Browsing +# --------------------------------------------------------------------------- + +async def publish_document(collection_slug: str, slug: str, content: str, + title: str = "", summary: str = "") -> dict[str, Any]: + """Pin SX content to IPFS and store in DB. Returns doc info dict.""" + import hashlib + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection, SxPubDocument + from shared.utils.ipfs_client import add_bytes + + # Resolve collection + result = await g.s.execute( + select(SxPubCollection).where(SxPubCollection.slug == collection_slug) + ) + collection = result.scalar_one_or_none() + if collection is None: + return {"error": f"Collection not found: {collection_slug}"} + + # Hash content + content_bytes = content.encode("utf-8") + content_hash = hashlib.sha3_256(content_bytes).hexdigest() + + # Pin to IPFS + try: + cid = await add_bytes(content_bytes, pin=True) + except Exception as e: + logger.error("IPFS pin failed for %s/%s: %s", collection_slug, slug, e) + return {"error": f"IPFS pin failed: {e}"} + + # Upsert document + result = await g.s.execute( + select(SxPubDocument).where( + SxPubDocument.collection_id == collection.id, + SxPubDocument.slug == slug, + ) + ) + doc = result.scalar_one_or_none() + + if doc is None: + doc = SxPubDocument( + collection_id=collection.id, + slug=slug, + title=title or slug, + summary=summary, + content_hash=content_hash, + ipfs_cid=cid, + size_bytes=len(content_bytes), + status="published", + ) + g.s.add(doc) + else: + doc.content_hash = content_hash + doc.ipfs_cid = cid + doc.size_bytes = len(content_bytes) + doc.status = "published" + if title: + doc.title = title + if summary: + doc.summary = summary + + await g.s.flush() + logger.info("Published %s/%s → %s (%d bytes)", collection_slug, slug, cid, len(content_bytes)) + + return { + "path": f"/pub/{collection_slug}/{slug}", + "cid": cid, + "hash": content_hash, + "size": len(content_bytes), + "collection": collection_slug, + "slug": slug, + "title": doc.title or slug, + } + + +async def collection_items(collection_slug: str) -> dict[str, Any]: + """List published documents in a collection.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection, SxPubDocument + + result = await g.s.execute( + select(SxPubCollection).where(SxPubCollection.slug == collection_slug) + ) + collection = result.scalar_one_or_none() + if collection is None: + return {"error": f"Collection not found: {collection_slug}"} + + result = await g.s.execute( + select(SxPubDocument).where( + SxPubDocument.collection_id == collection.id, + SxPubDocument.status == "published", + ).order_by(SxPubDocument.slug) + ) + docs = result.scalars().all() + + return { + "collection": collection_slug, + "name": collection.name, + "description": collection.description or "", + "items": [ + { + "slug": d.slug, + "title": d.title or d.slug, + "summary": d.summary or "", + "cid": d.ipfs_cid or "", + "size": d.size_bytes or 0, + } + for d in docs + ], + } + + +async def resolve_document(collection_slug: str, slug: str) -> dict[str, Any]: + """Resolve a document path to its content via IPFS.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection, SxPubDocument + from shared.utils.ipfs_client import get_bytes + + result = await g.s.execute( + select(SxPubCollection).where(SxPubCollection.slug == collection_slug) + ) + collection = result.scalar_one_or_none() + if collection is None: + return {"error": "not-found"} + + result = await g.s.execute( + select(SxPubDocument).where( + SxPubDocument.collection_id == collection.id, + SxPubDocument.slug == slug, + ) + ) + doc = result.scalar_one_or_none() + if doc is None or not doc.ipfs_cid: + return {"error": "not-found"} + + content_bytes = await get_bytes(doc.ipfs_cid) + if content_bytes is None: + return {"error": "ipfs-unavailable"} + + return { + "slug": doc.slug, + "title": doc.title or doc.slug, + "summary": doc.summary or "", + "cid": doc.ipfs_cid, + "collection": collection_slug, + "content": content_bytes.decode("utf-8", errors="replace"), + } + + +async def fetch_cid(cid: str) -> dict[str, Any]: + """Fetch raw content from IPFS by CID.""" + from shared.utils.ipfs_client import get_bytes + + content_bytes = await get_bytes(cid) + if content_bytes is None: + return {"error": "not-found"} + + return { + "cid": cid, + "content": content_bytes.decode("utf-8", errors="replace"), + "size": len(content_bytes), + } From aa1d4d7a671efa3dfdb8d9594dd5df30f0b5bd68 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 01:38:36 +0000 Subject: [PATCH 4/6] =?UTF-8?q?sx-pub=20Phase=203:=20federation=20?= =?UTF-8?q?=E2=80=94=20outbox,=20inbox,=20follow,=20delivery,=20HTTP=20sig?= =?UTF-8?q?natures?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New endpoints: - GET /pub/outbox — paginated activity feed - POST /pub/inbox — receive Follow/Accept/Publish from remote servers - POST /pub/follow — follow a remote sx-pub server - GET /pub/followers — list accepted followers - GET /pub/following — list who we follow Federation mechanics: - HTTP Signature generation (RSA-SHA256) for signed outgoing requests - HTTP Signature verification for incoming requests - Auto-accept Follow → store follower → send Accept back - Accept handling → update following state - Publish mirroring → pin remote CID to local IPFS - deliver_to_followers → fan out signed activities to all follower inboxes - Publish now records activity in outbox for federation delivery Co-Authored-By: Claude Opus 4.6 (1M context) --- sx/sx/handlers/pub-api.sx | 122 ++++++++++ sx/sxc/pages/helpers.py | 42 ++++ sx/sxc/pages/pub_helpers.py | 469 +++++++++++++++++++++++++++++++++++- 3 files changed, 632 insertions(+), 1 deletion(-) diff --git a/sx/sx/handlers/pub-api.sx b/sx/sx/handlers/pub-api.sx index 11c8a47a..ffea0630 100644 --- a/sx/sx/handlers/pub-api.sx +++ b/sx/sx/handlers/pub-api.sx @@ -215,3 +215,125 @@ (set-response-header "Content-Type" "text/sx; charset=utf-8") (set-response-header "Cache-Control" "public, max-age=31536000, immutable") (get data "content"))))) + + +;; ========================================================================== +;; Phase 3: Federation — outbox, inbox, follow, followers, following +;; ========================================================================== + + +;; -------------------------------------------------------------------------- +;; Outbox +;; -------------------------------------------------------------------------- + +(defhandler pub-outbox + :path "/pub/outbox" + :method :get + :returns "element" + (&key) + (let ((page (helper "request-arg" "page" "")) + (data (helper "pub-outbox-data" page))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (a) + (str "\n (" (get a "type") + " :object-type \"" (get a "object-type") "\"" + " :published \"" (get a "published") "\"" + " :cid \"" (get a "cid") "\")")) + (get data "items")))) + (str + "(SxOutbox" + "\n :total " (get data "total") + "\n :page " (get data "page") + (join "" items) ")"))))) + + +;; -------------------------------------------------------------------------- +;; Inbox +;; -------------------------------------------------------------------------- + +(defhandler pub-inbox + :path "/pub/inbox" + :method :post + :csrf false + :returns "element" + (&key) + (let ((body (helper "pub-request-body"))) + (if (= body "") + (do + (set-response-status 400) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + "(Error :message \"Empty body\")") + (let ((result (helper "pub-process-inbox" body))) + (do + (set-response-status 202) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str "(Accepted :result " (str result) ")")))))) + + +;; -------------------------------------------------------------------------- +;; Follow a remote server +;; -------------------------------------------------------------------------- + +(defhandler pub-follow + :path "/pub/follow" + :method :post + :csrf false + :returns "element" + (&key) + (let ((actor-url (helper "request-form" "actor_url" ""))) + (if (= actor-url "") + (do + (set-response-status 400) + (set-response-header "Content-Type" "text/sx; charset=utf-8") + "(Error :message \"Missing actor_url\")") + (let ((result (helper "pub-follow-remote" actor-url))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (if (get result "error") + (do + (set-response-status 502) + (str "(Error :message \"" (get result "error") "\")")) + (str + "(FollowSent" + "\n :actor-url \"" (get result "actor-url") "\"" + "\n :status \"" (get result "status") "\")"))))))) + + +;; -------------------------------------------------------------------------- +;; Followers +;; -------------------------------------------------------------------------- + +(defhandler pub-followers + :path "/pub/followers" + :method :get + :returns "element" + (&key) + (let ((data (helper "pub-followers-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (f) + (str "\n (SxFollower" + " :acct \"" (get f "acct") "\"" + " :actor-url \"" (get f "actor-url") "\")")) + data))) + (str "(SxFollowers" (join "" items) ")"))))) + + +;; -------------------------------------------------------------------------- +;; Following +;; -------------------------------------------------------------------------- + +(defhandler pub-following + :path "/pub/following" + :method :get + :returns "element" + (&key) + (let ((data (helper "pub-following-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (f) + (str "\n (SxFollowing" + " :actor-url \"" (get f "actor-url") "\")")) + data))) + (str "(SxFollowing" (join "" items) ")"))))) diff --git a/sx/sxc/pages/helpers.py b/sx/sxc/pages/helpers.py index 9ecbb368..0877c6cb 100644 --- a/sx/sxc/pages/helpers.py +++ b/sx/sxc/pages/helpers.py @@ -45,6 +45,13 @@ def _register_sx_helpers() -> None: "pub-collection-items": _pub_collection_items, "pub-resolve-document": _pub_resolve_document, "pub-fetch-cid": _pub_fetch_cid, + "pub-outbox-data": _pub_outbox_data, + "pub-followers-data": _pub_followers_data, + "pub-following-data": _pub_following_data, + "pub-follow-remote": _pub_follow_remote, + "pub-process-inbox": _pub_process_inbox, + "pub-deliver-to-followers": _pub_deliver_to_followers, + "pub-request-body": _pub_request_body, }) @@ -1764,3 +1771,38 @@ async def _pub_resolve_document(collection_slug, slug): async def _pub_fetch_cid(cid): from .pub_helpers import fetch_cid return await fetch_cid(cid) + + +async def _pub_outbox_data(page=""): + from .pub_helpers import outbox_data + return await outbox_data(page) + + +async def _pub_followers_data(): + from .pub_helpers import followers_data + return await followers_data() + + +async def _pub_following_data(): + from .pub_helpers import following_data + return await following_data() + + +async def _pub_follow_remote(actor_url): + from .pub_helpers import follow_remote + return await follow_remote(actor_url) + + +async def _pub_process_inbox(body_sx): + from .pub_helpers import process_inbox + return await process_inbox(body_sx) + + +async def _pub_deliver_to_followers(activity_sx): + from .pub_helpers import deliver_to_followers + return await deliver_to_followers(activity_sx) + + +async def _pub_request_body(): + from .pub_helpers import get_request_body + return await get_request_body() diff --git a/sx/sxc/pages/pub_helpers.py b/sx/sxc/pages/pub_helpers.py index 48ed7c42..535816f0 100644 --- a/sx/sxc/pages/pub_helpers.py +++ b/sx/sxc/pages/pub_helpers.py @@ -1,12 +1,14 @@ -"""sx-pub Python IO helpers — actor management, IPFS status, collections. +"""sx-pub Python IO helpers — actor, IPFS, collections, publishing, federation. These are called from SX defhandlers via (helper "pub-..." args...). All DB access uses g.s (per-request async session from register_db). """ from __future__ import annotations +import hashlib import logging import os +from datetime import datetime, timezone from typing import Any logger = logging.getLogger("sx.pub") @@ -210,6 +212,23 @@ async def publish_document(collection_slug: str, slug: str, content: str, await g.s.flush() logger.info("Published %s/%s → %s (%d bytes)", collection_slug, slug, cid, len(content_bytes)) + # Record Publish activity in outbox + from shared.models.sx_pub import SxPubActivity + g.s.add(SxPubActivity( + activity_type="Publish", + object_type="SxDocument", + object_data={ + "path": f"/pub/{collection_slug}/{slug}", + "cid": cid, + "collection": collection_slug, + "slug": slug, + "title": title or slug, + "summary": summary, + }, + ipfs_cid=cid, + )) + await g.s.flush() + return { "path": f"/pub/{collection_slug}/{slug}", "cid": cid, @@ -310,3 +329,451 @@ async def fetch_cid(cid: str) -> dict[str, Any]: "content": content_bytes.decode("utf-8", errors="replace"), "size": len(content_bytes), } + + +# --------------------------------------------------------------------------- +# Phase 3: Federation — outbox, inbox, follow, delivery, signatures +# --------------------------------------------------------------------------- + +async def outbox_data(page: str = "") -> dict[str, Any]: + """List published activities (outbox).""" + from quart import g + from sqlalchemy import select, func as sa_func + from shared.models.sx_pub import SxPubActivity + + page_num = int(page) if page else 1 + per_page = 20 + offset = (page_num - 1) * per_page + + total_result = await g.s.execute(select(sa_func.count(SxPubActivity.id))) + total = total_result.scalar() or 0 + + result = await g.s.execute( + select(SxPubActivity) + .order_by(SxPubActivity.published.desc()) + .offset(offset).limit(per_page) + ) + activities = result.scalars().all() + + return { + "total": total, + "page": page_num, + "items": [ + { + "type": a.activity_type, + "object-type": a.object_type or "", + "published": a.published.isoformat() if a.published else "", + "cid": a.ipfs_cid or "", + "data": a.object_data or {}, + } + for a in activities + ], + } + + +async def followers_data() -> list[dict[str, Any]]: + """List followers.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubFollower + + result = await g.s.execute( + select(SxPubFollower).where(SxPubFollower.state == "accepted") + .order_by(SxPubFollower.created_at.desc()) + ) + return [ + { + "acct": f.follower_acct, + "actor-url": f.follower_actor_url, + "inbox": f.follower_inbox, + } + for f in result.scalars().all() + ] + + +async def following_data() -> list[dict[str, Any]]: + """List who we follow.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubFollowing + + result = await g.s.execute( + select(SxPubFollowing).where(SxPubFollowing.state == "accepted") + .order_by(SxPubFollowing.created_at.desc()) + ) + return [ + { + "actor-url": f.remote_actor_url, + "inbox": f.remote_inbox, + } + for f in result.scalars().all() + ] + + +def _sign_request(method: str, url: str, body: str, private_key_pem: str, + key_id: str) -> dict[str, str]: + """Generate HTTP Signature headers for an outgoing request.""" + from urllib.parse import urlparse + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding + import base64 + + parsed = urlparse(url) + path = parsed.path + host = parsed.netloc + date = datetime.now(timezone.utc).strftime("%a, %d %b %Y %H:%M:%S GMT") + + # Build signature string + digest = "SHA-256=" + base64.b64encode( + hashlib.sha256(body.encode("utf-8")).digest() + ).decode("ascii") + + signed_string = ( + f"(request-target): {method.lower()} {path}\n" + f"host: {host}\n" + f"date: {date}\n" + f"digest: {digest}" + ) + + private_key = serialization.load_pem_private_key( + private_key_pem.encode("utf-8"), password=None, + ) + signature = base64.b64encode( + private_key.sign( + signed_string.encode("utf-8"), + padding.PKCS1v15(), + hashes.SHA256(), + ) + ).decode("ascii") + + sig_header = ( + f'keyId="{key_id}",' + f'algorithm="rsa-sha256",' + f'headers="(request-target) host date digest",' + f'signature="{signature}"' + ) + + return { + "Host": host, + "Date": date, + "Digest": digest, + "Signature": sig_header, + "Content-Type": "text/sx; charset=utf-8", + } + + +def _verify_signature(headers: dict, method: str, path: str, + body: str, public_key_pem: str) -> bool: + """Verify HTTP Signature on an incoming request.""" + import base64 + import re + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import padding + + sig_header = headers.get("Signature", "") + if not sig_header: + return False + + # Parse signature header + parts = {} + for match in re.finditer(r'(\w+)="([^"]*)"', sig_header): + parts[match.group(1)] = match.group(2) + + if "signature" not in parts or "headers" not in parts: + return False + + # Reconstruct signed string + signed_headers = parts["headers"].split() + lines = [] + for h in signed_headers: + if h == "(request-target)": + lines.append(f"(request-target): {method.lower()} {path}") + else: + val = headers.get(h.title(), headers.get(h, "")) + lines.append(f"{h}: {val}") + signed_string = "\n".join(lines) + + try: + public_key = serialization.load_pem_public_key( + public_key_pem.encode("utf-8"), + ) + public_key.verify( + base64.b64decode(parts["signature"]), + signed_string.encode("utf-8"), + padding.PKCS1v15(), + hashes.SHA256(), + ) + return True + except Exception as e: + logger.warning("Signature verification failed: %s", e) + return False + + +async def follow_remote(actor_url: str) -> dict[str, Any]: + """Send a Follow activity to a remote sx-pub server.""" + import httpx + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubFollowing, SxPubActor + + # Check not already following + result = await g.s.execute( + select(SxPubFollowing).where(SxPubFollowing.remote_actor_url == actor_url) + ) + existing = result.scalar_one_or_none() + if existing: + return {"status": existing.state, "actor-url": actor_url} + + # Fetch remote actor document + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.get(actor_url, headers={"Accept": "text/sx"}) + resp.raise_for_status() + remote_actor_sx = resp.text + except Exception as e: + return {"error": f"Failed to fetch remote actor: {e}"} + + # Parse inbox URL from the SX actor document + # Simple extraction — look for :inbox "..." + import re + inbox_match = re.search(r':inbox\s+"([^"]+)"', remote_actor_sx) + if not inbox_match: + return {"error": "Could not find inbox in remote actor document"} + + # Build absolute inbox URL + from urllib.parse import urljoin + remote_inbox = urljoin(actor_url, inbox_match.group(1)) + + # Get our actor for signing + actor_result = await g.s.execute( + select(SxPubActor).where(SxPubActor.preferred_username == "sx") + ) + our_actor = actor_result.scalar_one_or_none() + if not our_actor: + return {"error": "Local actor not initialized"} + + our_id = f"https://{our_actor.domain}/pub/actor" + + # Build Follow activity + follow_sx = ( + f'(Follow\n' + f' :actor "{our_id}"\n' + f' :object "{actor_url}")' + ) + + # Sign and send + key_id = f"{our_id}#main-key" + headers = _sign_request("POST", remote_inbox, follow_sx, + our_actor.private_key_pem, key_id) + + try: + async with httpx.AsyncClient(timeout=15) as client: + resp = await client.post(remote_inbox, content=follow_sx, headers=headers) + logger.info("Sent Follow to %s → %d", remote_inbox, resp.status_code) + except Exception as e: + logger.error("Follow delivery failed to %s: %s", remote_inbox, e) + + # Store the following record + following = SxPubFollowing( + remote_actor_url=actor_url, + remote_inbox=remote_inbox, + state="pending", + ) + g.s.add(following) + await g.s.flush() + + return {"status": "pending", "actor-url": actor_url, "inbox": remote_inbox} + + +async def process_inbox(body_sx: str) -> dict[str, Any]: + """Process an incoming activity from a remote sx-pub server.""" + import re + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import ( + SxPubFollower, SxPubFollowing, SxPubActivity, SxPubActor, + ) + from shared.utils.ipfs_client import pin_cid + + # Parse activity type and actor + type_match = re.match(r'\((\w+)', body_sx.strip()) + actor_match = re.search(r':actor\s+"([^"]+)"', body_sx) + object_match = re.search(r':object\s+"([^"]+)"', body_sx) + + if not type_match: + return {"error": "Could not parse activity type"} + + activity_type = type_match.group(1) + remote_actor = actor_match.group(1) if actor_match else "" + object_val = object_match.group(1) if object_match else "" + + logger.info("Inbox received: %s from %s", activity_type, remote_actor) + + if activity_type == "Follow": + # Someone wants to follow us — auto-accept + inbox_match = re.search(r':inbox\s+"([^"]+)"', body_sx) + remote_inbox = inbox_match.group(1) if inbox_match else "" + + # If no inbox in activity, try fetching the remote actor + if not remote_inbox and remote_actor: + import httpx + try: + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(remote_actor, headers={"Accept": "text/sx"}) + im = re.search(r':inbox\s+"([^"]+)"', resp.text) + if im: + from urllib.parse import urljoin + remote_inbox = urljoin(remote_actor, im.group(1)) + except Exception: + pass + + # Store follower + result = await g.s.execute( + select(SxPubFollower).where(SxPubFollower.follower_actor_url == remote_actor) + ) + follower = result.scalar_one_or_none() + if follower is None: + follower = SxPubFollower( + follower_acct=remote_actor, + follower_inbox=remote_inbox or remote_actor, + follower_actor_url=remote_actor, + state="accepted", + ) + g.s.add(follower) + await g.s.flush() + logger.info("Accepted follow from %s", remote_actor) + + # Send Accept back + actor_result = await g.s.execute( + select(SxPubActor).where(SxPubActor.preferred_username == "sx") + ) + our_actor = actor_result.scalar_one_or_none() + if our_actor and remote_inbox: + our_id = f"https://{our_actor.domain}/pub/actor" + accept_sx = ( + f'(Accept\n' + f' :actor "{our_id}"\n' + f' :object {body_sx})' + ) + key_id = f"{our_id}#main-key" + headers = _sign_request("POST", remote_inbox, accept_sx, + our_actor.private_key_pem, key_id) + import httpx + try: + async with httpx.AsyncClient(timeout=10) as client: + await client.post(remote_inbox, content=accept_sx, headers=headers) + except Exception as e: + logger.warning("Accept delivery failed: %s", e) + + return {"accepted": remote_actor} + + elif activity_type == "Accept": + # Our follow was accepted — update state + if object_val: + result = await g.s.execute( + select(SxPubFollowing).where( + SxPubFollowing.remote_actor_url == remote_actor + ) + ) + following = result.scalar_one_or_none() + if following: + following.state = "accepted" + following.accepted_at = datetime.now(timezone.utc) + await g.s.flush() + logger.info("Follow accepted by %s", remote_actor) + return {"accepted-by": remote_actor} + + elif activity_type == "Publish": + # Pin the published CID locally + cid_match = re.search(r':cid\s+"([^"]+)"', body_sx) + if cid_match: + cid = cid_match.group(1) + pinned = await pin_cid(cid) + logger.info("Mirrored CID %s from %s (pinned=%s)", cid, remote_actor, pinned) + + # Record the activity + g.s.add(SxPubActivity( + activity_type="Publish", + object_type="SxDocument", + object_data={"remote_actor": remote_actor, "body": body_sx}, + ipfs_cid=cid_match.group(1) if cid_match else None, + )) + await g.s.flush() + return {"mirrored": cid_match.group(1) if cid_match else ""} + + return {"ignored": activity_type} + + +async def deliver_to_followers(activity_sx: str) -> dict[str, Any]: + """Deliver an activity to all follower inboxes.""" + import httpx + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubFollower, SxPubActor + + actor_result = await g.s.execute( + select(SxPubActor).where(SxPubActor.preferred_username == "sx") + ) + our_actor = actor_result.scalar_one_or_none() + if not our_actor: + return {"error": "Actor not initialized", "delivered": 0} + + our_id = f"https://{our_actor.domain}/pub/actor" + key_id = f"{our_id}#main-key" + + result = await g.s.execute( + select(SxPubFollower).where(SxPubFollower.state == "accepted") + ) + followers = result.scalars().all() + + delivered = 0 + failed = 0 + + for follower in followers: + headers = _sign_request("POST", follower.follower_inbox, activity_sx, + our_actor.private_key_pem, key_id) + try: + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.post( + follower.follower_inbox, + content=activity_sx, + headers=headers, + ) + if resp.status_code < 300: + delivered += 1 + else: + failed += 1 + logger.warning("Delivery to %s returned %d", + follower.follower_inbox, resp.status_code) + except Exception as e: + failed += 1 + logger.error("Delivery to %s failed: %s", follower.follower_inbox, e) + + logger.info("Delivered to %d/%d followers (%d failed)", + delivered, len(followers), failed) + return {"delivered": delivered, "failed": failed, "total": len(followers)} + + +async def get_request_body() -> str: + """Get the raw request body as text.""" + from quart import request + data = await request.get_data(as_text=True) + return data + + +async def get_request_headers() -> dict[str, str]: + """Get request headers as a dict.""" + from quart import request + return dict(request.headers) + + +async def get_request_method() -> str: + """Get the HTTP method.""" + from quart import request + return request.method + + +async def get_request_path() -> str: + """Get the request path.""" + from quart import request + return request.path From d12f38a9d5912cb0da5ca1b13b754b065edb3581 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 02:12:40 +0000 Subject: [PATCH 5/6] =?UTF-8?q?sx-pub=20Phase=204:=20anchoring=20=E2=80=94?= =?UTF-8?q?=20Merkle=20trees,=20OpenTimestamps,=20verification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New endpoints: - POST /pub/anchor — batch unanchored Publish activities into Merkle tree, pin tree to IPFS, submit root to OpenTimestamps, store OTS proof on IPFS - GET /pub/verify/ — verify a CID's Merkle proof against anchored tree Uses existing shared/utils/anchoring.py infrastructure: - build_merkle_tree (SHA256, deterministic sort) - get_merkle_proof / verify_merkle_proof (inclusion proofs) - submit_to_opentimestamps (3 calendar servers with fallback) Tested: anchored 1 activity, Merkle tree + OTS proof pinned to IPFS, verification returns :verified true with full proof chain. Co-Authored-By: Claude Opus 4.6 (1M context) --- sx/sx/handlers/pub-api.sx | 56 +++++++++++++ sx/sxc/pages/helpers.py | 12 +++ sx/sxc/pages/pub_helpers.py | 162 ++++++++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+) diff --git a/sx/sx/handlers/pub-api.sx b/sx/sx/handlers/pub-api.sx index ffea0630..a99b7497 100644 --- a/sx/sx/handlers/pub-api.sx +++ b/sx/sx/handlers/pub-api.sx @@ -337,3 +337,59 @@ " :actor-url \"" (get f "actor-url") "\")")) data))) (str "(SxFollowing" (join "" items) ")"))))) + + +;; ========================================================================== +;; Phase 4: Anchoring — Merkle trees, OTS, verification +;; ========================================================================== + + +;; -------------------------------------------------------------------------- +;; Anchor pending activities +;; -------------------------------------------------------------------------- + +(defhandler pub-anchor + :path "/pub/anchor" + :method :post + :csrf false + :returns "element" + (&key) + (let ((result (helper "pub-anchor-pending"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (if (= (get result "status") "nothing-to-anchor") + "(Anchor :status \"nothing-to-anchor\" :count 0)" + (str + "(Anchor" + "\n :status \"" (get result "status") "\"" + "\n :count " (get result "count") + "\n :merkle-root \"" (get result "merkle-root") "\"" + "\n :tree-cid \"" (get result "tree-cid") "\"" + "\n :ots-proof-cid \"" (get result "ots-proof-cid") "\")"))))) + + +;; -------------------------------------------------------------------------- +;; Verify a CID's anchor +;; -------------------------------------------------------------------------- + +(defhandler pub-verify + :path "/pub/verify/" + :method :get + :returns "element" + (&key cid) + (let ((data (helper "pub-verify-anchor" cid))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (if (get data "error") + (do + (set-response-status 404) + (str "(Error :message \"" (get data "error") "\")")) + (str + "(AnchorVerification" + "\n :cid \"" (get data "cid") "\"" + "\n :status \"" (get data "status") "\"" + "\n :verified " (get data "verified") + "\n :merkle-root \"" (get data "merkle-root") "\"" + "\n :tree-cid \"" (get data "tree-cid") "\"" + "\n :ots-proof-cid \"" (get data "ots-proof-cid") "\"" + "\n :published \"" (get data "published") "\")"))))) diff --git a/sx/sxc/pages/helpers.py b/sx/sxc/pages/helpers.py index 0877c6cb..6b51db7b 100644 --- a/sx/sxc/pages/helpers.py +++ b/sx/sxc/pages/helpers.py @@ -52,6 +52,8 @@ def _register_sx_helpers() -> None: "pub-process-inbox": _pub_process_inbox, "pub-deliver-to-followers": _pub_deliver_to_followers, "pub-request-body": _pub_request_body, + "pub-anchor-pending": _pub_anchor_pending, + "pub-verify-anchor": _pub_verify_anchor, }) @@ -1806,3 +1808,13 @@ async def _pub_deliver_to_followers(activity_sx): async def _pub_request_body(): from .pub_helpers import get_request_body return await get_request_body() + + +async def _pub_anchor_pending(): + from .pub_helpers import anchor_pending + return await anchor_pending() + + +async def _pub_verify_anchor(cid): + from .pub_helpers import verify_cid_anchor + return await verify_cid_anchor(cid) diff --git a/sx/sxc/pages/pub_helpers.py b/sx/sxc/pages/pub_helpers.py index 535816f0..97883905 100644 --- a/sx/sxc/pages/pub_helpers.py +++ b/sx/sxc/pages/pub_helpers.py @@ -777,3 +777,165 @@ async def get_request_path() -> str: """Get the request path.""" from quart import request return request.path + + +# --------------------------------------------------------------------------- +# Phase 4: Anchoring — Merkle trees, OTS, verification +# --------------------------------------------------------------------------- + +async def anchor_pending() -> dict[str, Any]: + """Anchor all unanchored Publish activities into a Merkle tree. + + 1. Collect unanchored activities (by CID) + 2. Build Merkle tree + 3. Pin tree to IPFS + 4. Submit root to OpenTimestamps + 5. Store proof on IPFS + 6. Record anchor in DB + """ + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubActivity + from shared.utils.anchoring import ( + build_merkle_tree, submit_to_opentimestamps, + ) + from shared.utils.ipfs_client import add_json, add_bytes, is_available + + # Find unanchored Publish activities with CIDs + result = await g.s.execute( + select(SxPubActivity).where( + SxPubActivity.activity_type == "Publish", + SxPubActivity.ipfs_cid.isnot(None), + SxPubActivity.object_data["anchor_tree_cid"].is_(None), + ).order_by(SxPubActivity.created_at.asc()) + .limit(100) + ) + activities = result.scalars().all() + + if not activities: + return {"status": "nothing-to-anchor", "count": 0} + + # Build Merkle tree from CIDs + cids = [a.ipfs_cid for a in activities if a.ipfs_cid] + if not cids: + return {"status": "no-cids", "count": 0} + + tree = build_merkle_tree(cids) + merkle_root = tree["root"] + + # Pin tree to IPFS + tree_cid = None + ots_proof_cid = None + + if await is_available(): + try: + tree_data = { + "root": merkle_root, + "leaves": tree["leaves"], + "cids": cids, + "created_at": datetime.now(timezone.utc).isoformat(), + } + tree_cid = await add_json(tree_data) + logger.info("Merkle tree pinned: %s (%d leaves)", tree_cid, len(cids)) + except Exception as e: + logger.error("IPFS tree storage failed: %s", e) + + # Submit to OpenTimestamps + ots_proof = await submit_to_opentimestamps(merkle_root) + if ots_proof and await is_available(): + try: + ots_proof_cid = await add_bytes(ots_proof) + logger.info("OTS proof pinned: %s", ots_proof_cid) + except Exception as e: + logger.error("IPFS OTS proof storage failed: %s", e) + + # Record anchor in activities (store in object_data) + anchor_info = { + "merkle_root": merkle_root, + "tree_cid": tree_cid, + "ots_proof_cid": ots_proof_cid, + "activity_count": len(activities), + "anchored_at": datetime.now(timezone.utc).isoformat(), + } + + for a in activities: + data = dict(a.object_data or {}) + data["anchor_tree_cid"] = tree_cid + data["anchor_merkle_root"] = merkle_root + data["anchor_ots_cid"] = ots_proof_cid + a.object_data = data + + # Also record anchor as its own activity + from shared.models.sx_pub import SxPubActivity as SPA + g.s.add(SPA( + activity_type="Anchor", + object_type="MerkleTree", + object_data=anchor_info, + ipfs_cid=tree_cid, + )) + await g.s.flush() + + logger.info("Anchored %d activities, root=%s, tree=%s, ots=%s", + len(activities), merkle_root, tree_cid, ots_proof_cid) + + return { + "status": "anchored", + "count": len(activities), + "merkle-root": merkle_root, + "tree-cid": tree_cid or "", + "ots-proof-cid": ots_proof_cid or "", + } + + +async def verify_cid_anchor(cid: str) -> dict[str, Any]: + """Verify the anchor proof for a specific CID.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubActivity + from shared.utils.anchoring import build_merkle_tree, verify_merkle_proof + from shared.utils.ipfs_client import get_bytes + + # Find the Publish activity for this CID + result = await g.s.execute( + select(SxPubActivity).where( + SxPubActivity.activity_type == "Publish", + SxPubActivity.ipfs_cid == cid, + ) + ) + activity = result.scalar_one_or_none() + if not activity: + return {"error": "not-found", "cid": cid} + + data = activity.object_data or {} + tree_cid = data.get("anchor_tree_cid") + merkle_root = data.get("anchor_merkle_root") + ots_cid = data.get("anchor_ots_cid") + + if not tree_cid: + return {"status": "not-anchored", "cid": cid} + + # Fetch the Merkle tree from IPFS to verify + verified = False + if tree_cid: + tree_bytes = await get_bytes(tree_cid) + if tree_bytes: + import json + try: + tree_data = json.loads(tree_bytes) + tree = build_merkle_tree(tree_data["cids"]) + from shared.utils.anchoring import get_merkle_proof + proof = get_merkle_proof(tree, cid) + if proof is not None: + verified = verify_merkle_proof(cid, proof, tree["root"]) + except Exception as e: + logger.warning("Merkle verification failed: %s", e) + + return { + "status": "anchored" if verified else "unverified", + "cid": cid, + "merkle-root": merkle_root or "", + "tree-cid": tree_cid or "", + "ots-proof-cid": ots_cid or "", + "verified": "true" if verified else "false", + "published": activity.published.isoformat() if activity.published else "", + } From 68fcdd6cc0eee0093c9bb7e526348d45601d5bf5 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 02:18:45 +0000 Subject: [PATCH 6/6] sx-pub: live dashboard UI on the plan page Server-rendered dashboard showing live data from sx-pub API: - Server status (DB, IPFS, actor, domain) - Actor identity card with public key - Collections grid with paths - Published documents with CIDs and sizes - Recent outbox activity feed - Followers list - API endpoint links for direct access All phases marked as complete. Co-Authored-By: Claude Opus 4.6 (1M context) --- sx/sx/plans/sx-pub.sx | 127 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 122 insertions(+), 5 deletions(-) diff --git a/sx/sx/plans/sx-pub.sx b/sx/sx/plans/sx-pub.sx index 76c561d8..ffefae7c 100644 --- a/sx/sx/plans/sx-pub.sx +++ b/sx/sx/plans/sx-pub.sx @@ -244,14 +244,131 @@ (~docs/section :title "Implementation Phases" :id "phases" (div :class "space-y-4" (div :class "rounded border border-emerald-200 bg-emerald-50 p-4" - (h4 :class "font-semibold text-emerald-800" "Phase 1: Foundation") + (h4 :class "font-semibold text-emerald-800" "Phase 1: Foundation ✓") (p :class "text-emerald-700 text-sm" "DB schema, async IPFS client, actor endpoint, webfinger, " (code "/pub/actor") " returns SX actor document.")) (div :class "rounded border border-sky-200 bg-sky-50 p-4" - (h4 :class "font-semibold text-sky-800" "Phase 2: Publishing") + (h4 :class "font-semibold text-sky-800" "Phase 2: Publishing ✓") (p :class "text-sky-700 text-sm" "Pin to IPFS, path→CID index, collection browsing. Publish the actual SX spec files as the first content.")) (div :class "rounded border border-violet-200 bg-violet-50 p-4" - (h4 :class "font-semibold text-violet-800" "Phase 3: Federation") + (h4 :class "font-semibold text-violet-800" "Phase 3: Federation ✓") (p :class "text-violet-700 text-sm" "Inbox/outbox, follow/accept, HTTP signature verification, activity delivery, content mirroring.")) (div :class "rounded border border-amber-200 bg-amber-50 p-4" - (h4 :class "font-semibold text-amber-800" "Phase 4: Anchoring") - (p :class "text-amber-700 text-sm" "Merkle trees, OpenTimestamps, Bitcoin proof, provenance UI.")))))) + (h4 :class "font-semibold text-amber-800" "Phase 4: Anchoring ✓") + (p :class "text-amber-700 text-sm" "Merkle trees, OpenTimestamps, Bitcoin proof, provenance verification.")))) + + ;; ----------------------------------------------------------------------- + ;; Live Dashboard + ;; ----------------------------------------------------------------------- + + (~docs/section :title "Live Dashboard" :id "dashboard" + (p "Live data from the sx-pub API — server-rendered from the same endpoints.") + + ;; --- Status --- + (~docs/subsection :title "Server Status" + (let ((status (helper "pub-status-data"))) + (div :class "grid grid-cols-2 sm:grid-cols-4 gap-3" + (div :class "rounded border border-stone-200 p-3 text-center" + (p :class "text-xs text-stone-400 uppercase" "DB") + (p :class "font-semibold text-sm" (get status "db"))) + (div :class "rounded border border-stone-200 p-3 text-center" + (p :class "text-xs text-stone-400 uppercase" "IPFS") + (p :class "font-semibold text-sm" (get status "ipfs"))) + (div :class "rounded border border-stone-200 p-3 text-center" + (p :class "text-xs text-stone-400 uppercase" "Actor") + (p :class "font-semibold text-sm" (get status "actor"))) + (div :class "rounded border border-stone-200 p-3 text-center" + (p :class "text-xs text-stone-400 uppercase" "Domain") + (p :class "font-semibold text-sm" (or (get status "domain") "—")))))) + + ;; --- Actor --- + (~docs/subsection :title "Actor Identity" + (let ((actor (helper "pub-actor-data"))) + (div :class "rounded border border-stone-200 bg-stone-50 p-4 space-y-2" + (div :class "flex items-center gap-3" + (div :class "w-10 h-10 rounded-full bg-violet-100 flex items-center justify-center text-violet-700 font-bold" "sx") + (div + (p :class "font-semibold" (get actor "display-name")) + (p :class "text-sm text-stone-500" (str "@" (get actor "preferred-username") "@" (get actor "domain"))))) + (p :class "text-sm text-stone-600" (get actor "summary")) + (details :class "text-xs" + (summary :class "text-stone-400 cursor-pointer" "Public key") + (pre :class "mt-2 bg-stone-100 rounded p-2 text-xs overflow-x-auto" (get actor "public-key-pem")))))) + + ;; --- Collections --- + (~docs/subsection :title "Collections" + (let ((collections (helper "pub-collections-data"))) + (div :class "grid gap-3" + (map (fn (c) + (div :class "rounded border border-stone-200 p-4 hover:border-violet-300 transition-colors" + (div :class "flex items-center justify-between" + (div + (h4 :class "font-semibold text-stone-800" (get c "name")) + (p :class "text-sm text-stone-500" (get c "description"))) + (span :class "text-xs font-mono text-violet-600 bg-violet-50 px-2 py-1 rounded" + (str "/pub/" (get c "slug")))))) + collections)))) + + ;; --- Published Documents --- + (~docs/subsection :title "Published Documents" + (let ((specs (helper "pub-collection-items" "core-specs"))) + (when (not (get specs "error")) + (div :class "space-y-2" + (h4 :class "text-sm font-semibold text-stone-500 uppercase tracking-wide" + (get specs "name")) + (map (fn (d) + (when (!= (get d "slug") "") + (div :class "rounded border border-stone-200 p-3 flex items-center justify-between" + (div + (p :class "font-medium text-stone-800" (get d "title")) + (p :class "text-xs text-stone-400" (get d "summary"))) + (div :class "text-right" + (p :class "text-xs font-mono text-emerald-600 truncate max-w-48" (get d "cid")) + (p :class "text-xs text-stone-400" (str (get d "size") " bytes")))))) + (get specs "items")))))) + + ;; --- Outbox --- + (~docs/subsection :title "Recent Activity" + (let ((outbox (helper "pub-outbox-data" ""))) + (if (= (get outbox "total") 0) + (p :class "text-sm text-stone-400 italic" "No activities yet.") + (div :class "space-y-2" + (p :class "text-xs text-stone-400" (str (get outbox "total") " total activities")) + (map (fn (a) + (when (!= (get a "type") "") + (div :class "rounded border border-stone-200 p-3 flex items-center gap-3" + (span :class "text-xs font-semibold text-white bg-violet-500 px-2 py-0.5 rounded" + (get a "type")) + (span :class "text-xs text-stone-500" (get a "published")) + (when (!= (get a "cid") "") + (span :class "text-xs font-mono text-emerald-600 truncate max-w-48" + (get a "cid")))))) + (get outbox "items")))))) + + ;; --- Followers --- + (~docs/subsection :title "Followers" + (let ((followers (helper "pub-followers-data"))) + (if (empty? followers) + (p :class "text-sm text-stone-400 italic" "No followers yet.") + (div :class "space-y-2" + (map (fn (f) + (when (!= (get f "acct") "") + (div :class "rounded border border-stone-200 p-3 flex items-center gap-2" + (div :class "w-8 h-8 rounded-full bg-sky-100 flex items-center justify-center text-sky-700 text-xs font-bold" "F") + (p :class "text-sm font-mono text-stone-600 truncate" (get f "acct"))))) + followers)))) + + ;; --- API Endpoints --- + (~docs/subsection :title "Try the API" + (p :class "text-sm text-stone-600 mb-2" "All endpoints return " (code "text/sx") ". Try them directly:") + (div :class "grid grid-cols-2 sm:grid-cols-3 gap-2" + (map (fn (endpoint) + (a :href (get endpoint "href") + :class "block rounded border border-stone-200 p-2 text-center hover:border-violet-300 hover:bg-violet-50 transition-colors text-xs font-mono text-stone-600" + (get endpoint "label"))) + (list + {"label" "GET /pub/actor" "href" "/pub/actor"} + {"label" "GET /pub/status" "href" "/pub/status"} + {"label" "GET /pub/collections" "href" "/pub/collections"} + {"label" "GET /pub/outbox" "href" "/pub/outbox"} + {"label" "GET /pub/followers" "href" "/pub/followers"} + {"label" "GET /pub/following" "href" "/pub/following"}))))))))