sx-pub Phase 1: DB schema, IPFS wiring, actor + webfinger + collections + status endpoints

Foundation for the sx-pub federated publishing protocol:

- 6 SQLAlchemy models: actor, collections, documents, activities, followers, following
- Conditional DB enablement in sx_docs (DATABASE_URL present → enable DB)
- Python IO helpers: get_or_create_actor (auto-generates RSA keypair),
  list_collections, check_status, seed_default_collections
- 4 defhandler endpoints returning text/sx (no JSON):
  /pub/actor, /pub/webfinger, /pub/collections, /pub/status
- Alembic migration infrastructure for sx service
- Docker compose: DB + pgbouncer + IPFS + env vars

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

docker-compose.dev-pub.yml

@@ -11,6 +11,8 @@ services:
       SECRET_KEY: "${SECRET_KEY:-pub-dev-secret}"
       REDIS_URL: redis://redis:6379/0
       DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:5432/sx_pub
+      ALEMBIC_DATABASE_URL: postgresql+psycopg://postgres:change-me@db:5432/sx_pub
+      SX_PUB_DOMAIN: pub.sx-web.org
       WORKERS: "1"
       ENVIRONMENT: development
       RELOAD: "true"

shared/models/sx_pub.py

@@ -0,0 +1,164 @@
+"""sx-pub ORM models — federated SX publishing protocol.
+
+Tables for the sx-pub actor, content collections, published documents,
+outbox activities, and federation relationships (followers/following).
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import (
+    String, Integer, DateTime, Text,
+    ForeignKey, UniqueConstraint, Index, func,
+)
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from shared.db.base import Base
+
+
+class SxPubActor(Base):
+    """Singleton actor for this sx-pub instance."""
+    __tablename__ = "sx_pub_actor"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    preferred_username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False)
+    display_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+    public_key_pem: Mapped[str] = mapped_column(Text, nullable=False)
+    private_key_pem: Mapped[str] = mapped_column(Text, nullable=False)
+    domain: Mapped[str] = mapped_column(String(255), nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+
+    def __repr__(self) -> str:
+        return f"<SxPubActor @{self.preferred_username}>"
+
+
+class SxPubCollection(Base):
+    """Named grouping of published documents (e.g. core-specs, platforms)."""
+    __tablename__ = "sx_pub_collections"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    slug: Mapped[str] = mapped_column(String(128), unique=True, nullable=False)
+    name: Mapped[str] = mapped_column(String(255), nullable=False)
+    description: Mapped[str | None] = mapped_column(Text, nullable=True)
+    sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+
+    documents = relationship("SxPubDocument", back_populates="collection", lazy="selectin")
+
+    def __repr__(self) -> str:
+        return f"<SxPubCollection {self.slug}>"
+
+
+class SxPubDocument(Base):
+    """Published content — path→CID index entry."""
+    __tablename__ = "sx_pub_documents"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    collection_id: Mapped[int] = mapped_column(
+        Integer, ForeignKey("sx_pub_collections.id", ondelete="CASCADE"), nullable=False,
+    )
+    slug: Mapped[str] = mapped_column(String(255), nullable=False)
+    title: Mapped[str | None] = mapped_column(String(512), nullable=True)
+    summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+    content_hash: Mapped[str] = mapped_column(String(128), nullable=False)
+    ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    requires: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    status: Mapped[str] = mapped_column(
+        String(20), nullable=False, default="draft", server_default="draft",
+    )
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+
+    collection = relationship("SxPubCollection", back_populates="documents")
+
+    __table_args__ = (
+        UniqueConstraint("collection_id", "slug", name="uq_pub_doc_collection_slug"),
+        Index("ix_pub_doc_collection", "collection_id"),
+        Index("ix_pub_doc_status", "status"),
+        Index("ix_pub_doc_cid", "ipfs_cid"),
+    )
+
+    def __repr__(self) -> str:
+        return f"<SxPubDocument {self.slug} cid={self.ipfs_cid}>"
+
+
+class SxPubActivity(Base):
+    """Outbox activity (Publish, Follow, Announce, Anchor)."""
+    __tablename__ = "sx_pub_activities"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    activity_type: Mapped[str] = mapped_column(String(64), nullable=False)
+    object_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
+    object_data: Mapped[dict | None] = mapped_column(JSONB, nullable=True)
+    ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True)
+    published: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+
+    __table_args__ = (
+        Index("ix_pub_activity_type", "activity_type"),
+        Index("ix_pub_activity_published", "published"),
+    )
+
+    def __repr__(self) -> str:
+        return f"<SxPubActivity {self.activity_type}>"
+
+
+class SxPubFollower(Base):
+    """Remote server that follows us."""
+    __tablename__ = "sx_pub_followers"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    follower_acct: Mapped[str] = mapped_column(String(512), nullable=False)
+    follower_inbox: Mapped[str] = mapped_column(String(512), nullable=False)
+    follower_actor_url: Mapped[str] = mapped_column(String(512), nullable=False)
+    follower_public_key: Mapped[str | None] = mapped_column(Text, nullable=True)
+    state: Mapped[str] = mapped_column(
+        String(20), nullable=False, default="accepted", server_default="accepted",
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+
+    __table_args__ = (
+        UniqueConstraint("follower_acct", name="uq_pub_follower_acct"),
+        Index("ix_pub_follower_state", "state"),
+    )
+
+    def __repr__(self) -> str:
+        return f"<SxPubFollower {self.follower_acct}>"
+
+
+class SxPubFollowing(Base):
+    """Remote server we follow."""
+    __tablename__ = "sx_pub_following"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    remote_actor_url: Mapped[str] = mapped_column(String(512), unique=True, nullable=False)
+    remote_inbox: Mapped[str] = mapped_column(String(512), nullable=False)
+    state: Mapped[str] = mapped_column(
+        String(20), nullable=False, default="pending", server_default="pending",
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    accepted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+
+    __table_args__ = (
+        Index("ix_pub_following_state", "state"),
+    )
+
+    def __repr__(self) -> str:
+        return f"<SxPubFollowing {self.remote_actor_url}>"

sx/alembic.ini

@@ -0,0 +1,35 @@
+[alembic]
+script_location = alembic
+sqlalchemy.url =
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s

sx/alembic/env.py

@@ -0,0 +1,17 @@
+from alembic import context
+from shared.db.alembic_env import run_alembic
+
+MODELS = [
+    "shared.models.sx_pub",
+]
+
+TABLES = frozenset({
+    "sx_pub_actor",
+    "sx_pub_collections",
+    "sx_pub_documents",
+    "sx_pub_activities",
+    "sx_pub_followers",
+    "sx_pub_following",
+})
+
+run_alembic(context.config, MODELS, TABLES)

sx/app.py

@@ -62,8 +62,10 @@ def create_app() -> "Quart":
 
     extra_kw = {}
     if SX_STANDALONE:
-        extra_kw["no_db"] = True
         extra_kw["no_oauth"] = True
+        # Enable DB if DATABASE_URL is set (needed for sx-pub)
+        if not os.getenv("DATABASE_URL"):
+            extra_kw["no_db"] = True
 
     app = create_base_app(
         "sx",

sx/sx/handlers/pub-api.sx

@@ -0,0 +1,98 @@
+;; ==========================================================================
+;; sx-pub Phase 1 API endpoints — actor, webfinger, collections, status
+;;
+;; All responses are text/sx. No JSON.
+;; ==========================================================================
+
+
+;; --------------------------------------------------------------------------
+;; Actor
+;; --------------------------------------------------------------------------
+
+(defhandler pub-actor
+  :path "/pub/actor"
+  :method :get
+  :returns "element"
+  (&key)
+  (let ((actor (helper "pub-actor-data")))
+    (do
+      (set-response-header "Content-Type" "text/sx; charset=utf-8")
+      (str
+        "(SxActor"
+        "\n  :id \"https://" (get actor "domain") "/pub/actor\""
+        "\n  :type \"SxPublisher\""
+        "\n  :name \"" (get actor "display-name") "\""
+        "\n  :summary \"" (get actor "summary") "\""
+        "\n  :inbox \"/pub/inbox\""
+        "\n  :outbox \"/pub/outbox\""
+        "\n  :followers \"/pub/followers\""
+        "\n  :following \"/pub/following\""
+        "\n  :public-key-pem \"" (get actor "public-key-pem") "\")"))))
+
+
+;; --------------------------------------------------------------------------
+;; Webfinger
+;; --------------------------------------------------------------------------
+
+(defhandler pub-webfinger
+  :path "/pub/webfinger"
+  :method :get
+  :returns "element"
+  (&key)
+  (let ((resource (helper "request-arg" "resource" ""))
+        (actor (helper "pub-actor-data")))
+    (let ((expected (str "acct:" (get actor "preferred-username") "@" (get actor "domain"))))
+      (if (!= resource expected)
+        (do
+          (set-response-status 404)
+          (str "(Error :message \"Resource not found\")"))
+        (do
+          (set-response-header "Content-Type" "text/sx; charset=utf-8")
+          (str
+            "(SxWebfinger"
+            "\n  :subject \"" expected "\""
+            "\n  :actor \"https://" (get actor "domain") "/pub/actor\""
+            "\n  :type \"SxPublisher\")"))))))
+
+
+;; --------------------------------------------------------------------------
+;; Collections
+;; --------------------------------------------------------------------------
+
+(defhandler pub-collections
+  :path "/pub/collections"
+  :method :get
+  :returns "element"
+  (&key)
+  (let ((collections (helper "pub-collections-data")))
+    (do
+      (set-response-header "Content-Type" "text/sx; charset=utf-8")
+      (let ((items (map (fn (c)
+              (str "\n  (SxCollection"
+                " :slug \"" (get c "slug") "\""
+                " :name \"" (get c "name") "\""
+                " :description \"" (get c "description") "\""
+                " :href \"/pub/" (get c "slug") "\")"))
+              collections)))
+        (str "(SxCollections" (join "" items) ")")))))
+
+
+;; --------------------------------------------------------------------------
+;; Status
+;; --------------------------------------------------------------------------
+
+(defhandler pub-status
+  :path "/pub/status"
+  :method :get
+  :returns "element"
+  (&key)
+  (let ((status (helper "pub-status-data")))
+    (do
+      (set-response-header "Content-Type" "text/sx; charset=utf-8")
+      (str
+        "(SxPubStatus"
+        "\n  :healthy " (get status "healthy")
+        "\n  :db \"" (get status "db") "\""
+        "\n  :ipfs \"" (get status "ipfs") "\""
+        "\n  :actor \"" (get status "actor") "\""
+        "\n  :domain \"" (or (get status "domain") "unknown") "\")"))))

sx/sxc/pages/helpers.py

@@ -37,6 +37,10 @@ def _register_sx_helpers() -> None:
         "spec-explorer-data": _spec_explorer_data,
         "spec-explorer-data-by-slug": _spec_explorer_data_by_slug,
         "handler-source": _handler_source,
+        # sx-pub helpers (only functional when DATABASE_URL is set)
+        "pub-actor-data": _pub_actor_data,
+        "pub-collections-data": _pub_collections_data,
+        "pub-status-data": _pub_status_data,
     })
 
 
@@ -1717,3 +1721,22 @@ def _page_helpers_demo_data() -> dict:
     results["attr-keys"] = list(ATTR_DETAILS.keys())
 
     return results
+
+
+# ---------------------------------------------------------------------------
+# sx-pub helpers — thin wrappers for SX access
+# ---------------------------------------------------------------------------
+
+async def _pub_actor_data():
+    from .pub_helpers import get_or_create_actor
+    return await get_or_create_actor()
+
+
+async def _pub_collections_data():
+    from .pub_helpers import list_collections
+    return await list_collections()
+
+
+async def _pub_status_data():
+    from .pub_helpers import check_status
+    return await check_status()

sx/sxc/pages/pub_helpers.py

@@ -0,0 +1,144 @@
+"""sx-pub Python IO helpers — actor management, IPFS status, collections.
+
+These are called from SX defhandlers via (helper "pub-..." args...).
+All DB access uses g.s (per-request async session from register_db).
+"""
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any
+
+logger = logging.getLogger("sx.pub")
+
+SX_PUB_DOMAIN = os.getenv("SX_PUB_DOMAIN", "pub.sx-web.org")
+
+
+async def get_or_create_actor() -> dict[str, Any]:
+    """Get or create the singleton sx-pub actor. Auto-generates RSA keypair."""
+    from quart import g
+    from sqlalchemy import select
+    from shared.models.sx_pub import SxPubActor
+
+    result = await g.s.execute(
+        select(SxPubActor).where(SxPubActor.preferred_username == "sx")
+    )
+    actor = result.scalar_one_or_none()
+
+    if actor is None:
+        from cryptography.hazmat.primitives.asymmetric import rsa
+        from cryptography.hazmat.primitives import serialization
+
+        private_key = rsa.generate_private_key(
+            public_exponent=65537, key_size=2048,
+        )
+        private_pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode("utf-8")
+        public_pem = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        ).decode("utf-8")
+
+        actor = SxPubActor(
+            preferred_username="sx",
+            display_name="SX Language",
+            summary="Federated SX specification publisher",
+            public_key_pem=public_pem,
+            private_key_pem=private_pem,
+            domain=SX_PUB_DOMAIN,
+        )
+        g.s.add(actor)
+        await g.s.flush()
+        logger.info("Created sx-pub actor id=%d domain=%s", actor.id, SX_PUB_DOMAIN)
+
+        # Seed default collections on first run
+        await seed_default_collections()
+
+    return {
+        "preferred-username": actor.preferred_username,
+        "display-name": actor.display_name or actor.preferred_username,
+        "summary": actor.summary or "",
+        "public-key-pem": actor.public_key_pem,
+        "domain": actor.domain,
+    }
+
+
+async def seed_default_collections() -> None:
+    """Create default collections if they don't exist."""
+    from quart import g
+    from sqlalchemy import select
+    from shared.models.sx_pub import SxPubCollection
+
+    defaults = [
+        ("core-specs", "Core Specifications", "Language spec files — evaluator, parser, primitives, render", 0),
+        ("platforms", "Platforms", "Host platform implementations — JavaScript, Python, OCaml", 1),
+        ("components", "Components", "Reusable UI components published as content-addressed SX", 2),
+        ("libraries", "Libraries", "SX library modules — stdlib, signals, freeze, web forms", 3),
+    ]
+
+    for slug, name, description, order in defaults:
+        exists = await g.s.execute(
+            select(SxPubCollection).where(SxPubCollection.slug == slug)
+        )
+        if exists.scalar_one_or_none() is None:
+            g.s.add(SxPubCollection(
+                slug=slug, name=name, description=description, sort_order=order,
+            ))
+    await g.s.flush()
+    logger.info("Seeded %d default collections", len(defaults))
+
+
+async def list_collections() -> list[dict[str, Any]]:
+    """List all pub collections."""
+    from quart import g
+    from sqlalchemy import select
+    from shared.models.sx_pub import SxPubCollection
+
+    result = await g.s.execute(
+        select(SxPubCollection).order_by(SxPubCollection.sort_order)
+    )
+    return [
+        {
+            "slug": c.slug,
+            "name": c.name,
+            "description": c.description or "",
+        }
+        for c in result.scalars().all()
+    ]
+
+
+async def check_status() -> dict[str, Any]:
+    """Health check — DB, IPFS, actor."""
+    status: dict[str, Any] = {"healthy": "true"}
+
+    # DB
+    try:
+        from quart import g
+        from sqlalchemy import text
+        await g.s.execute(text("SELECT 1"))
+        status["db"] = "connected"
+    except Exception as e:
+        status["db"] = f"error: {e}"
+        status["healthy"] = "false"
+
+    # IPFS
+    try:
+        from shared.utils.ipfs_client import is_available
+        ok = await is_available()
+        status["ipfs"] = "available" if ok else "unavailable"
+    except Exception as e:
+        status["ipfs"] = f"error: {e}"
+
+    # Actor
+    try:
+        actor = await get_or_create_actor()
+        status["actor"] = actor["preferred-username"]
+        status["domain"] = actor["domain"]
+    except Exception as e:
+        status["actor"] = f"error: {e}"
+        status["healthy"] = "false"
+
+    return status