From 7b3d76329185c1c3615780341d20d92c19f2a075 Mon Sep 17 00:00:00 2001 From: giles Date: Wed, 25 Mar 2026 01:17:27 +0000 Subject: [PATCH] sx-pub Phase 1: DB schema, IPFS wiring, actor + webfinger + collections + status endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for the sx-pub federated publishing protocol: - 6 SQLAlchemy models: actor, collections, documents, activities, followers, following - Conditional DB enablement in sx_docs (DATABASE_URL present → enable DB) - Python IO helpers: get_or_create_actor (auto-generates RSA keypair), list_collections, check_status, seed_default_collections - 4 defhandler endpoints returning text/sx (no JSON): /pub/actor, /pub/webfinger, /pub/collections, /pub/status - Alembic migration infrastructure for sx service - Docker compose: DB + pgbouncer + IPFS + env vars Co-Authored-By: Claude Opus 4.6 (1M context) --- docker-compose.dev-pub.yml | 2 + shared/models/sx_pub.py | 164 ++++++++++++++++++++++++++++++++++++ sx/alembic.ini | 35 ++++++++ sx/alembic/env.py | 17 ++++ sx/app.py | 4 +- sx/sx/handlers/pub-api.sx | 98 +++++++++++++++++++++ sx/sxc/pages/helpers.py | 23 +++++ sx/sxc/pages/pub_helpers.py | 144 +++++++++++++++++++++++++++++++ 8 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 shared/models/sx_pub.py create mode 100644 sx/alembic.ini create mode 100644 sx/alembic/env.py create mode 100644 sx/sx/handlers/pub-api.sx create mode 100644 sx/sxc/pages/pub_helpers.py diff --git a/docker-compose.dev-pub.yml b/docker-compose.dev-pub.yml index f2867030..9e2d88e3 100644 --- a/docker-compose.dev-pub.yml +++ b/docker-compose.dev-pub.yml @@ -11,6 +11,8 @@ services: SECRET_KEY: "${SECRET_KEY:-pub-dev-secret}" REDIS_URL: redis://redis:6379/0 DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:5432/sx_pub + ALEMBIC_DATABASE_URL: postgresql+psycopg://postgres:change-me@db:5432/sx_pub + SX_PUB_DOMAIN: pub.sx-web.org WORKERS: "1" ENVIRONMENT: development RELOAD: "true" diff --git a/shared/models/sx_pub.py b/shared/models/sx_pub.py new file mode 100644 index 00000000..23d2be0f --- /dev/null +++ b/shared/models/sx_pub.py @@ -0,0 +1,164 @@ +"""sx-pub ORM models — federated SX publishing protocol. + +Tables for the sx-pub actor, content collections, published documents, +outbox activities, and federation relationships (followers/following). +""" +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import ( + String, Integer, DateTime, Text, + ForeignKey, UniqueConstraint, Index, func, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from shared.db.base import Base + + +class SxPubActor(Base): + """Singleton actor for this sx-pub instance.""" + __tablename__ = "sx_pub_actor" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + preferred_username: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) + display_name: Mapped[str | None] = mapped_column(String(255), nullable=True) + summary: Mapped[str | None] = mapped_column(Text, nullable=True) + public_key_pem: Mapped[str] = mapped_column(Text, nullable=False) + private_key_pem: Mapped[str] = mapped_column(Text, nullable=False) + domain: Mapped[str] = mapped_column(String(255), nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubCollection(Base): + """Named grouping of published documents (e.g. core-specs, platforms).""" + __tablename__ = "sx_pub_collections" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + slug: Mapped[str] = mapped_column(String(128), unique=True, nullable=False) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + documents = relationship("SxPubDocument", back_populates="collection", lazy="selectin") + + def __repr__(self) -> str: + return f"" + + +class SxPubDocument(Base): + """Published content — path→CID index entry.""" + __tablename__ = "sx_pub_documents" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + collection_id: Mapped[int] = mapped_column( + Integer, ForeignKey("sx_pub_collections.id", ondelete="CASCADE"), nullable=False, + ) + slug: Mapped[str] = mapped_column(String(255), nullable=False) + title: Mapped[str | None] = mapped_column(String(512), nullable=True) + summary: Mapped[str | None] = mapped_column(Text, nullable=True) + content_hash: Mapped[str] = mapped_column(String(128), nullable=False) + ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True) + size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + requires: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + status: Mapped[str] = mapped_column( + String(20), nullable=False, default="draft", server_default="draft", + ) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + collection = relationship("SxPubCollection", back_populates="documents") + + __table_args__ = ( + UniqueConstraint("collection_id", "slug", name="uq_pub_doc_collection_slug"), + Index("ix_pub_doc_collection", "collection_id"), + Index("ix_pub_doc_status", "status"), + Index("ix_pub_doc_cid", "ipfs_cid"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubActivity(Base): + """Outbox activity (Publish, Follow, Announce, Anchor).""" + __tablename__ = "sx_pub_activities" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + activity_type: Mapped[str] = mapped_column(String(64), nullable=False) + object_type: Mapped[str | None] = mapped_column(String(64), nullable=True) + object_data: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + ipfs_cid: Mapped[str | None] = mapped_column(String(128), nullable=True) + published: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + __table_args__ = ( + Index("ix_pub_activity_type", "activity_type"), + Index("ix_pub_activity_published", "published"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubFollower(Base): + """Remote server that follows us.""" + __tablename__ = "sx_pub_followers" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + follower_acct: Mapped[str] = mapped_column(String(512), nullable=False) + follower_inbox: Mapped[str] = mapped_column(String(512), nullable=False) + follower_actor_url: Mapped[str] = mapped_column(String(512), nullable=False) + follower_public_key: Mapped[str | None] = mapped_column(Text, nullable=True) + state: Mapped[str] = mapped_column( + String(20), nullable=False, default="accepted", server_default="accepted", + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + + __table_args__ = ( + UniqueConstraint("follower_acct", name="uq_pub_follower_acct"), + Index("ix_pub_follower_state", "state"), + ) + + def __repr__(self) -> str: + return f"" + + +class SxPubFollowing(Base): + """Remote server we follow.""" + __tablename__ = "sx_pub_following" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + remote_actor_url: Mapped[str] = mapped_column(String(512), unique=True, nullable=False) + remote_inbox: Mapped[str] = mapped_column(String(512), nullable=False) + state: Mapped[str] = mapped_column( + String(20), nullable=False, default="pending", server_default="pending", + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now(), + ) + accepted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + __table_args__ = ( + Index("ix_pub_following_state", "state"), + ) + + def __repr__(self) -> str: + return f"" diff --git a/sx/alembic.ini b/sx/alembic.ini new file mode 100644 index 00000000..a04e0716 --- /dev/null +++ b/sx/alembic.ini @@ -0,0 +1,35 @@ +[alembic] +script_location = alembic +sqlalchemy.url = + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s diff --git a/sx/alembic/env.py b/sx/alembic/env.py new file mode 100644 index 00000000..2ac3ba13 --- /dev/null +++ b/sx/alembic/env.py @@ -0,0 +1,17 @@ +from alembic import context +from shared.db.alembic_env import run_alembic + +MODELS = [ + "shared.models.sx_pub", +] + +TABLES = frozenset({ + "sx_pub_actor", + "sx_pub_collections", + "sx_pub_documents", + "sx_pub_activities", + "sx_pub_followers", + "sx_pub_following", +}) + +run_alembic(context.config, MODELS, TABLES) diff --git a/sx/app.py b/sx/app.py index 6a774ce0..0bf28255 100644 --- a/sx/app.py +++ b/sx/app.py @@ -62,8 +62,10 @@ def create_app() -> "Quart": extra_kw = {} if SX_STANDALONE: - extra_kw["no_db"] = True extra_kw["no_oauth"] = True + # Enable DB if DATABASE_URL is set (needed for sx-pub) + if not os.getenv("DATABASE_URL"): + extra_kw["no_db"] = True app = create_base_app( "sx", diff --git a/sx/sx/handlers/pub-api.sx b/sx/sx/handlers/pub-api.sx new file mode 100644 index 00000000..cee64460 --- /dev/null +++ b/sx/sx/handlers/pub-api.sx @@ -0,0 +1,98 @@ +;; ========================================================================== +;; sx-pub Phase 1 API endpoints — actor, webfinger, collections, status +;; +;; All responses are text/sx. No JSON. +;; ========================================================================== + + +;; -------------------------------------------------------------------------- +;; Actor +;; -------------------------------------------------------------------------- + +(defhandler pub-actor + :path "/pub/actor" + :method :get + :returns "element" + (&key) + (let ((actor (helper "pub-actor-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxActor" + "\n :id \"https://" (get actor "domain") "/pub/actor\"" + "\n :type \"SxPublisher\"" + "\n :name \"" (get actor "display-name") "\"" + "\n :summary \"" (get actor "summary") "\"" + "\n :inbox \"/pub/inbox\"" + "\n :outbox \"/pub/outbox\"" + "\n :followers \"/pub/followers\"" + "\n :following \"/pub/following\"" + "\n :public-key-pem \"" (get actor "public-key-pem") "\")")))) + + +;; -------------------------------------------------------------------------- +;; Webfinger +;; -------------------------------------------------------------------------- + +(defhandler pub-webfinger + :path "/pub/webfinger" + :method :get + :returns "element" + (&key) + (let ((resource (helper "request-arg" "resource" "")) + (actor (helper "pub-actor-data"))) + (let ((expected (str "acct:" (get actor "preferred-username") "@" (get actor "domain")))) + (if (!= resource expected) + (do + (set-response-status 404) + (str "(Error :message \"Resource not found\")")) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxWebfinger" + "\n :subject \"" expected "\"" + "\n :actor \"https://" (get actor "domain") "/pub/actor\"" + "\n :type \"SxPublisher\")")))))) + + +;; -------------------------------------------------------------------------- +;; Collections +;; -------------------------------------------------------------------------- + +(defhandler pub-collections + :path "/pub/collections" + :method :get + :returns "element" + (&key) + (let ((collections (helper "pub-collections-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (let ((items (map (fn (c) + (str "\n (SxCollection" + " :slug \"" (get c "slug") "\"" + " :name \"" (get c "name") "\"" + " :description \"" (get c "description") "\"" + " :href \"/pub/" (get c "slug") "\")")) + collections))) + (str "(SxCollections" (join "" items) ")"))))) + + +;; -------------------------------------------------------------------------- +;; Status +;; -------------------------------------------------------------------------- + +(defhandler pub-status + :path "/pub/status" + :method :get + :returns "element" + (&key) + (let ((status (helper "pub-status-data"))) + (do + (set-response-header "Content-Type" "text/sx; charset=utf-8") + (str + "(SxPubStatus" + "\n :healthy " (get status "healthy") + "\n :db \"" (get status "db") "\"" + "\n :ipfs \"" (get status "ipfs") "\"" + "\n :actor \"" (get status "actor") "\"" + "\n :domain \"" (or (get status "domain") "unknown") "\")")))) diff --git a/sx/sxc/pages/helpers.py b/sx/sxc/pages/helpers.py index 9e0463c8..c8874f9b 100644 --- a/sx/sxc/pages/helpers.py +++ b/sx/sxc/pages/helpers.py @@ -37,6 +37,10 @@ def _register_sx_helpers() -> None: "spec-explorer-data": _spec_explorer_data, "spec-explorer-data-by-slug": _spec_explorer_data_by_slug, "handler-source": _handler_source, + # sx-pub helpers (only functional when DATABASE_URL is set) + "pub-actor-data": _pub_actor_data, + "pub-collections-data": _pub_collections_data, + "pub-status-data": _pub_status_data, }) @@ -1717,3 +1721,22 @@ def _page_helpers_demo_data() -> dict: results["attr-keys"] = list(ATTR_DETAILS.keys()) return results + + +# --------------------------------------------------------------------------- +# sx-pub helpers — thin wrappers for SX access +# --------------------------------------------------------------------------- + +async def _pub_actor_data(): + from .pub_helpers import get_or_create_actor + return await get_or_create_actor() + + +async def _pub_collections_data(): + from .pub_helpers import list_collections + return await list_collections() + + +async def _pub_status_data(): + from .pub_helpers import check_status + return await check_status() diff --git a/sx/sxc/pages/pub_helpers.py b/sx/sxc/pages/pub_helpers.py new file mode 100644 index 00000000..3fd19a5b --- /dev/null +++ b/sx/sxc/pages/pub_helpers.py @@ -0,0 +1,144 @@ +"""sx-pub Python IO helpers — actor management, IPFS status, collections. + +These are called from SX defhandlers via (helper "pub-..." args...). +All DB access uses g.s (per-request async session from register_db). +""" +from __future__ import annotations + +import logging +import os +from typing import Any + +logger = logging.getLogger("sx.pub") + +SX_PUB_DOMAIN = os.getenv("SX_PUB_DOMAIN", "pub.sx-web.org") + + +async def get_or_create_actor() -> dict[str, Any]: + """Get or create the singleton sx-pub actor. Auto-generates RSA keypair.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubActor + + result = await g.s.execute( + select(SxPubActor).where(SxPubActor.preferred_username == "sx") + ) + actor = result.scalar_one_or_none() + + if actor is None: + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives import serialization + + private_key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, + ) + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + public_pem = private_key.public_key().public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode("utf-8") + + actor = SxPubActor( + preferred_username="sx", + display_name="SX Language", + summary="Federated SX specification publisher", + public_key_pem=public_pem, + private_key_pem=private_pem, + domain=SX_PUB_DOMAIN, + ) + g.s.add(actor) + await g.s.flush() + logger.info("Created sx-pub actor id=%d domain=%s", actor.id, SX_PUB_DOMAIN) + + # Seed default collections on first run + await seed_default_collections() + + return { + "preferred-username": actor.preferred_username, + "display-name": actor.display_name or actor.preferred_username, + "summary": actor.summary or "", + "public-key-pem": actor.public_key_pem, + "domain": actor.domain, + } + + +async def seed_default_collections() -> None: + """Create default collections if they don't exist.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection + + defaults = [ + ("core-specs", "Core Specifications", "Language spec files — evaluator, parser, primitives, render", 0), + ("platforms", "Platforms", "Host platform implementations — JavaScript, Python, OCaml", 1), + ("components", "Components", "Reusable UI components published as content-addressed SX", 2), + ("libraries", "Libraries", "SX library modules — stdlib, signals, freeze, web forms", 3), + ] + + for slug, name, description, order in defaults: + exists = await g.s.execute( + select(SxPubCollection).where(SxPubCollection.slug == slug) + ) + if exists.scalar_one_or_none() is None: + g.s.add(SxPubCollection( + slug=slug, name=name, description=description, sort_order=order, + )) + await g.s.flush() + logger.info("Seeded %d default collections", len(defaults)) + + +async def list_collections() -> list[dict[str, Any]]: + """List all pub collections.""" + from quart import g + from sqlalchemy import select + from shared.models.sx_pub import SxPubCollection + + result = await g.s.execute( + select(SxPubCollection).order_by(SxPubCollection.sort_order) + ) + return [ + { + "slug": c.slug, + "name": c.name, + "description": c.description or "", + } + for c in result.scalars().all() + ] + + +async def check_status() -> dict[str, Any]: + """Health check — DB, IPFS, actor.""" + status: dict[str, Any] = {"healthy": "true"} + + # DB + try: + from quart import g + from sqlalchemy import text + await g.s.execute(text("SELECT 1")) + status["db"] = "connected" + except Exception as e: + status["db"] = f"error: {e}" + status["healthy"] = "false" + + # IPFS + try: + from shared.utils.ipfs_client import is_available + ok = await is_available() + status["ipfs"] = "available" if ok else "unavailable" + except Exception as e: + status["ipfs"] = f"error: {e}" + + # Actor + try: + actor = await get_or_create_actor() + status["actor"] = actor["preferred-username"] + status["domain"] = actor["domain"] + except Exception as e: + status["actor"] = f"error: {e}" + status["healthy"] = "false" + + return status