From 5730cd0f220ee12ca256548ebf2da5d612b607c5 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 11 Jan 2026 09:58:36 +0000 Subject: [PATCH] Add documentation routes and update README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update README with comprehensive documentation covering ActivityPub, OpenTimestamps anchoring, L1 integration, and all API endpoints - Add /docs routes to serve markdown documentation as styled HTML - Include common library documentation in web interface 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- README.md | 488 +++++++++++++++++++++++++++++++++++------------------- server.py | 184 ++++++++++++++++++++ 2 files changed, 498 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 47cd777..f180d79 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,34 @@ # Art DAG L2 Server - ActivityPub -Ownership registry and ActivityPub federation for Art DAG. +Ownership registry and ActivityPub federation for Art DAG. Manages asset provenance, cryptographic anchoring, and distributed identity. -## What it does +## Features -- **Registry**: Maintains owned assets with content hashes -- **Activities**: Creates signed ownership claims (Create activities) -- **Federation**: ActivityPub endpoints for follow/share -- **L1 Integration**: Records completed L1 runs as owned assets -- **Authentication**: User registration, login, JWT tokens +- **Asset Registry**: Content-addressed assets with provenance tracking +- **ActivityPub Federation**: Standard protocol for distributed social networking +- **OpenTimestamps Anchoring**: Cryptographic proof of existence on Bitcoin blockchain +- **L1 Integration**: Record and verify L1 rendering runs +- **Storage Providers**: S3, IPFS, and local storage backends +- **Scoped Authentication**: Secure token-based auth for federated L1 servers -## Setup +## Dependencies + +- **PostgreSQL**: Primary data storage +- **artdag-common**: Shared templates and middleware +- **cryptography**: RSA key generation and signing +- **httpx**: Async HTTP client for federation + +## Quick Start ```bash +# Install dependencies pip install -r requirements.txt -# Configure (optional - defaults shown) -export ARTDAG_DOMAIN=artdag.rose-ash.com +# Configure +export ARTDAG_DOMAIN=artdag.example.com export ARTDAG_USER=giles -export ARTDAG_DATA=~/.artdag/l2 export DATABASE_URL=postgresql://artdag:artdag@localhost:5432/artdag -export L1_SERVERS=https://celery-artdag.rose-ash.com +export L1_SERVERS=https://celery-artdag.example.com # Generate signing keys (required for federation) python setup_keys.py @@ -29,142 +37,265 @@ python setup_keys.py python server.py ``` -## JWT Secret Configuration - -The JWT secret is used to sign authentication tokens. **Without a persistent secret, tokens are invalidated on server restart.** - -### Generate a secret +## Docker Deployment ```bash -# Generate a 64-character hex secret +docker stack deploy -c docker-compose.yml artdag-l2 +``` + +## Configuration + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `ARTDAG_DOMAIN` | `artdag.rose-ash.com` | Domain for ActivityPub actors | +| `ARTDAG_USER` | `giles` | Default username | +| `ARTDAG_DATA` | `~/.artdag/l2` | Data directory | +| `DATABASE_URL` | `postgresql://artdag:artdag@localhost:5432/artdag` | PostgreSQL connection | +| `L1_SERVERS` | - | Comma-separated list of L1 server URLs | +| `JWT_SECRET` | (generated) | JWT signing secret | +| `HOST` | `0.0.0.0` | Server bind address | +| `PORT` | `8200` | Server port | + +### JWT Secret + +The JWT secret signs authentication tokens. Without a persistent secret, tokens are invalidated on restart. + +```bash +# Generate a secret openssl rand -hex 32 -# Or with Python -python -c "import secrets; print(secrets.token_hex(32))" + +# Set in environment +export JWT_SECRET="your-generated-secret" + +# Or use Docker secrets (recommended for production) +echo "your-secret" | docker secret create jwt_secret - ``` -### Local development +### RSA Keys + +ActivityPub requires RSA keys for signing activities: ```bash -export JWT_SECRET="your-generated-secret-here" -python server.py -``` - -### Docker Swarm (recommended for production) - -Create a Docker secret: -```bash -# From a generated value -openssl rand -hex 32 | docker secret create jwt_secret - - -# Or from a file -echo "your-secret-here" > jwt_secret.txt -docker secret create jwt_secret jwt_secret.txt -rm jwt_secret.txt -``` - -Reference in docker-compose.yml: -```yaml -services: - l2-server: - secrets: - - jwt_secret - -secrets: - jwt_secret: - external: true -``` - -The server reads secrets from `/run/secrets/jwt_secret` automatically. - -## Key Setup - -ActivityPub requires RSA keys for signing activities. Generate them: - -```bash -# Local +# Generate keys python setup_keys.py # Or with custom paths python setup_keys.py --data-dir /data/l2 --user giles - -# In Docker, exec into container or mount volume -docker exec -it python setup_keys.py ``` -Keys are stored in `$ARTDAG_DATA/keys/`: -- `{username}.pem` - Private key (chmod 600, NEVER share) -- `{username}.pub` - Public key (included in actor profile) - -**Important**: Private keys are gitignored. Back them up securely. Losing them invalidates all your signatures. - -## L1 Renderers Configuration - -The `L1_SERVERS` environment variable defines which L1 rendering servers are available to users. Users can attach to these servers from the Renderers page to run effects and manage media. - -```bash -# Single server (default) -export L1_SERVERS=https://celery-artdag.rose-ash.com - -# Multiple servers (comma-separated) -export L1_SERVERS=https://celery-artdag.rose-ash.com,https://renderer2.example.com,https://renderer3.example.com -``` - -When a user attaches to an L1 server: -1. L2 creates a **scoped token** that only works for that specific L1 -2. User is redirected to the L1's `/auth` endpoint with the scoped token -3. L1 calls back to L2's `/auth/verify` endpoint to validate the token -4. L2 verifies the token scope matches the requesting L1 -5. L1 sets its own local cookie, logging the user in -6. The attachment is recorded in L2's `user_renderers` table - -### Security Features - -**No shared secrets**: L1 servers verify tokens by calling L2's `/auth/verify` endpoint. Any L1 provider can federate without the JWT secret. - -**L1 authorization**: Only servers listed in `L1_SERVERS` can verify tokens. L1 must identify itself in verify requests. - -**Scoped tokens**: Tokens issued for attachment contain an `l1_server` claim. A token scoped to L1-A cannot be used on L1-B, preventing malicious L1s from stealing tokens for use elsewhere. - -**Federated logout**: When a user logs out of L2: -1. L2 revokes the token in its database (`revoked_tokens` table) -2. L2 calls `/auth/revoke` on all attached L1s to revoke their local copies -3. All attachments are removed from `user_renderers` - -Even if a malicious L1 ignores the revoke call, the token will fail verification at L2 because it's in the revocation table. - -**Token revocation on L1**: L1 servers maintain their own Redis-based revocation list and check it on every authenticated request. - -Users can manage attachments at `/renderers`. +Keys stored in `$ARTDAG_DATA/keys/`: +- `{username}.pem` - Private key (chmod 600) +- `{username}.pub` - Public key (in actor profile) ## Web UI -The server provides a web interface: - | Path | Description | |------|-------------| -| `/` | Home page with stats and README | +| `/` | Home page with stats | +| `/login` | Login form | +| `/register` | Registration form | +| `/logout` | Log out | | `/assets` | Browse registered assets | | `/asset/{name}` | Asset detail page | -| `/activities` | View published activities | -| `/activity/{id}` | Activity detail page | -| `/users` | List registered users | -| `/renderers` | Manage L1 renderer connections | -| `/anchors/ui` | OpenTimestamps anchor management | -| `/login` | Login page | -| `/register` | Registration page | -| `/logout` | Log out | +| `/activities` | Published activities | +| `/activity/{id}` | Activity detail | +| `/users` | Registered users | +| `/renderers` | L1 renderer connections | +| `/anchors/ui` | OpenTimestamps management | +| `/storage` | Storage provider config | | `/download/client` | Download CLI client | -## Client Commands +## API Reference -### Upload Media +Interactive docs: http://localhost:8200/docs -Register a media asset (image, video, audio) with a content hash: +### Authentication + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/auth/register` | Register new user | +| POST | `/auth/login` | Login, get JWT token | +| GET | `/auth/me` | Get current user info | +| POST | `/auth/verify` | Verify token (for L1 servers) | + +### Assets + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/assets` | List all assets | +| GET | `/assets/{name}` | Get asset by name | +| POST | `/assets` | Register new asset | +| PATCH | `/assets/{name}` | Update asset metadata | +| POST | `/assets/record-run` | Record L1 run as asset | +| POST | `/assets/publish-cache` | Publish L1 cache item | +| GET | `/assets/by-run-id/{run_id}` | Find asset by L1 run ID | + +### ActivityPub + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/.well-known/webfinger` | Actor discovery | +| GET | `/users/{username}` | Actor profile | +| GET | `/users/{username}/outbox` | Published activities | +| POST | `/users/{username}/inbox` | Receive activities | +| GET | `/users/{username}/followers` | Followers list | +| GET | `/objects/{hash}` | Get object by content hash | +| GET | `/activities` | List activities (paginated) | +| GET | `/activities/{ref}` | Get activity by reference | +| GET | `/activity/{index}` | Get activity by index | + +### OpenTimestamps Anchoring + +| Method | Path | Description | +|--------|------|-------------| +| POST | `/anchors/create` | Create timestamp anchor | +| GET | `/anchors` | List all anchors | +| GET | `/anchors/{merkle_root}` | Get anchor details | +| GET | `/anchors/{merkle_root}/tree` | Get merkle tree | +| GET | `/anchors/verify/{activity_id}` | Verify activity timestamp | +| POST | `/anchors/{merkle_root}/upgrade` | Upgrade pending timestamp | +| GET | `/anchors/ui` | Anchor management UI | +| POST | `/anchors/test-ots` | Test OTS functionality | + +### Renderers (L1 Connections) + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/renderers` | List attached L1 servers | +| GET | `/renderers/attach` | Initiate L1 attachment | +| POST | `/renderers/detach` | Detach from L1 server | + +### Storage Providers + +| Method | Path | Description | +|--------|------|-------------| +| GET | `/storage` | List storage providers | +| POST | `/storage` | Add provider (form) | +| POST | `/storage/add` | Add provider (JSON) | +| GET | `/storage/{id}` | Get provider details | +| PATCH | `/storage/{id}` | Update provider | +| DELETE | `/storage/{id}` | Delete provider | +| POST | `/storage/{id}/test` | Test connection | +| GET | `/storage/type/{type}` | Get form for provider type | + +## L1 Renderer Integration + +L2 coordinates with L1 rendering servers for distributed processing. + +### Configuration ```bash -curl -X POST http://localhost:8200/assets \ - -H "Content-Type: application/json" \ +# Single L1 server +export L1_SERVERS=https://celery-artdag.rose-ash.com + +# Multiple L1 servers +export L1_SERVERS=https://server1.example.com,https://server2.example.com +``` + +### Attachment Flow + +1. User visits `/renderers` and clicks "Attach" +2. L2 creates a **scoped token** bound to the specific L1 +3. User redirected to L1's `/auth?auth_token=...` +4. L1 calls L2's `/auth/verify` to validate +5. L2 checks token scope matches requesting L1 +6. L1 sets local cookie, attachment recorded in `user_renderers` + +### Security + +- **Scoped tokens**: Tokens bound to specific L1; can't be used elsewhere +- **No shared secrets**: L1 verifies via L2's `/auth/verify` endpoint +- **Federated logout**: L2 revokes tokens on all attached L1s + +## OpenTimestamps Anchoring + +Cryptographic proof of existence using Bitcoin blockchain. + +### How It Works + +1. Activities are collected into merkle trees +2. Merkle root submitted to Bitcoin via OpenTimestamps +3. Pending proofs upgraded when Bitcoin confirms +4. Final proof verifiable without trusted third parties + +### Verification + +```bash +# Verify an activity's timestamp +curl https://artdag.example.com/anchors/verify/123 + +# Returns: +{ + "activity_id": 123, + "merkle_root": "abc123...", + "status": "confirmed", + "bitcoin_block": 800000, + "verified_at": "2026-01-01T..." +} +``` + +## Data Model + +### PostgreSQL Tables + +| Table | Description | +|-------|-------------| +| `users` | Registered users with hashed passwords | +| `assets` | Asset registry with content hashes | +| `activities` | Signed ActivityPub activities | +| `followers` | Follower relationships | +| `anchors` | OpenTimestamps anchor records | +| `anchor_activities` | Activity-to-anchor mappings | +| `user_renderers` | L1 attachment records | +| `revoked_tokens` | Token revocation list | +| `storage_providers` | Storage configurations | + +### Asset Structure + +```json +{ + "name": "my-video", + "content_hash": "sha3-256:abc123...", + "asset_type": "video", + "owner": "@giles@artdag.rose-ash.com", + "created_at": "2026-01-01T...", + "provenance": { + "inputs": [...], + "recipe": "beat-sync", + "l1_server": "https://celery-artdag.rose-ash.com", + "run_id": "..." + }, + "tags": ["art", "generated"] +} +``` + +### Activity Structure + +```json +{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": "Create", + "actor": "https://artdag.rose-ash.com/users/giles", + "object": { + "type": "Document", + "name": "my-video", + "content": "sha3-256:abc123...", + "attributedTo": "https://artdag.rose-ash.com/users/giles" + }, + "published": "2026-01-01T..." +} +``` + +## CLI Commands + +### Register Asset + +```bash +curl -X POST https://artdag.example.com/assets \ -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ -d '{ "name": "my-video", "content_hash": "abc123...", @@ -173,14 +304,12 @@ curl -X POST http://localhost:8200/assets \ }' ``` -### Upload Recipe - -Record an L1 run as an owned asset. This fetches the run details from the L1 server and registers the output: +### Record L1 Run ```bash -curl -X POST http://localhost:8200/assets/record-run \ - -H "Content-Type: application/json" \ +curl -X POST https://artdag.example.com/assets/record-run \ -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ -d '{ "run_id": "uuid-from-l1", "l1_server": "https://celery-artdag.rose-ash.com", @@ -188,62 +317,73 @@ curl -X POST http://localhost:8200/assets/record-run \ }' ``` -## API Endpoints +### Publish L1 Cache Item -### Server Info -| Method | Path | Description | -|--------|------|-------------| -| GET | `/` | Home page with stats | - -### Assets -| Method | Path | Description | -|--------|------|-------------| -| GET | `/assets` | List all assets | -| GET | `/assets/{name}` | Get asset by name | -| POST | `/assets` | Upload media - register new asset | -| POST | `/assets/record-run` | Upload recipe - record L1 run | - -### ActivityPub -| Method | Path | Description | -|--------|------|-------------| -| GET | `/.well-known/webfinger?resource=acct:user@domain` | Actor discovery | -| GET | `/users/{username}` | Actor profile | -| GET | `/users/{username}/outbox` | Published activities | -| POST | `/users/{username}/inbox` | Receive activities | -| GET | `/users/{username}/followers` | Followers list | -| GET | `/objects/{content_hash}` | Get object by hash | -| GET | `/activities/{index}` | Get activity by index | - -### Authentication -| Method | Path | Description | -|--------|------|-------------| -| POST | `/auth/register` | Register new user (API) | -| POST | `/auth/login` | Login, get JWT token (API) | -| GET | `/auth/me` | Get current user | -| POST | `/auth/verify` | Verify token (for L1 servers) | - -## Data Storage - -Data stored in PostgreSQL: -- `users` - Registered users -- `assets` - Asset registry -- `activities` - Signed activities -- `followers` - Followers list - -RSA keys stored in `$ARTDAG_DATA/keys/` (files, not database). +```bash +curl -X POST https://artdag.example.com/assets/publish-cache \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{ + "content_hash": "abc123...", + "l1_server": "https://celery-artdag.rose-ash.com", + "name": "my-asset", + "asset_type": "video" + }' +``` ## Architecture ``` -L2 Server (port 8200) +L2 Server (FastAPI) │ - ├── POST /assets (upload media) → Register asset → Create activity → Sign + ├── Web UI (Jinja2 + HTMX + Tailwind) │ - ├── POST /assets/record-run (upload recipe) → Fetch L1 run → Register output + ├── /assets → Asset Registry │ │ - │ └── GET L1_SERVER/runs/{id} + │ └── PostgreSQL (assets table) │ - ├── GET /users/{user}/outbox → Return signed activities + ├── /users/{user}/outbox → ActivityPub + │ │ + │ ├── Sign activities (RSA) + │ └── PostgreSQL (activities table) │ - └── POST /users/{user}/inbox → Receive Follow requests + ├── /anchors → OpenTimestamps + │ │ + │ ├── Merkle tree construction + │ └── Bitcoin anchoring + │ + ├── /auth/verify → L1 Token Verification + │ │ + │ └── Scoped token validation + │ + └── /storage → Storage Providers + │ + ├── S3 (boto3) + ├── IPFS (ipfs_client) + └── Local filesystem +``` + +## Federation + +L2 implements ActivityPub for federated asset sharing. + +### Discovery + +```bash +# Webfinger lookup +curl "https://artdag.example.com/.well-known/webfinger?resource=acct:giles@artdag.example.com" +``` + +### Actor Profile + +```bash +curl -H "Accept: application/activity+json" \ + https://artdag.example.com/users/giles +``` + +### Outbox + +```bash +curl -H "Accept: application/activity+json" \ + https://artdag.example.com/users/giles/outbox ``` diff --git a/server.py b/server.py index 5bd30eb..723399f 100644 --- a/server.py +++ b/server.py @@ -3682,6 +3682,190 @@ async def download_client(): ) +# ============================================================================ +# Documentation Routes +# ============================================================================ + +# Documentation paths +L2_DOCS_DIR = Path(__file__).parent +COMMON_DOCS_DIR = Path(__file__).parent.parent / "common" + +L2_DOCS_MAP = { + "l2": L2_DOCS_DIR / "README.md", + "common": COMMON_DOCS_DIR / "README.md", +} + + +def render_markdown(content: str) -> str: + """Convert markdown to HTML with basic styling.""" + import re + + # Escape HTML first + content = content.replace("&", "&").replace("<", "<").replace(">", ">") + + # Code blocks (``` ... ```) + def code_block_replace(match): + lang = match.group(1) or "" + code = match.group(2) + return f'
{code}
' + content = re.sub(r'```(\w*)\n(.*?)```', code_block_replace, content, flags=re.DOTALL) + + # Inline code + content = re.sub(r'`([^`]+)`', r'\1', content) + + # Headers + content = re.sub(r'^### (.+)$', r'

\1

', content, flags=re.MULTILINE) + content = re.sub(r'^## (.+)$', r'

\1

', content, flags=re.MULTILINE) + content = re.sub(r'^# (.+)$', r'

\1

', content, flags=re.MULTILINE) + + # Bold and italic + content = re.sub(r'\*\*([^*]+)\*\*', r'\1', content) + content = re.sub(r'\*([^*]+)\*', r'\1', content) + + # Links + content = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', r'\1', content) + + # Tables + def table_replace(match): + lines = match.group(0).strip().split('\n') + if len(lines) < 2: + return match.group(0) + + header = lines[0] + rows = lines[2:] if len(lines) > 2 else [] + + header_cells = [cell.strip() for cell in header.split('|')[1:-1]] + header_html = ''.join(f'{cell}' for cell in header_cells) + + rows_html = '' + for row in rows: + cells = [cell.strip() for cell in row.split('|')[1:-1]] + cells_html = ''.join(f'{cell}' for cell in cells) + rows_html += f'{cells_html}' + + return f'{header_html}{rows_html}
' + + content = re.sub(r'(\|[^\n]+\|\n)+', table_replace, content) + + # Bullet points + content = re.sub(r'^- (.+)$', r'
  • \1
    • ', content, flags=re.MULTILINE) + content = re.sub(r'(]*>.*\n?)+', r'
      \g<0>
    ', content) + + # Paragraphs (lines not starting with < or whitespace) + lines = content.split('\n') + result = [] + in_paragraph = False + for line in lines: + stripped = line.strip() + if not stripped: + if in_paragraph: + result.append('

    ') + in_paragraph = False + result.append('') + elif stripped.startswith('<'): + if in_paragraph: + result.append('

    ') + in_paragraph = False + result.append(line) + else: + if not in_paragraph: + result.append('

    ') + in_paragraph = True + result.append(line) + if in_paragraph: + result.append('

    ') + content = '\n'.join(result) + + return content + + +@app.get("/docs", response_class=HTMLResponse) +async def docs_index(request: Request): + """Documentation index page.""" + user = await get_optional_user(request) + + html = f""" + + + Documentation - Art DAG L2 + + + + + +
    +

    Documentation

    +
    + +

    L2 Server (ActivityPub)

    +

    Ownership registry, ActivityPub federation, and OpenTimestamps anchoring.

    +     + +

    Common Library

    +

    Shared components: Jinja2 templates, middleware, content negotiation, and utilities.

    +     +
    +
    + +""" + return HTMLResponse(html) + + +@app.get("/docs/{doc_name}", response_class=HTMLResponse) +async def docs_page(doc_name: str, request: Request): + """Render a markdown documentation file as HTML.""" + if doc_name not in L2_DOCS_MAP: + raise HTTPException(404, f"Documentation '{doc_name}' not found") + + doc_path = L2_DOCS_MAP[doc_name] + if not doc_path.exists(): + raise HTTPException(404, f"Documentation file not found: {doc_path}") + + content = doc_path.read_text() + html_content = render_markdown(content) + + html = f""" + + + {doc_name.upper()} - Art DAG Documentation + + + + + +
    +
    + ← Back to Documentation +
    +
    + {html_content} +
    +
    + +""" + return HTMLResponse(html) + + if __name__ == "__main__": import uvicorn uvicorn.run("server:app", host="0.0.0.0", port=8200, workers=4)