art-dag/activity-pub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add user-attachable storage system

Browse Source 
Phase 1 of distributed storage implementation:

Database:
- user_storage table for storage providers (Pinata, web3.storage, local)
- storage_pins table to track what's stored where
- source_url/source_type columns on assets for reconstruction

Storage Providers:
- Abstract StorageProvider base class
- PinataProvider for Pinata IPFS pinning
- Web3StorageProvider for web3.storage
- LocalStorageProvider for filesystem storage
- Factory function create_provider()

API Endpoints:
- GET/POST /storage - list/add storage providers
- GET/PATCH/DELETE /storage/{id} - manage individual providers
- POST /storage/{id}/test - test connectivity

UI:
- /storage page with provider cards
- Add provider form (Pinata, web3.storage, local)
- Test/remove buttons per provider
- Usage stats (capacity, donated, used, pins)

50% donation model: half of user capacity is available for
system use to store shared content across the network.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-09 23:19:10 +00:00
parent 64749af3fc
commit 1e3d1bb65e
3 changed files with 1113 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

214
db.py
View File

@@ -117,6 +117,32 @@ CREATE TABLE IF NOT EXISTS revoked_tokens (
expires_at TIMESTAMPTZ NOT NULL
);
-- User storage providers (IPFS pinning services, local storage, etc.)
CREATE TABLE IF NOT EXISTS user_storage (
id SERIAL PRIMARY KEY,
username VARCHAR(255) NOT NULL REFERENCES users(username),
provider_type VARCHAR(50) NOT NULL, -- 'pinata', 'web3storage', 'filebase', 'local'
provider_name VARCHAR(255), -- User-friendly name
config JSONB NOT NULL DEFAULT '{}', -- API keys, endpoints, paths
capacity_gb INTEGER NOT NULL, -- Total capacity user is contributing
is_active BOOLEAN DEFAULT true,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(username, provider_type, provider_name)
);
-- Track what's stored where
CREATE TABLE IF NOT EXISTS storage_pins (
id SERIAL PRIMARY KEY,
content_hash VARCHAR(64) NOT NULL,
storage_id INTEGER NOT NULL REFERENCES user_storage(id) ON DELETE CASCADE,
ipfs_cid VARCHAR(128),
pin_type VARCHAR(20) NOT NULL, -- 'user_content', 'donated', 'system'
size_bytes BIGINT,
pinned_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(content_hash, storage_id)
);
-- Indexes
CREATE INDEX IF NOT EXISTS idx_users_created_at ON users(created_at);
CREATE INDEX IF NOT EXISTS idx_assets_content_hash ON assets(content_hash);
@@ -129,6 +155,20 @@ CREATE INDEX IF NOT EXISTS idx_activities_anchor ON activities(anchor_root);
CREATE INDEX IF NOT EXISTS idx_anchors_created ON anchors(created_at DESC);
CREATE INDEX IF NOT EXISTS idx_followers_username ON followers(username);
CREATE INDEX IF NOT EXISTS idx_revoked_tokens_expires ON revoked_tokens(expires_at);
CREATE INDEX IF NOT EXISTS idx_user_storage_username ON user_storage(username);
CREATE INDEX IF NOT EXISTS idx_storage_pins_hash ON storage_pins(content_hash);
CREATE INDEX IF NOT EXISTS idx_storage_pins_storage ON storage_pins(storage_id);
-- Add source URL columns to assets if they don't exist
DO $$ BEGIN
ALTER TABLE assets ADD COLUMN source_url TEXT;
EXCEPTION WHEN duplicate_column THEN NULL;
END $$;
DO $$ BEGIN
ALTER TABLE assets ADD COLUMN source_type VARCHAR(50);
EXCEPTION WHEN duplicate_column THEN NULL;
END $$;
"""
@@ -772,6 +812,180 @@ async def detach_renderer(username: str, l1_url: str) -> bool:
return "DELETE 1" in result
# ============ User Storage ============
async def get_user_storage(username: str) -> list[dict]:
"""Get all storage providers for a user."""
async with get_connection() as conn:
rows = await conn.fetch(
"""SELECT id, username, provider_type, provider_name, config,
capacity_gb, is_active, created_at, updated_at
FROM user_storage WHERE username = $1
ORDER BY created_at""",
username
)
return [dict(row) for row in rows]
async def get_storage_by_id(storage_id: int) -> Optional[dict]:
"""Get a storage provider by ID."""
async with get_connection() as conn:
row = await conn.fetchrow(
"""SELECT id, username, provider_type, provider_name, config,
capacity_gb, is_active, created_at, updated_at
FROM user_storage WHERE id = $1""",
storage_id
)
return dict(row) if row else None
async def add_user_storage(
username: str,
provider_type: str,
provider_name: str,
config: dict,
capacity_gb: int
) -> Optional[int]:
"""Add a storage provider for a user. Returns storage ID."""
async with get_connection() as conn:
try:
row = await conn.fetchrow(
"""INSERT INTO user_storage (username, provider_type, provider_name, config, capacity_gb)
VALUES ($1, $2, $3, $4, $5)
RETURNING id""",
username, provider_type, provider_name, json.dumps(config), capacity_gb
)
return row["id"] if row else None
except Exception:
return None
async def update_user_storage(
storage_id: int,
config: Optional[dict] = None,
capacity_gb: Optional[int] = None,
is_active: Optional[bool] = None
) -> bool:
"""Update a storage provider."""
updates = []
params = []
param_num = 1
if config is not None:
updates.append(f"config = ${param_num}")
params.append(json.dumps(config))
param_num += 1
if capacity_gb is not None:
updates.append(f"capacity_gb = ${param_num}")
params.append(capacity_gb)
param_num += 1
if is_active is not None:
updates.append(f"is_active = ${param_num}")
params.append(is_active)
param_num += 1
if not updates:
return False
updates.append("updated_at = NOW()")
params.append(storage_id)
async with get_connection() as conn:
result = await conn.execute(
f"UPDATE user_storage SET {', '.join(updates)} WHERE id = ${param_num}",
*params
)
return "UPDATE 1" in result
async def remove_user_storage(storage_id: int) -> bool:
"""Remove a storage provider. Cascades to storage_pins."""
async with get_connection() as conn:
result = await conn.execute(
"DELETE FROM user_storage WHERE id = $1",
storage_id
)
return "DELETE 1" in result
async def get_storage_usage(storage_id: int) -> dict:
"""Get storage usage stats for a provider."""
async with get_connection() as conn:
row = await conn.fetchrow(
"""SELECT
COUNT(*) as pin_count,
COALESCE(SUM(size_bytes), 0) as used_bytes
FROM storage_pins WHERE storage_id = $1""",
storage_id
)
return {"pin_count": row["pin_count"], "used_bytes": row["used_bytes"]}
async def add_storage_pin(
content_hash: str,
storage_id: int,
ipfs_cid: Optional[str],
pin_type: str,
size_bytes: int
) -> Optional[int]:
"""Add a pin record. Returns pin ID."""
async with get_connection() as conn:
try:
row = await conn.fetchrow(
"""INSERT INTO storage_pins (content_hash, storage_id, ipfs_cid, pin_type, size_bytes)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (content_hash, storage_id) DO UPDATE SET
ipfs_cid = EXCLUDED.ipfs_cid,
pin_type = EXCLUDED.pin_type,
size_bytes = EXCLUDED.size_bytes,
pinned_at = NOW()
RETURNING id""",
content_hash, storage_id, ipfs_cid, pin_type, size_bytes
)
return row["id"] if row else None
except Exception:
return None
async def remove_storage_pin(content_hash: str, storage_id: int) -> bool:
"""Remove a pin record."""
async with get_connection() as conn:
result = await conn.execute(
"DELETE FROM storage_pins WHERE content_hash = $1 AND storage_id = $2",
content_hash, storage_id
)
return "DELETE 1" in result
async def get_pins_for_content(content_hash: str) -> list[dict]:
"""Get all storage locations where content is pinned."""
async with get_connection() as conn:
rows = await conn.fetch(
"""SELECT sp.*, us.provider_type, us.provider_name, us.username
FROM storage_pins sp
JOIN user_storage us ON sp.storage_id = us.id
WHERE sp.content_hash = $1""",
content_hash
)
return [dict(row) for row in rows]
async def get_all_active_storage() -> list[dict]:
"""Get all active storage providers (for distributed pinning)."""
async with get_connection() as conn:
rows = await conn.fetch(
"""SELECT us.*,
COALESCE(SUM(sp.size_bytes), 0) as used_bytes,
COUNT(sp.id) as pin_count
FROM user_storage us
LEFT JOIN storage_pins sp ON us.id = sp.storage_id
WHERE us.is_active = true
GROUP BY us.id
ORDER BY us.created_at"""
)
return [dict(row) for row in rows]
# ============ Token Revocation ============
async def revoke_token(token_hash: str, username: str, expires_at) -> bool: