Add storage engine configuration to L1, matching L2 implementation

- Copy storage_providers.py from L2 (Pinata, web3.storage, NFT.Storage,
  Infura, Filebase, Storj, local storage providers)
- Add storage management endpoints: GET/POST/PATCH/DELETE /storage
- Add provider-specific pages at /storage/type/{provider_type}
- Include connection testing via POST /storage/{id}/test
- Add HTML UI pages with dark theme matching L2

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-10 02:41:50 +00:00
parent 5ce28abe52
commit 2a4f24b7ee
2 changed files with 1549 additions and 0 deletions

540
server.py
View File

@@ -43,6 +43,7 @@ from tasks import render_effect, execute_dag, build_effect_dag
from contextlib import asynccontextmanager
from cache_manager import L1CacheManager, get_cache_manager
import database
import storage_providers
# L1 public URL for redirects
L1_PUBLIC_URL = os.environ.get("L1_PUBLIC_URL", "http://localhost:8100")
@@ -3177,6 +3178,21 @@ class PublishRequest(BaseModel):
asset_type: str = "image" # image, video, etc.
class AddStorageRequest(BaseModel):
"""Request to add a storage provider."""
provider_type: str # 'pinata', 'web3storage', 'local', etc.
provider_name: Optional[str] = None # User-friendly name
config: dict # Provider-specific config (api_key, path, etc.)
capacity_gb: int # Storage capacity in GB
class UpdateStorageRequest(BaseModel):
"""Request to update a storage provider."""
config: Optional[dict] = None
capacity_gb: Optional[int] = None
is_active: Optional[bool] = None
@app.get("/cache/{content_hash}/meta")
async def get_cache_meta(content_hash: str, ctx: UserContext = Depends(get_required_user_context)):
"""Get metadata for a cached file."""
@@ -4403,6 +4419,530 @@ async def ui_run_partial(run_id: str, request: Request):
return html
# ============ User Storage Configuration ============
STORAGE_PROVIDERS_INFO = {
"pinata": {"name": "Pinata", "desc": "1GB free, IPFS pinning", "color": "blue"},
"web3storage": {"name": "web3.storage", "desc": "IPFS + Filecoin", "color": "green"},
"nftstorage": {"name": "NFT.Storage", "desc": "Free for NFTs", "color": "pink"},
"infura": {"name": "Infura IPFS", "desc": "5GB free", "color": "orange"},
"filebase": {"name": "Filebase", "desc": "5GB free, S3+IPFS", "color": "cyan"},
"storj": {"name": "Storj", "desc": "25GB free", "color": "indigo"},
"local": {"name": "Local Storage", "desc": "Your own disk", "color": "purple"},
}
@app.get("/storage")
async def list_storage(request: Request):
"""List user's storage providers. HTML for browsers, JSON for API."""
accept = request.headers.get("accept", "")
wants_json = "application/json" in accept and "text/html" not in accept
ctx = await get_user_context_from_cookie(request)
if not ctx:
if wants_json:
raise HTTPException(401, "Authentication required")
return RedirectResponse(url="/auth", status_code=302)
storages = await database.get_user_storage(ctx.actor_id)
# Add usage stats to each storage
for storage in storages:
usage = await database.get_storage_usage(storage["id"])
storage["used_bytes"] = usage["used_bytes"]
storage["pin_count"] = usage["pin_count"]
storage["donated_gb"] = storage["capacity_gb"] // 2
# Mask sensitive config keys for display
if storage.get("config"):
config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
masked = {}
for k, v in config.items():
if "key" in k.lower() or "token" in k.lower() or "secret" in k.lower():
masked[k] = v[:4] + "..." + v[-4:] if len(str(v)) > 8 else "****"
else:
masked[k] = v
storage["config_display"] = masked
if wants_json:
return {"storages": storages}
return await ui_storage_page(ctx.username, storages, request)
@app.post("/storage")
async def add_storage(req: AddStorageRequest, ctx: UserContext = Depends(get_required_user_context)):
"""Add a storage provider."""
valid_types = ["pinata", "web3storage", "nftstorage", "infura", "filebase", "storj", "local"]
if req.provider_type not in valid_types:
raise HTTPException(400, f"Invalid provider type: {req.provider_type}")
# Test the provider connection before saving
provider = storage_providers.create_provider(req.provider_type, {
**req.config,
"capacity_gb": req.capacity_gb
})
if not provider:
raise HTTPException(400, "Failed to create provider with given config")
success, message = await provider.test_connection()
if not success:
raise HTTPException(400, f"Provider connection failed: {message}")
# Save to database
provider_name = req.provider_name or f"{req.provider_type}-{ctx.username}"
storage_id = await database.add_user_storage(
actor_id=ctx.actor_id,
provider_type=req.provider_type,
provider_name=provider_name,
config=req.config,
capacity_gb=req.capacity_gb
)
if not storage_id:
raise HTTPException(500, "Failed to save storage provider")
return {"id": storage_id, "message": f"Storage provider added: {provider_name}"}
@app.post("/storage/add")
async def add_storage_form(
request: Request,
provider_type: str = Form(...),
provider_name: Optional[str] = Form(None),
description: Optional[str] = Form(None),
capacity_gb: int = Form(5),
api_key: Optional[str] = Form(None),
secret_key: Optional[str] = Form(None),
api_token: Optional[str] = Form(None),
project_id: Optional[str] = Form(None),
project_secret: Optional[str] = Form(None),
access_key: Optional[str] = Form(None),
bucket: Optional[str] = Form(None),
path: Optional[str] = Form(None),
):
"""Add a storage provider via HTML form (cookie auth)."""
ctx = await get_user_context_from_cookie(request)
if not ctx:
return HTMLResponse('<div class="text-red-400">Not authenticated</div>', status_code=401)
valid_types = ["pinata", "web3storage", "nftstorage", "infura", "filebase", "storj", "local"]
if provider_type not in valid_types:
return HTMLResponse(f'<div class="text-red-400">Invalid provider type: {provider_type}</div>')
# Build config based on provider type
config = {}
if provider_type == "pinata":
if not api_key or not secret_key:
return HTMLResponse('<div class="text-red-400">Pinata requires API Key and Secret Key</div>')
config = {"api_key": api_key, "secret_key": secret_key}
elif provider_type == "web3storage":
if not api_token:
return HTMLResponse('<div class="text-red-400">web3.storage requires API Token</div>')
config = {"api_token": api_token}
elif provider_type == "nftstorage":
if not api_token:
return HTMLResponse('<div class="text-red-400">NFT.Storage requires API Token</div>')
config = {"api_token": api_token}
elif provider_type == "infura":
if not project_id or not project_secret:
return HTMLResponse('<div class="text-red-400">Infura requires Project ID and Project Secret</div>')
config = {"project_id": project_id, "project_secret": project_secret}
elif provider_type == "filebase":
if not access_key or not secret_key or not bucket:
return HTMLResponse('<div class="text-red-400">Filebase requires Access Key, Secret Key, and Bucket</div>')
config = {"access_key": access_key, "secret_key": secret_key, "bucket": bucket}
elif provider_type == "storj":
if not access_key or not secret_key or not bucket:
return HTMLResponse('<div class="text-red-400">Storj requires Access Key, Secret Key, and Bucket</div>')
config = {"access_key": access_key, "secret_key": secret_key, "bucket": bucket}
elif provider_type == "local":
if not path:
return HTMLResponse('<div class="text-red-400">Local storage requires a path</div>')
config = {"path": path}
# Test the provider connection before saving
provider = storage_providers.create_provider(provider_type, {
**config,
"capacity_gb": capacity_gb
})
if not provider:
return HTMLResponse('<div class="text-red-400">Failed to create provider with given config</div>')
success, message = await provider.test_connection()
if not success:
return HTMLResponse(f'<div class="text-red-400">Provider connection failed: {message}</div>')
# Save to database
name = provider_name or f"{provider_type}-{ctx.username}-{len(await database.get_user_storage_by_type(ctx.actor_id, provider_type)) + 1}"
storage_id = await database.add_user_storage(
actor_id=ctx.actor_id,
provider_type=provider_type,
provider_name=name,
config=config,
capacity_gb=capacity_gb,
description=description
)
if not storage_id:
return HTMLResponse('<div class="text-red-400">Failed to save storage provider</div>')
return HTMLResponse(f'''
<div class="text-green-400 mb-2">Storage provider "{name}" added successfully!</div>
<script>setTimeout(() => window.location.href = '/storage/type/{provider_type}', 1500);</script>
''')
@app.get("/storage/{storage_id}")
async def get_storage(storage_id: int, ctx: UserContext = Depends(get_required_user_context)):
"""Get a specific storage provider."""
storage = await database.get_storage_by_id(storage_id)
if not storage:
raise HTTPException(404, "Storage provider not found")
if storage["actor_id"] != ctx.actor_id:
raise HTTPException(403, "Not authorized")
usage = await database.get_storage_usage(storage_id)
storage["used_bytes"] = usage["used_bytes"]
storage["pin_count"] = usage["pin_count"]
storage["donated_gb"] = storage["capacity_gb"] // 2
return storage
@app.patch("/storage/{storage_id}")
async def update_storage(storage_id: int, req: UpdateStorageRequest, ctx: UserContext = Depends(get_required_user_context)):
"""Update a storage provider."""
storage = await database.get_storage_by_id(storage_id)
if not storage:
raise HTTPException(404, "Storage provider not found")
if storage["actor_id"] != ctx.actor_id:
raise HTTPException(403, "Not authorized")
# If updating config, test the new connection
if req.config:
existing_config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
new_config = {**existing_config, **req.config}
provider = storage_providers.create_provider(storage["provider_type"], {
**new_config,
"capacity_gb": req.capacity_gb or storage["capacity_gb"]
})
if provider:
success, message = await provider.test_connection()
if not success:
raise HTTPException(400, f"Provider connection failed: {message}")
success = await database.update_user_storage(
storage_id,
config=req.config,
capacity_gb=req.capacity_gb,
is_active=req.is_active
)
if not success:
raise HTTPException(500, "Failed to update storage provider")
return {"message": "Storage provider updated"}
@app.delete("/storage/{storage_id}")
async def remove_storage(storage_id: int, request: Request):
"""Remove a storage provider."""
ctx = await get_user_context_from_cookie(request)
if not ctx:
raise HTTPException(401, "Not authenticated")
storage = await database.get_storage_by_id(storage_id)
if not storage:
raise HTTPException(404, "Storage provider not found")
if storage["actor_id"] != ctx.actor_id:
raise HTTPException(403, "Not authorized")
success = await database.remove_user_storage(storage_id)
if not success:
raise HTTPException(500, "Failed to remove storage provider")
if wants_html(request):
return HTMLResponse("")
return {"message": "Storage provider removed"}
@app.post("/storage/{storage_id}/test")
async def test_storage(storage_id: int, request: Request):
"""Test storage provider connectivity."""
ctx = await get_user_context_from_cookie(request)
if not ctx:
if wants_html(request):
return HTMLResponse('<span class="text-red-400">Not authenticated</span>', status_code=401)
raise HTTPException(401, "Not authenticated")
storage = await database.get_storage_by_id(storage_id)
if not storage:
if wants_html(request):
return HTMLResponse('<span class="text-red-400">Storage not found</span>', status_code=404)
raise HTTPException(404, "Storage provider not found")
if storage["actor_id"] != ctx.actor_id:
if wants_html(request):
return HTMLResponse('<span class="text-red-400">Not authorized</span>', status_code=403)
raise HTTPException(403, "Not authorized")
config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
provider = storage_providers.create_provider(storage["provider_type"], {
**config,
"capacity_gb": storage["capacity_gb"]
})
if not provider:
if wants_html(request):
return HTMLResponse('<span class="text-red-400">Failed to create provider</span>')
raise HTTPException(500, "Failed to create provider")
success, message = await provider.test_connection()
if wants_html(request):
if success:
return HTMLResponse(f'<span class="text-green-400">{message}</span>')
return HTMLResponse(f'<span class="text-red-400">{message}</span>')
return {"success": success, "message": message}
@app.get("/storage/type/{provider_type}")
async def storage_type_page(provider_type: str, request: Request):
"""Page for managing storage configs of a specific type."""
ctx = await get_user_context_from_cookie(request)
if not ctx:
return RedirectResponse(url="/auth", status_code=302)
if provider_type not in STORAGE_PROVIDERS_INFO:
raise HTTPException(404, "Invalid provider type")
storages = await database.get_user_storage_by_type(ctx.actor_id, provider_type)
# Add usage stats and mask config
for storage in storages:
usage = await database.get_storage_usage(storage["id"])
storage["used_bytes"] = usage["used_bytes"]
storage["pin_count"] = usage["pin_count"]
# Mask sensitive config keys
if storage.get("config"):
config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
masked = {}
for k, v in config.items():
if "key" in k.lower() or "token" in k.lower() or "secret" in k.lower():
masked[k] = v[:4] + "..." + v[-4:] if len(str(v)) > 8 else "****"
else:
masked[k] = v
storage["config_display"] = masked
info = STORAGE_PROVIDERS_INFO[provider_type]
return await ui_storage_type_page(ctx.username, provider_type, info, storages, request)
async def ui_storage_page(username: str, storages: list, request: Request) -> HTMLResponse:
"""Render the main storage management page."""
# Count by provider type
type_counts = {}
for s in storages:
ptype = s["provider_type"]
type_counts[ptype] = type_counts.get(ptype, 0) + 1
# Build provider type cards
cards = ""
for ptype, info in STORAGE_PROVIDERS_INFO.items():
count = type_counts.get(ptype, 0)
count_badge = f'<span class="ml-2 px-2 py-0.5 bg-{info["color"]}-600 text-white text-xs rounded-full">{count}</span>' if count > 0 else ""
cards += f'''
<a href="/storage/type/{ptype}" class="block p-4 bg-gray-800 rounded-lg hover:bg-gray-700 transition">
<div class="flex items-center justify-between">
<div>
<h3 class="text-lg font-medium text-{info["color"]}-400">{info["name"]}</h3>
<p class="text-sm text-gray-400">{info["desc"]}</p>
</div>
{count_badge}
</div>
</a>
'''
# Total stats
total_capacity = sum(s["capacity_gb"] for s in storages)
total_used = sum(s["used_bytes"] for s in storages)
total_pins = sum(s["pin_count"] for s in storages)
html = f'''
<!DOCTYPE html>
<html class="dark">
<head>
<title>Storage - Art DAG L1</title>
<script src="https://cdn.tailwindcss.com"></script>
<script src="https://unpkg.com/htmx.org@1.9.10"></script>
</head>
<body class="bg-gray-900 text-white min-h-screen">
<nav class="bg-gray-800 border-b border-gray-700 px-6 py-3">
<div class="flex items-center justify-between">
<a href="/" class="text-xl font-bold text-blue-400">Art DAG L1</a>
<div class="flex items-center gap-4">
<a href="/media" class="text-gray-300 hover:text-white">Media</a>
<a href="/ui/runs" class="text-gray-300 hover:text-white">Runs</a>
<a href="/storage" class="text-blue-400">Storage</a>
<span class="text-gray-400">@{username}</span>
</div>
</div>
</nav>
<main class="max-w-4xl mx-auto px-6 py-8">
<h1 class="text-2xl font-bold mb-6">Storage Providers</h1>
<div class="mb-8 p-4 bg-gray-800 rounded-lg">
<div class="flex items-center justify-between">
<div>
<h2 class="text-lg font-medium">Total Storage</h2>
<p class="text-gray-400">{len(storages)} providers configured</p>
</div>
<div class="text-right">
<p class="text-2xl font-bold">{total_used / (1024**3):.1f} / {total_capacity} GB</p>
<p class="text-sm text-gray-400">{total_pins} items pinned</p>
</div>
</div>
</div>
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
{cards}
</div>
</main>
</body>
</html>
'''
return HTMLResponse(html)
async def ui_storage_type_page(username: str, provider_type: str, info: dict, storages: list, request: Request) -> HTMLResponse:
"""Render storage management page for a specific provider type."""
# Build storage list
storage_rows = ""
for s in storages:
used_gb = s["used_bytes"] / (1024**3)
status_class = "bg-green-600" if s.get("is_active", True) else "bg-gray-600"
status_text = "Active" if s.get("is_active", True) else "Inactive"
config_display = ""
if s.get("config_display"):
for k, v in s["config_display"].items():
config_display += f'<span class="text-gray-500">{k}:</span> <span class="text-gray-300">{v}</span><br>'
storage_rows += f'''
<div id="storage-{s["id"]}" class="p-4 bg-gray-800 rounded-lg mb-4">
<div class="flex items-center justify-between mb-2">
<h3 class="text-lg font-medium">{s.get("provider_name", "Unnamed")}</h3>
<div class="flex items-center gap-2">
<span class="px-2 py-1 {status_class} text-white text-xs rounded-full">{status_text}</span>
<button hx-post="/storage/{s["id"]}/test" hx-target="#test-result-{s["id"]}" hx-swap="innerHTML"
class="px-2 py-1 bg-blue-600 hover:bg-blue-700 text-white text-xs rounded">Test</button>
<button hx-delete="/storage/{s["id"]}" hx-confirm="Remove this storage config?"
hx-target="#storage-{s["id"]}" hx-swap="outerHTML"
class="px-2 py-1 bg-red-600 hover:bg-red-700 text-white text-xs rounded">Remove</button>
</div>
</div>
<div class="text-sm text-gray-400 mb-2">
{used_gb:.2f} / {s["capacity_gb"]} GB used ({s["pin_count"]} items)
</div>
<div class="text-xs font-mono">{config_display}</div>
<div id="test-result-{s["id"]}" class="mt-2 text-sm"></div>
</div>
'''
if not storages:
storage_rows = f'<p class="text-gray-400 text-center py-8">No {info["name"]} configs yet. Add one below.</p>'
# Build form fields based on provider type
form_fields = ""
if provider_type == "pinata":
form_fields = '''
<input type="text" name="api_key" placeholder="API Key" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
<input type="password" name="secret_key" placeholder="Secret Key" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
'''
elif provider_type in ["web3storage", "nftstorage"]:
form_fields = '''
<input type="password" name="api_token" placeholder="API Token" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
'''
elif provider_type == "infura":
form_fields = '''
<input type="text" name="project_id" placeholder="Project ID" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
<input type="password" name="project_secret" placeholder="Project Secret" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
'''
elif provider_type in ["filebase", "storj"]:
form_fields = '''
<input type="text" name="access_key" placeholder="Access Key" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
<input type="password" name="secret_key" placeholder="Secret Key" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
<input type="text" name="bucket" placeholder="Bucket Name" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
'''
elif provider_type == "local":
form_fields = '''
<input type="text" name="path" placeholder="/path/to/storage" required
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
'''
html = f'''
<!DOCTYPE html>
<html class="dark">
<head>
<title>{info["name"]} Storage - Art DAG L1</title>
<script src="https://cdn.tailwindcss.com"></script>
<script src="https://unpkg.com/htmx.org@1.9.10"></script>
</head>
<body class="bg-gray-900 text-white min-h-screen">
<nav class="bg-gray-800 border-b border-gray-700 px-6 py-3">
<div class="flex items-center justify-between">
<a href="/" class="text-xl font-bold text-blue-400">Art DAG L1</a>
<div class="flex items-center gap-4">
<a href="/media" class="text-gray-300 hover:text-white">Media</a>
<a href="/ui/runs" class="text-gray-300 hover:text-white">Runs</a>
<a href="/storage" class="text-gray-300 hover:text-white">Storage</a>
<span class="text-gray-400">@{username}</span>
</div>
</div>
</nav>
<main class="max-w-3xl mx-auto px-6 py-8">
<div class="flex items-center gap-4 mb-6">
<a href="/storage" class="text-gray-400 hover:text-white">&larr; Back</a>
<h1 class="text-2xl font-bold text-{info["color"]}-400">{info["name"]}</h1>
</div>
<div class="mb-8">
{storage_rows}
</div>
<div class="p-4 bg-gray-800 rounded-lg">
<h2 class="text-lg font-medium mb-4">Add New {info["name"]} Config</h2>
<form hx-post="/storage/add" hx-target="#add-result" hx-swap="innerHTML" class="space-y-3">
<input type="hidden" name="provider_type" value="{provider_type}">
<input type="text" name="provider_name" placeholder="Name (optional)"
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
{form_fields}
<input type="number" name="capacity_gb" value="5" min="1"
class="w-full p-2 bg-gray-700 border border-gray-600 rounded">
<button type="submit" class="w-full py-2 bg-{info["color"]}-600 hover:bg-{info["color"]}-700 text-white rounded">
Add Storage Provider
</button>
</form>
<div id="add-result" class="mt-3"></div>
</div>
</main>
</body>
</html>
'''
return HTMLResponse(html)
# ============ Client Download ============
CLIENT_TARBALL = Path(__file__).parent / "artdag-client.tar.gz"