Add user-attachable storage system
Phase 1 of distributed storage implementation:
Database:
- user_storage table for storage providers (Pinata, web3.storage, local)
- storage_pins table to track what's stored where
- source_url/source_type columns on assets for reconstruction
Storage Providers:
- Abstract StorageProvider base class
- PinataProvider for Pinata IPFS pinning
- Web3StorageProvider for web3.storage
- LocalStorageProvider for filesystem storage
- Factory function create_provider()
API Endpoints:
- GET/POST /storage - list/add storage providers
- GET/PATCH/DELETE /storage/{id} - manage individual providers
- POST /storage/{id}/test - test connectivity
UI:
- /storage page with provider cards
- Add provider form (Pinata, web3.storage, local)
- Test/remove buttons per provider
- Usage stats (capacity, donated, used, pins)
50% donation model: half of user capacity is available for
system use to store shared content across the network.
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
388
server.py
388
server.py
@@ -195,6 +195,27 @@ class UpdateAssetRequest(BaseModel):
|
||||
ipfs_cid: Optional[str] = None # IPFS content identifier
|
||||
|
||||
|
||||
class AddStorageRequest(BaseModel):
|
||||
"""Request to add a storage provider."""
|
||||
provider_type: str # 'pinata', 'web3storage', 'local'
|
||||
provider_name: Optional[str] = None # User-friendly name
|
||||
config: dict # Provider-specific config (api_key, path, etc.)
|
||||
capacity_gb: int # Storage capacity in GB
|
||||
|
||||
|
||||
class UpdateStorageRequest(BaseModel):
|
||||
"""Request to update a storage provider."""
|
||||
config: Optional[dict] = None
|
||||
capacity_gb: Optional[int] = None
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
class SetAssetSourceRequest(BaseModel):
|
||||
"""Request to set source URL for an asset."""
|
||||
source_url: str
|
||||
source_type: str # 'youtube', 'local', 'url'
|
||||
|
||||
|
||||
# ============ Storage (Database) ============
|
||||
|
||||
async def load_registry() -> dict:
|
||||
@@ -324,6 +345,7 @@ def base_html(title: str, content: str, username: str = None) -> str:
|
||||
<a href="/users" class="text-gray-400 hover:text-white transition-colors">Users</a>
|
||||
<a href="/anchors/ui" class="text-gray-400 hover:text-white transition-colors">Anchors</a>
|
||||
<a href="/renderers" class="text-gray-400 hover:text-white transition-colors">Renderers</a>
|
||||
<a href="/storage" class="text-gray-400 hover:text-white transition-colors">Storage</a>
|
||||
<a href="/download/client" class="text-gray-400 hover:text-white transition-colors ml-auto" title="Download CLI client">Download Client</a>
|
||||
</nav>
|
||||
|
||||
@@ -3017,6 +3039,372 @@ async def detach_renderer(request: Request):
|
||||
''')
|
||||
|
||||
|
||||
# ============ User Storage ============
|
||||
|
||||
import storage_providers
|
||||
|
||||
|
||||
@app.get("/storage")
|
||||
async def list_storage(request: Request, user: User = Depends(get_optional_user)):
|
||||
"""List user's storage providers. HTML for browsers, JSON for API."""
|
||||
if not user:
|
||||
if wants_html(request):
|
||||
return RedirectResponse(url="/login", status_code=302)
|
||||
raise HTTPException(401, "Authentication required")
|
||||
|
||||
storages = await db.get_user_storage(user.username)
|
||||
|
||||
# Add usage stats to each storage
|
||||
for storage in storages:
|
||||
usage = await db.get_storage_usage(storage["id"])
|
||||
storage["used_bytes"] = usage["used_bytes"]
|
||||
storage["pin_count"] = usage["pin_count"]
|
||||
storage["donated_gb"] = storage["capacity_gb"] // 2
|
||||
# Mask sensitive config keys for display
|
||||
if storage.get("config"):
|
||||
config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
|
||||
masked = {}
|
||||
for k, v in config.items():
|
||||
if "key" in k.lower() or "token" in k.lower() or "secret" in k.lower():
|
||||
masked[k] = v[:4] + "..." + v[-4:] if len(str(v)) > 8 else "****"
|
||||
else:
|
||||
masked[k] = v
|
||||
storage["config_display"] = masked
|
||||
|
||||
if wants_html(request):
|
||||
return await ui_storage_page(user.username, storages, request)
|
||||
|
||||
return {"storages": storages}
|
||||
|
||||
|
||||
@app.post("/storage")
|
||||
async def add_storage(req: AddStorageRequest, user: User = Depends(get_required_user)):
|
||||
"""Add a storage provider."""
|
||||
# Validate provider type
|
||||
if req.provider_type not in ["pinata", "web3storage", "local"]:
|
||||
raise HTTPException(400, f"Invalid provider type: {req.provider_type}")
|
||||
|
||||
# Test the provider connection before saving
|
||||
provider = storage_providers.create_provider(req.provider_type, {
|
||||
**req.config,
|
||||
"capacity_gb": req.capacity_gb
|
||||
})
|
||||
if not provider:
|
||||
raise HTTPException(400, "Failed to create provider with given config")
|
||||
|
||||
success, message = await provider.test_connection()
|
||||
if not success:
|
||||
raise HTTPException(400, f"Provider connection failed: {message}")
|
||||
|
||||
# Save to database
|
||||
provider_name = req.provider_name or f"{req.provider_type}-{user.username}"
|
||||
storage_id = await db.add_user_storage(
|
||||
username=user.username,
|
||||
provider_type=req.provider_type,
|
||||
provider_name=provider_name,
|
||||
config=req.config,
|
||||
capacity_gb=req.capacity_gb
|
||||
)
|
||||
|
||||
if not storage_id:
|
||||
raise HTTPException(500, "Failed to save storage provider")
|
||||
|
||||
return {"id": storage_id, "message": f"Storage provider added: {provider_name}"}
|
||||
|
||||
|
||||
@app.get("/storage/{storage_id}")
|
||||
async def get_storage(storage_id: int, user: User = Depends(get_required_user)):
|
||||
"""Get a specific storage provider."""
|
||||
storage = await db.get_storage_by_id(storage_id)
|
||||
if not storage:
|
||||
raise HTTPException(404, "Storage provider not found")
|
||||
if storage["username"] != user.username:
|
||||
raise HTTPException(403, "Not authorized")
|
||||
|
||||
usage = await db.get_storage_usage(storage_id)
|
||||
storage["used_bytes"] = usage["used_bytes"]
|
||||
storage["pin_count"] = usage["pin_count"]
|
||||
storage["donated_gb"] = storage["capacity_gb"] // 2
|
||||
|
||||
return storage
|
||||
|
||||
|
||||
@app.patch("/storage/{storage_id}")
|
||||
async def update_storage(storage_id: int, req: UpdateStorageRequest, user: User = Depends(get_required_user)):
|
||||
"""Update a storage provider."""
|
||||
storage = await db.get_storage_by_id(storage_id)
|
||||
if not storage:
|
||||
raise HTTPException(404, "Storage provider not found")
|
||||
if storage["username"] != user.username:
|
||||
raise HTTPException(403, "Not authorized")
|
||||
|
||||
# If updating config, test the new connection
|
||||
if req.config:
|
||||
existing_config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
|
||||
new_config = {**existing_config, **req.config}
|
||||
provider = storage_providers.create_provider(storage["provider_type"], {
|
||||
**new_config,
|
||||
"capacity_gb": req.capacity_gb or storage["capacity_gb"]
|
||||
})
|
||||
if provider:
|
||||
success, message = await provider.test_connection()
|
||||
if not success:
|
||||
raise HTTPException(400, f"Provider connection failed: {message}")
|
||||
|
||||
success = await db.update_user_storage(
|
||||
storage_id,
|
||||
config=req.config,
|
||||
capacity_gb=req.capacity_gb,
|
||||
is_active=req.is_active
|
||||
)
|
||||
|
||||
if not success:
|
||||
raise HTTPException(500, "Failed to update storage provider")
|
||||
|
||||
return {"message": "Storage provider updated"}
|
||||
|
||||
|
||||
@app.delete("/storage/{storage_id}")
|
||||
async def remove_storage(storage_id: int, user: User = Depends(get_required_user)):
|
||||
"""Remove a storage provider."""
|
||||
storage = await db.get_storage_by_id(storage_id)
|
||||
if not storage:
|
||||
raise HTTPException(404, "Storage provider not found")
|
||||
if storage["username"] != user.username:
|
||||
raise HTTPException(403, "Not authorized")
|
||||
|
||||
success = await db.remove_user_storage(storage_id)
|
||||
if not success:
|
||||
raise HTTPException(500, "Failed to remove storage provider")
|
||||
|
||||
return {"message": "Storage provider removed"}
|
||||
|
||||
|
||||
@app.post("/storage/{storage_id}/test")
|
||||
async def test_storage(storage_id: int, request: Request, user: User = Depends(get_required_user)):
|
||||
"""Test storage provider connectivity."""
|
||||
storage = await db.get_storage_by_id(storage_id)
|
||||
if not storage:
|
||||
raise HTTPException(404, "Storage provider not found")
|
||||
if storage["username"] != user.username:
|
||||
raise HTTPException(403, "Not authorized")
|
||||
|
||||
config = storage["config"] if isinstance(storage["config"], dict) else json.loads(storage["config"])
|
||||
provider = storage_providers.create_provider(storage["provider_type"], {
|
||||
**config,
|
||||
"capacity_gb": storage["capacity_gb"]
|
||||
})
|
||||
|
||||
if not provider:
|
||||
if wants_html(request):
|
||||
return HTMLResponse('<span class="text-red-400">Failed to create provider</span>')
|
||||
raise HTTPException(500, "Failed to create provider")
|
||||
|
||||
success, message = await provider.test_connection()
|
||||
|
||||
if wants_html(request):
|
||||
if success:
|
||||
return HTMLResponse(f'<span class="text-green-400">{message}</span>')
|
||||
return HTMLResponse(f'<span class="text-red-400">{message}</span>')
|
||||
|
||||
return {"success": success, "message": message}
|
||||
|
||||
|
||||
async def ui_storage_page(username: str, storages: list, request: Request) -> HTMLResponse:
|
||||
"""Render storage settings page."""
|
||||
|
||||
def format_bytes(b):
|
||||
if b > 1024**3:
|
||||
return f"{b / 1024**3:.1f} GB"
|
||||
if b > 1024**2:
|
||||
return f"{b / 1024**2:.1f} MB"
|
||||
if b > 1024:
|
||||
return f"{b / 1024:.1f} KB"
|
||||
return f"{b} bytes"
|
||||
|
||||
storage_rows = ""
|
||||
for s in storages:
|
||||
status_class = "bg-green-600" if s["is_active"] else "bg-gray-600"
|
||||
status_text = "Active" if s["is_active"] else "Inactive"
|
||||
config_display = s.get("config_display", {})
|
||||
config_html = ", ".join(f"{k}: {v}" for k, v in config_display.items() if k != "path")
|
||||
|
||||
storage_rows += f'''
|
||||
<div class="p-4 bg-dark-600 rounded-lg mb-4">
|
||||
<div class="flex justify-between items-start mb-3">
|
||||
<div>
|
||||
<h3 class="font-semibold text-white">{s["provider_name"] or s["provider_type"]}</h3>
|
||||
<span class="text-sm text-gray-400">{s["provider_type"]}</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="px-2 py-1 {status_class} text-white text-xs rounded-full">{status_text}</span>
|
||||
<button hx-post="/storage/{s["id"]}/test" hx-target="#test-result-{s["id"]}" hx-swap="innerHTML"
|
||||
class="px-2 py-1 bg-blue-600 hover:bg-blue-700 text-white text-xs rounded">Test</button>
|
||||
<button hx-delete="/storage/{s["id"]}" hx-confirm="Remove this storage provider?"
|
||||
hx-target="closest div.p-4" hx-swap="outerHTML"
|
||||
class="px-2 py-1 bg-red-600 hover:bg-red-700 text-white text-xs rounded">Remove</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm mb-3">
|
||||
<div>
|
||||
<div class="text-gray-400">Capacity</div>
|
||||
<div class="text-white">{s["capacity_gb"]} GB</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="text-gray-400">Donated</div>
|
||||
<div class="text-white">{s["donated_gb"]} GB</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="text-gray-400">Used</div>
|
||||
<div class="text-white">{format_bytes(s["used_bytes"])}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="text-gray-400">Pins</div>
|
||||
<div class="text-white">{s["pin_count"]}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500">{config_html}</div>
|
||||
<div id="test-result-{s["id"]}" class="mt-2 text-sm"></div>
|
||||
</div>
|
||||
'''
|
||||
|
||||
if not storages:
|
||||
storage_rows = '<p class="text-gray-400 text-center py-8">No storage providers configured.</p>'
|
||||
|
||||
content = f'''
|
||||
<div class="max-w-4xl mx-auto">
|
||||
<div class="flex justify-between items-center mb-6">
|
||||
<h1 class="text-2xl font-bold text-white">Storage Providers</h1>
|
||||
</div>
|
||||
|
||||
<div class="bg-dark-700 rounded-lg p-6 mb-6">
|
||||
<p class="text-gray-400 mb-4">
|
||||
Attach your own storage to help power the network. 50% of your capacity is donated to store
|
||||
shared content, making popular assets more resilient.
|
||||
</p>
|
||||
|
||||
<h2 class="text-lg font-semibold text-white mb-4">Add Storage Provider</h2>
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-4">
|
||||
<button onclick="showAddForm('pinata')"
|
||||
class="p-4 bg-dark-600 hover:bg-dark-500 rounded-lg text-left transition-colors">
|
||||
<div class="font-semibold text-white">Pinata</div>
|
||||
<div class="text-sm text-gray-400">IPFS pinning service</div>
|
||||
</button>
|
||||
<button onclick="showAddForm('web3storage')"
|
||||
class="p-4 bg-dark-600 hover:bg-dark-500 rounded-lg text-left transition-colors">
|
||||
<div class="font-semibold text-white">web3.storage</div>
|
||||
<div class="text-sm text-gray-400">IPFS + Filecoin</div>
|
||||
</button>
|
||||
<button onclick="showAddForm('local')"
|
||||
class="p-4 bg-dark-600 hover:bg-dark-500 rounded-lg text-left transition-colors">
|
||||
<div class="font-semibold text-white">Local Storage</div>
|
||||
<div class="text-sm text-gray-400">Your own disk</div>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div id="add-form" class="hidden bg-dark-600 rounded-lg p-4">
|
||||
<form id="storage-form" hx-post="/storage" hx-target="#add-result" hx-swap="innerHTML">
|
||||
<input type="hidden" name="provider_type" id="provider_type">
|
||||
|
||||
<div id="pinata-fields" class="hidden space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">API Key</label>
|
||||
<input type="text" name="api_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">Secret Key</label>
|
||||
<input type="password" name="secret_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="web3storage-fields" class="hidden space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">API Token</label>
|
||||
<input type="password" name="api_token" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="local-fields" class="hidden space-y-4">
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">Storage Path</label>
|
||||
<input type="text" name="path" placeholder="/path/to/storage" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-4 mt-4">
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">Provider Name (optional)</label>
|
||||
<input type="text" name="provider_name" placeholder="My Storage" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm text-gray-400 mb-1">Capacity (GB)</label>
|
||||
<input type="number" name="capacity_gb" value="5" min="1" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex justify-end gap-3 mt-4">
|
||||
<button type="button" onclick="hideAddForm()" class="px-4 py-2 bg-gray-600 hover:bg-gray-700 text-white rounded">Cancel</button>
|
||||
<button type="submit" class="px-4 py-2 bg-green-600 hover:bg-green-700 text-white rounded">Add Storage</button>
|
||||
</div>
|
||||
</form>
|
||||
<div id="add-result" class="mt-4"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2 class="text-lg font-semibold text-white mb-4">Your Storage Providers</h2>
|
||||
<div id="storage-list">
|
||||
{storage_rows}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
function showAddForm(type) {{
|
||||
document.getElementById('add-form').classList.remove('hidden');
|
||||
document.getElementById('provider_type').value = type;
|
||||
|
||||
// Hide all field groups
|
||||
document.getElementById('pinata-fields').classList.add('hidden');
|
||||
document.getElementById('web3storage-fields').classList.add('hidden');
|
||||
document.getElementById('local-fields').classList.add('hidden');
|
||||
|
||||
// Show the relevant fields
|
||||
document.getElementById(type + '-fields').classList.remove('hidden');
|
||||
}}
|
||||
|
||||
function hideAddForm() {{
|
||||
document.getElementById('add-form').classList.add('hidden');
|
||||
}}
|
||||
|
||||
// Handle form submission to build proper JSON
|
||||
document.getElementById('storage-form').addEventListener('htmx:configRequest', function(evt) {{
|
||||
const formData = new FormData(evt.detail.elt);
|
||||
const providerType = formData.get('provider_type');
|
||||
const config = {{}};
|
||||
|
||||
if (providerType === 'pinata') {{
|
||||
config.api_key = formData.get('api_key');
|
||||
config.secret_key = formData.get('secret_key');
|
||||
}} else if (providerType === 'web3storage') {{
|
||||
config.api_token = formData.get('api_token');
|
||||
}} else if (providerType === 'local') {{
|
||||
config.path = formData.get('path');
|
||||
}}
|
||||
|
||||
evt.detail.headers['Content-Type'] = 'application/json';
|
||||
evt.detail.parameters = JSON.stringify({{
|
||||
provider_type: providerType,
|
||||
provider_name: formData.get('provider_name') || null,
|
||||
config: config,
|
||||
capacity_gb: parseInt(formData.get('capacity_gb'))
|
||||
}});
|
||||
}});
|
||||
</script>
|
||||
'''
|
||||
|
||||
return HTMLResponse(base_html("Storage", content, username))
|
||||
|
||||
|
||||
# ============ Client Download ============
|
||||
|
||||
CLIENT_TARBALL = Path(__file__).parent / "artdag-client.tar.gz"
|
||||
|
||||
Reference in New Issue
Block a user