Add support for more decentralized storage providers
Added 4 new storage providers: - NFT.Storage (free for NFT data) - Infura IPFS (5GB free) - Filebase (5GB free, S3-compatible IPFS) - Storj (25GB free, decentralized cloud) Updated UI with 7 total storage options in a 4-column grid, each with distinct colored borders for visibility. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
119
server.py
119
server.py
@@ -3089,7 +3089,8 @@ async def list_storage(request: Request, user: User = Depends(get_optional_user)
|
|||||||
async def add_storage(req: AddStorageRequest, user: User = Depends(get_required_user)):
|
async def add_storage(req: AddStorageRequest, user: User = Depends(get_required_user)):
|
||||||
"""Add a storage provider."""
|
"""Add a storage provider."""
|
||||||
# Validate provider type
|
# Validate provider type
|
||||||
if req.provider_type not in ["pinata", "web3storage", "local"]:
|
valid_types = ["pinata", "web3storage", "nftstorage", "infura", "filebase", "storj", "local"]
|
||||||
|
if req.provider_type not in valid_types:
|
||||||
raise HTTPException(400, f"Invalid provider type: {req.provider_type}")
|
raise HTTPException(400, f"Invalid provider type: {req.provider_type}")
|
||||||
|
|
||||||
# Test the provider connection before saving
|
# Test the provider connection before saving
|
||||||
@@ -3129,6 +3130,10 @@ async def add_storage_form(
|
|||||||
api_key: Optional[str] = Form(None),
|
api_key: Optional[str] = Form(None),
|
||||||
secret_key: Optional[str] = Form(None),
|
secret_key: Optional[str] = Form(None),
|
||||||
api_token: Optional[str] = Form(None),
|
api_token: Optional[str] = Form(None),
|
||||||
|
project_id: Optional[str] = Form(None),
|
||||||
|
project_secret: Optional[str] = Form(None),
|
||||||
|
access_key: Optional[str] = Form(None),
|
||||||
|
bucket: Optional[str] = Form(None),
|
||||||
path: Optional[str] = Form(None),
|
path: Optional[str] = Form(None),
|
||||||
):
|
):
|
||||||
"""Add a storage provider via HTML form (cookie auth)."""
|
"""Add a storage provider via HTML form (cookie auth)."""
|
||||||
@@ -3137,7 +3142,8 @@ async def add_storage_form(
|
|||||||
return HTMLResponse('<div class="text-red-400">Not authenticated</div>', status_code=401)
|
return HTMLResponse('<div class="text-red-400">Not authenticated</div>', status_code=401)
|
||||||
|
|
||||||
# Validate provider type
|
# Validate provider type
|
||||||
if provider_type not in ["pinata", "web3storage", "local"]:
|
valid_types = ["pinata", "web3storage", "nftstorage", "infura", "filebase", "storj", "local"]
|
||||||
|
if provider_type not in valid_types:
|
||||||
return HTMLResponse(f'<div class="text-red-400">Invalid provider type: {provider_type}</div>')
|
return HTMLResponse(f'<div class="text-red-400">Invalid provider type: {provider_type}</div>')
|
||||||
|
|
||||||
# Build config based on provider type
|
# Build config based on provider type
|
||||||
@@ -3150,6 +3156,22 @@ async def add_storage_form(
|
|||||||
if not api_token:
|
if not api_token:
|
||||||
return HTMLResponse('<div class="text-red-400">web3.storage requires API Token</div>')
|
return HTMLResponse('<div class="text-red-400">web3.storage requires API Token</div>')
|
||||||
config = {"api_token": api_token}
|
config = {"api_token": api_token}
|
||||||
|
elif provider_type == "nftstorage":
|
||||||
|
if not api_token:
|
||||||
|
return HTMLResponse('<div class="text-red-400">NFT.Storage requires API Token</div>')
|
||||||
|
config = {"api_token": api_token}
|
||||||
|
elif provider_type == "infura":
|
||||||
|
if not project_id or not project_secret:
|
||||||
|
return HTMLResponse('<div class="text-red-400">Infura requires Project ID and Project Secret</div>')
|
||||||
|
config = {"project_id": project_id, "project_secret": project_secret}
|
||||||
|
elif provider_type == "filebase":
|
||||||
|
if not access_key or not secret_key or not bucket:
|
||||||
|
return HTMLResponse('<div class="text-red-400">Filebase requires Access Key, Secret Key, and Bucket</div>')
|
||||||
|
config = {"access_key": access_key, "secret_key": secret_key, "bucket": bucket}
|
||||||
|
elif provider_type == "storj":
|
||||||
|
if not access_key or not secret_key or not bucket:
|
||||||
|
return HTMLResponse('<div class="text-red-400">Storj requires Access Key, Secret Key, and Bucket</div>')
|
||||||
|
config = {"access_key": access_key, "secret_key": secret_key, "bucket": bucket}
|
||||||
elif provider_type == "local":
|
elif provider_type == "local":
|
||||||
if not path:
|
if not path:
|
||||||
return HTMLResponse('<div class="text-red-400">Local storage requires a path</div>')
|
return HTMLResponse('<div class="text-red-400">Local storage requires a path</div>')
|
||||||
@@ -3359,21 +3381,41 @@ async def ui_storage_page(username: str, storages: list, request: Request) -> HT
|
|||||||
|
|
||||||
<h2 class="text-lg font-semibold text-white mb-4">Add Storage Provider</h2>
|
<h2 class="text-lg font-semibold text-white mb-4">Add Storage Provider</h2>
|
||||||
|
|
||||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-4">
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-3 mb-4">
|
||||||
<button onclick="showAddForm('pinata')"
|
<button onclick="showAddForm('pinata')"
|
||||||
class="p-4 bg-dark-800 hover:bg-dark-600 border-2 border-blue-600 hover:border-blue-400 rounded-lg text-left transition-colors">
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-blue-600 hover:border-blue-400 rounded-lg text-left transition-colors">
|
||||||
<div class="font-semibold text-white">Pinata</div>
|
<div class="font-semibold text-white text-sm">Pinata</div>
|
||||||
<div class="text-sm text-gray-400">IPFS pinning service</div>
|
<div class="text-xs text-gray-400">1GB free, IPFS</div>
|
||||||
</button>
|
</button>
|
||||||
<button onclick="showAddForm('web3storage')"
|
<button onclick="showAddForm('web3storage')"
|
||||||
class="p-4 bg-dark-800 hover:bg-dark-600 border-2 border-green-600 hover:border-green-400 rounded-lg text-left transition-colors">
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-green-600 hover:border-green-400 rounded-lg text-left transition-colors">
|
||||||
<div class="font-semibold text-white">web3.storage</div>
|
<div class="font-semibold text-white text-sm">web3.storage</div>
|
||||||
<div class="text-sm text-gray-400">IPFS + Filecoin</div>
|
<div class="text-xs text-gray-400">IPFS + Filecoin</div>
|
||||||
|
</button>
|
||||||
|
<button onclick="showAddForm('nftstorage')"
|
||||||
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-pink-600 hover:border-pink-400 rounded-lg text-left transition-colors">
|
||||||
|
<div class="font-semibold text-white text-sm">NFT.Storage</div>
|
||||||
|
<div class="text-xs text-gray-400">Free for NFTs</div>
|
||||||
|
</button>
|
||||||
|
<button onclick="showAddForm('infura')"
|
||||||
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-orange-600 hover:border-orange-400 rounded-lg text-left transition-colors">
|
||||||
|
<div class="font-semibold text-white text-sm">Infura IPFS</div>
|
||||||
|
<div class="text-xs text-gray-400">5GB free</div>
|
||||||
|
</button>
|
||||||
|
<button onclick="showAddForm('filebase')"
|
||||||
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-cyan-600 hover:border-cyan-400 rounded-lg text-left transition-colors">
|
||||||
|
<div class="font-semibold text-white text-sm">Filebase</div>
|
||||||
|
<div class="text-xs text-gray-400">5GB free, S3+IPFS</div>
|
||||||
|
</button>
|
||||||
|
<button onclick="showAddForm('storj')"
|
||||||
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-indigo-600 hover:border-indigo-400 rounded-lg text-left transition-colors">
|
||||||
|
<div class="font-semibold text-white text-sm">Storj</div>
|
||||||
|
<div class="text-xs text-gray-400">25GB free</div>
|
||||||
</button>
|
</button>
|
||||||
<button onclick="showAddForm('local')"
|
<button onclick="showAddForm('local')"
|
||||||
class="p-4 bg-dark-800 hover:bg-dark-600 border-2 border-purple-600 hover:border-purple-400 rounded-lg text-left transition-colors">
|
class="p-3 bg-dark-800 hover:bg-dark-600 border-2 border-purple-600 hover:border-purple-400 rounded-lg text-left transition-colors">
|
||||||
<div class="font-semibold text-white">Local Storage</div>
|
<div class="font-semibold text-white text-sm">Local Storage</div>
|
||||||
<div class="text-sm text-gray-400">Your own disk</div>
|
<div class="text-xs text-gray-400">Your own disk</div>
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -3399,6 +3441,54 @@ async def ui_storage_page(username: str, storages: list, request: Request) -> HT
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div id="nftstorage-fields" class="hidden space-y-4">
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">API Token</label>
|
||||||
|
<input type="password" name="api_token" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="infura-fields" class="hidden space-y-4">
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Project ID</label>
|
||||||
|
<input type="text" name="project_id" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Project Secret</label>
|
||||||
|
<input type="password" name="project_secret" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="filebase-fields" class="hidden space-y-4">
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Access Key</label>
|
||||||
|
<input type="text" name="access_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Secret Key</label>
|
||||||
|
<input type="password" name="secret_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Bucket Name</label>
|
||||||
|
<input type="text" name="bucket" placeholder="my-bucket" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div id="storj-fields" class="hidden space-y-4">
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Access Key</label>
|
||||||
|
<input type="text" name="access_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Secret Key</label>
|
||||||
|
<input type="password" name="secret_key" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm text-gray-400 mb-1">Bucket Name</label>
|
||||||
|
<input type="text" name="bucket" placeholder="my-bucket" class="w-full px-3 py-2 bg-dark-700 border border-dark-500 rounded text-white">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div id="local-fields" class="hidden space-y-4">
|
<div id="local-fields" class="hidden space-y-4">
|
||||||
<div>
|
<div>
|
||||||
<label class="block text-sm text-gray-400 mb-1">Storage Path</label>
|
<label class="block text-sm text-gray-400 mb-1">Storage Path</label>
|
||||||
@@ -3438,9 +3528,8 @@ async def ui_storage_page(username: str, storages: list, request: Request) -> HT
|
|||||||
document.getElementById('provider_type').value = type;
|
document.getElementById('provider_type').value = type;
|
||||||
|
|
||||||
// Hide all field groups
|
// Hide all field groups
|
||||||
document.getElementById('pinata-fields').classList.add('hidden');
|
const fieldGroups = ['pinata', 'web3storage', 'nftstorage', 'infura', 'filebase', 'storj', 'local'];
|
||||||
document.getElementById('web3storage-fields').classList.add('hidden');
|
fieldGroups.forEach(g => document.getElementById(g + '-fields').classList.add('hidden'));
|
||||||
document.getElementById('local-fields').classList.add('hidden');
|
|
||||||
|
|
||||||
// Show the relevant fields
|
// Show the relevant fields
|
||||||
document.getElementById(type + '-fields').classList.remove('hidden');
|
document.getElementById(type + '-fields').classList.remove('hidden');
|
||||||
|
|||||||
@@ -370,6 +370,479 @@ class Web3StorageProvider(StorageProvider):
|
|||||||
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
||||||
|
|
||||||
|
|
||||||
|
class NFTStorageProvider(StorageProvider):
|
||||||
|
"""NFT.Storage pinning service provider (free for NFT data)."""
|
||||||
|
|
||||||
|
provider_type = "nftstorage"
|
||||||
|
|
||||||
|
def __init__(self, api_token: str, capacity_gb: int = 5):
|
||||||
|
self.api_token = api_token
|
||||||
|
self.capacity_bytes = capacity_gb * 1024**3
|
||||||
|
self.base_url = "https://api.nft.storage"
|
||||||
|
|
||||||
|
def _headers(self) -> dict:
|
||||||
|
return {"Authorization": f"Bearer {self.api_token}"}
|
||||||
|
|
||||||
|
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Pin content to NFT.Storage."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_pin():
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/upload",
|
||||||
|
data=data,
|
||||||
|
headers={**self._headers(), "Content-Type": "application/octet-stream"},
|
||||||
|
timeout=120
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json().get("value", {}).get("cid")
|
||||||
|
|
||||||
|
cid = await asyncio.to_thread(do_pin)
|
||||||
|
logger.info(f"NFT.Storage: Pinned {content_hash[:16]}... as {cid}")
|
||||||
|
return cid
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"NFT.Storage pin failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def unpin(self, content_hash: str) -> bool:
|
||||||
|
"""NFT.Storage doesn't support unpinning - data is stored permanently."""
|
||||||
|
logger.warning("NFT.Storage: Unpinning not supported (permanent storage)")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get(self, content_hash: str) -> Optional[bytes]:
|
||||||
|
"""Get content from NFT.Storage - would need CID mapping."""
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def is_pinned(self, content_hash: str) -> bool:
|
||||||
|
"""Check if content is pinned - would need CID mapping."""
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def test_connection(self) -> tuple[bool, str]:
|
||||||
|
"""Test NFT.Storage API connectivity."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_test():
|
||||||
|
response = requests.get(
|
||||||
|
f"{self.base_url}/",
|
||||||
|
headers=self._headers(),
|
||||||
|
timeout=10
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return True, "Connected to NFT.Storage successfully"
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_test)
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
if e.response.status_code == 401:
|
||||||
|
return False, "Invalid API token"
|
||||||
|
return False, f"HTTP error: {e}"
|
||||||
|
except Exception as e:
|
||||||
|
return False, f"Connection failed: {e}"
|
||||||
|
|
||||||
|
def get_usage(self) -> dict:
|
||||||
|
"""Get NFT.Storage usage stats."""
|
||||||
|
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
||||||
|
|
||||||
|
|
||||||
|
class InfuraIPFSProvider(StorageProvider):
|
||||||
|
"""Infura IPFS pinning service provider."""
|
||||||
|
|
||||||
|
provider_type = "infura"
|
||||||
|
|
||||||
|
def __init__(self, project_id: str, project_secret: str, capacity_gb: int = 5):
|
||||||
|
self.project_id = project_id
|
||||||
|
self.project_secret = project_secret
|
||||||
|
self.capacity_bytes = capacity_gb * 1024**3
|
||||||
|
self.base_url = "https://ipfs.infura.io:5001/api/v0"
|
||||||
|
|
||||||
|
def _auth(self) -> tuple:
|
||||||
|
return (self.project_id, self.project_secret)
|
||||||
|
|
||||||
|
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Pin content to Infura IPFS."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_pin():
|
||||||
|
files = {"file": (filename or f"{content_hash[:16]}.bin", data)}
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/add",
|
||||||
|
files=files,
|
||||||
|
auth=self._auth(),
|
||||||
|
timeout=120
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json().get("Hash")
|
||||||
|
|
||||||
|
cid = await asyncio.to_thread(do_pin)
|
||||||
|
logger.info(f"Infura IPFS: Pinned {content_hash[:16]}... as {cid}")
|
||||||
|
return cid
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Infura IPFS pin failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def unpin(self, content_hash: str) -> bool:
|
||||||
|
"""Unpin content from Infura IPFS."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_unpin():
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/pin/rm",
|
||||||
|
params={"arg": content_hash},
|
||||||
|
auth=self._auth(),
|
||||||
|
timeout=30
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return True
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_unpin)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Infura IPFS unpin failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get(self, content_hash: str) -> Optional[bytes]:
|
||||||
|
"""Get content from Infura IPFS gateway."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_get():
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/cat",
|
||||||
|
params={"arg": content_hash},
|
||||||
|
auth=self._auth(),
|
||||||
|
timeout=120
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.content
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_get)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Infura IPFS get failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def is_pinned(self, content_hash: str) -> bool:
|
||||||
|
"""Check if content is pinned on Infura IPFS."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_check():
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/pin/ls",
|
||||||
|
params={"arg": content_hash},
|
||||||
|
auth=self._auth(),
|
||||||
|
timeout=30
|
||||||
|
)
|
||||||
|
return response.status_code == 200
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_check)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def test_connection(self) -> tuple[bool, str]:
|
||||||
|
"""Test Infura IPFS API connectivity."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
def do_test():
|
||||||
|
response = requests.post(
|
||||||
|
f"{self.base_url}/id",
|
||||||
|
auth=self._auth(),
|
||||||
|
timeout=10
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
return True, "Connected to Infura IPFS successfully"
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_test)
|
||||||
|
except requests.exceptions.HTTPError as e:
|
||||||
|
if e.response.status_code == 401:
|
||||||
|
return False, "Invalid project credentials"
|
||||||
|
return False, f"HTTP error: {e}"
|
||||||
|
except Exception as e:
|
||||||
|
return False, f"Connection failed: {e}"
|
||||||
|
|
||||||
|
def get_usage(self) -> dict:
|
||||||
|
"""Get Infura usage stats."""
|
||||||
|
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
||||||
|
|
||||||
|
|
||||||
|
class FilebaseProvider(StorageProvider):
|
||||||
|
"""Filebase S3-compatible IPFS pinning service."""
|
||||||
|
|
||||||
|
provider_type = "filebase"
|
||||||
|
|
||||||
|
def __init__(self, access_key: str, secret_key: str, bucket: str, capacity_gb: int = 5):
|
||||||
|
self.access_key = access_key
|
||||||
|
self.secret_key = secret_key
|
||||||
|
self.bucket = bucket
|
||||||
|
self.capacity_bytes = capacity_gb * 1024**3
|
||||||
|
self.endpoint = "https://s3.filebase.com"
|
||||||
|
|
||||||
|
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Pin content to Filebase."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_pin():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
key = filename or f"{content_hash[:16]}.bin"
|
||||||
|
s3.put_object(Bucket=self.bucket, Key=key, Body=data)
|
||||||
|
# Get CID from response headers
|
||||||
|
head = s3.head_object(Bucket=self.bucket, Key=key)
|
||||||
|
return head.get('Metadata', {}).get('cid', content_hash)
|
||||||
|
|
||||||
|
cid = await asyncio.to_thread(do_pin)
|
||||||
|
logger.info(f"Filebase: Pinned {content_hash[:16]}... as {cid}")
|
||||||
|
return cid
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Filebase pin failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def unpin(self, content_hash: str) -> bool:
|
||||||
|
"""Remove content from Filebase."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_unpin():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.delete_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_unpin)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Filebase unpin failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get(self, content_hash: str) -> Optional[bytes]:
|
||||||
|
"""Get content from Filebase."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_get():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
response = s3.get_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return response['Body'].read()
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_get)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Filebase get failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def is_pinned(self, content_hash: str) -> bool:
|
||||||
|
"""Check if content exists in Filebase."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_check():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.head_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_check)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def test_connection(self) -> tuple[bool, str]:
|
||||||
|
"""Test Filebase connectivity."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_test():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.head_bucket(Bucket=self.bucket)
|
||||||
|
return True, f"Connected to Filebase bucket '{self.bucket}'"
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_test)
|
||||||
|
except Exception as e:
|
||||||
|
if "404" in str(e):
|
||||||
|
return False, f"Bucket '{self.bucket}' not found"
|
||||||
|
if "403" in str(e):
|
||||||
|
return False, "Invalid credentials or no access to bucket"
|
||||||
|
return False, f"Connection failed: {e}"
|
||||||
|
|
||||||
|
def get_usage(self) -> dict:
|
||||||
|
"""Get Filebase usage stats."""
|
||||||
|
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
||||||
|
|
||||||
|
|
||||||
|
class StorjProvider(StorageProvider):
|
||||||
|
"""Storj decentralized cloud storage (S3-compatible)."""
|
||||||
|
|
||||||
|
provider_type = "storj"
|
||||||
|
|
||||||
|
def __init__(self, access_key: str, secret_key: str, bucket: str, capacity_gb: int = 25):
|
||||||
|
self.access_key = access_key
|
||||||
|
self.secret_key = secret_key
|
||||||
|
self.bucket = bucket
|
||||||
|
self.capacity_bytes = capacity_gb * 1024**3
|
||||||
|
self.endpoint = "https://gateway.storjshare.io"
|
||||||
|
|
||||||
|
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""Store content on Storj."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_pin():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
key = filename or content_hash
|
||||||
|
s3.put_object(Bucket=self.bucket, Key=key, Body=data)
|
||||||
|
return content_hash
|
||||||
|
|
||||||
|
result = await asyncio.to_thread(do_pin)
|
||||||
|
logger.info(f"Storj: Stored {content_hash[:16]}...")
|
||||||
|
return result
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Storj pin failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def unpin(self, content_hash: str) -> bool:
|
||||||
|
"""Remove content from Storj."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_unpin():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.delete_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_unpin)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Storj unpin failed: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def get(self, content_hash: str) -> Optional[bytes]:
|
||||||
|
"""Get content from Storj."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_get():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
response = s3.get_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return response['Body'].read()
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_get)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Storj get failed: {e}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def is_pinned(self, content_hash: str) -> bool:
|
||||||
|
"""Check if content exists on Storj."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_check():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.head_object(Bucket=self.bucket, Key=content_hash)
|
||||||
|
return True
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_check)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def test_connection(self) -> tuple[bool, str]:
|
||||||
|
"""Test Storj connectivity."""
|
||||||
|
try:
|
||||||
|
import asyncio
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
|
||||||
|
def do_test():
|
||||||
|
s3 = boto3.client(
|
||||||
|
's3',
|
||||||
|
endpoint_url=self.endpoint,
|
||||||
|
aws_access_key_id=self.access_key,
|
||||||
|
aws_secret_access_key=self.secret_key,
|
||||||
|
config=Config(signature_version='s3v4')
|
||||||
|
)
|
||||||
|
s3.head_bucket(Bucket=self.bucket)
|
||||||
|
return True, f"Connected to Storj bucket '{self.bucket}'"
|
||||||
|
|
||||||
|
return await asyncio.to_thread(do_test)
|
||||||
|
except Exception as e:
|
||||||
|
if "404" in str(e):
|
||||||
|
return False, f"Bucket '{self.bucket}' not found"
|
||||||
|
if "403" in str(e):
|
||||||
|
return False, "Invalid credentials or no access to bucket"
|
||||||
|
return False, f"Connection failed: {e}"
|
||||||
|
|
||||||
|
def get_usage(self) -> dict:
|
||||||
|
"""Get Storj usage stats."""
|
||||||
|
return {"used_bytes": 0, "capacity_bytes": self.capacity_bytes, "pin_count": 0}
|
||||||
|
|
||||||
|
|
||||||
class LocalStorageProvider(StorageProvider):
|
class LocalStorageProvider(StorageProvider):
|
||||||
"""Local filesystem storage provider."""
|
"""Local filesystem storage provider."""
|
||||||
|
|
||||||
@@ -477,7 +950,7 @@ def create_provider(provider_type: str, config: dict) -> Optional[StorageProvide
|
|||||||
Factory function to create a storage provider from config.
|
Factory function to create a storage provider from config.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
provider_type: 'pinata', 'web3storage', or 'local'
|
provider_type: One of 'pinata', 'web3storage', 'nftstorage', 'infura', 'filebase', 'storj', 'local'
|
||||||
config: Provider-specific configuration dict
|
config: Provider-specific configuration dict
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -493,7 +966,32 @@ def create_provider(provider_type: str, config: dict) -> Optional[StorageProvide
|
|||||||
elif provider_type == "web3storage":
|
elif provider_type == "web3storage":
|
||||||
return Web3StorageProvider(
|
return Web3StorageProvider(
|
||||||
api_token=config["api_token"],
|
api_token=config["api_token"],
|
||||||
capacity_gb=config.get("capacity_gb", 1)
|
capacity_gb=config.get("capacity_gb", 5)
|
||||||
|
)
|
||||||
|
elif provider_type == "nftstorage":
|
||||||
|
return NFTStorageProvider(
|
||||||
|
api_token=config["api_token"],
|
||||||
|
capacity_gb=config.get("capacity_gb", 5)
|
||||||
|
)
|
||||||
|
elif provider_type == "infura":
|
||||||
|
return InfuraIPFSProvider(
|
||||||
|
project_id=config["project_id"],
|
||||||
|
project_secret=config["project_secret"],
|
||||||
|
capacity_gb=config.get("capacity_gb", 5)
|
||||||
|
)
|
||||||
|
elif provider_type == "filebase":
|
||||||
|
return FilebaseProvider(
|
||||||
|
access_key=config["access_key"],
|
||||||
|
secret_key=config["secret_key"],
|
||||||
|
bucket=config["bucket"],
|
||||||
|
capacity_gb=config.get("capacity_gb", 5)
|
||||||
|
)
|
||||||
|
elif provider_type == "storj":
|
||||||
|
return StorjProvider(
|
||||||
|
access_key=config["access_key"],
|
||||||
|
secret_key=config["secret_key"],
|
||||||
|
bucket=config["bucket"],
|
||||||
|
capacity_gb=config.get("capacity_gb", 25)
|
||||||
)
|
)
|
||||||
elif provider_type == "local":
|
elif provider_type == "local":
|
||||||
return LocalStorageProvider(
|
return LocalStorageProvider(
|
||||||
|
|||||||
Reference in New Issue
Block a user