art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rename content_hash/output_hash to cid throughout

Browse Source 
Refactor to use IPFS CID as the primary content identifier:
- Update database schema: content_hash -> cid, output_hash -> output_cid
- Update all services, routers, and tasks to use cid terminology
- Update HTML templates to display CID instead of hash
- Update cache_manager parameter names
- Update README documentation

This completes the transition to CID-only content addressing.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-12 08:02:44 +00:00
parent 494a2a8650
commit 92d26b2b72
22 changed files with 981 additions and 988 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
storage_providers.py
View File

@@ -27,12 +27,12 @@ class StorageProvider(ABC):
provider_type: str = "unknown"
@abstractmethod
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""
Pin content to storage.
Args:
content_hash: SHA3-256 hash of the content
cid: SHA3-256 hash of the content
data: Raw bytes to store
filename: Optional filename hint
@@ -42,12 +42,12 @@ class StorageProvider(ABC):
pass
@abstractmethod
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""
Unpin content from storage.
Args:
content_hash: SHA3-256 hash of the content
cid: SHA3-256 hash of the content
Returns:
True if unpinned successfully
@@ -55,12 +55,12 @@ class StorageProvider(ABC):
pass
@abstractmethod
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""
Retrieve content from storage.
Args:
content_hash: SHA3-256 hash of the content
cid: SHA3-256 hash of the content
Returns:
Raw bytes or None if not found
@@ -68,7 +68,7 @@ class StorageProvider(ABC):
pass
@abstractmethod
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content is pinned in this storage."""
pass
@@ -111,16 +111,16 @@ class PinataProvider(StorageProvider):
"pinata_secret_api_key": self.secret_key,
}
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Pin content to Pinata."""
try:
import asyncio
def do_pin():
files = {"file": (filename or f"{content_hash[:16]}.bin", data)}
files = {"file": (filename or f"{cid[:16]}.bin", data)}
metadata = {
"name": filename or content_hash[:16],
"keyvalues": {"content_hash": content_hash}
"name": filename or cid[:16],
"keyvalues": {"cid": cid}
}
response = requests.post(
f"{self.base_url}/pinning/pinFileToIPFS",
@@ -133,22 +133,22 @@ class PinataProvider(StorageProvider):
return response.json().get("IpfsHash")
cid = await asyncio.to_thread(do_pin)
logger.info(f"Pinata: Pinned {content_hash[:16]}... as {cid}")
logger.info(f"Pinata: Pinned {cid[:16]}... as {cid}")
return cid
except Exception as e:
logger.error(f"Pinata pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""Unpin content from Pinata by finding its CID first."""
try:
import asyncio
def do_unpin():
# First find the pin by content_hash metadata
# First find the pin by cid metadata
response = requests.get(
f"{self.base_url}/data/pinList",
params={"metadata[keyvalues][content_hash]": content_hash, "status": "pinned"},
params={"metadata[keyvalues][cid]": cid, "status": "pinned"},
headers=self._headers(),
timeout=30
)
@@ -171,13 +171,13 @@ class PinataProvider(StorageProvider):
return True
result = await asyncio.to_thread(do_unpin)
logger.info(f"Pinata: Unpinned {content_hash[:16]}...")
logger.info(f"Pinata: Unpinned {cid[:16]}...")
return result
except Exception as e:
logger.error(f"Pinata unpin failed: {e}")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from Pinata via IPFS gateway."""
try:
import asyncio
@@ -186,7 +186,7 @@ class PinataProvider(StorageProvider):
# First find the CID
response = requests.get(
f"{self.base_url}/data/pinList",
params={"metadata[keyvalues][content_hash]": content_hash, "status": "pinned"},
params={"metadata[keyvalues][cid]": cid, "status": "pinned"},
headers=self._headers(),
timeout=30
)
@@ -213,7 +213,7 @@ class PinataProvider(StorageProvider):
logger.error(f"Pinata get failed: {e}")
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content is pinned on Pinata."""
try:
import asyncio
@@ -221,7 +221,7 @@ class PinataProvider(StorageProvider):
def do_check():
response = requests.get(
f"{self.base_url}/data/pinList",
params={"metadata[keyvalues][content_hash]": content_hash, "status": "pinned"},
params={"metadata[keyvalues][cid]": cid, "status": "pinned"},
headers=self._headers(),
timeout=30
)
@@ -286,7 +286,7 @@ class Web3StorageProvider(StorageProvider):
def _headers(self) -> dict:
return {"Authorization": f"Bearer {self.api_token}"}
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Pin content to web3.storage."""
try:
import asyncio
@@ -297,7 +297,7 @@ class Web3StorageProvider(StorageProvider):
data=data,
headers={
**self._headers(),
"X-Name": filename or content_hash[:16]
"X-Name": filename or cid[:16]
},
timeout=120
)
@@ -305,24 +305,24 @@ class Web3StorageProvider(StorageProvider):
return response.json().get("cid")
cid = await asyncio.to_thread(do_pin)
logger.info(f"web3.storage: Pinned {content_hash[:16]}... as {cid}")
logger.info(f"web3.storage: Pinned {cid[:16]}... as {cid}")
return cid
except Exception as e:
logger.error(f"web3.storage pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""web3.storage doesn't support unpinning - data is stored permanently."""
logger.warning("web3.storage: Unpinning not supported (permanent storage)")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from web3.storage - would need CID mapping."""
# web3.storage requires knowing the CID to fetch
# For now, return None - we'd need to maintain a mapping
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content is pinned - would need CID mapping."""
return False
@@ -383,7 +383,7 @@ class NFTStorageProvider(StorageProvider):
def _headers(self) -> dict:
return {"Authorization": f"Bearer {self.api_token}"}
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Pin content to NFT.Storage."""
try:
import asyncio
@@ -399,22 +399,22 @@ class NFTStorageProvider(StorageProvider):
return response.json().get("value", {}).get("cid")
cid = await asyncio.to_thread(do_pin)
logger.info(f"NFT.Storage: Pinned {content_hash[:16]}... as {cid}")
logger.info(f"NFT.Storage: Pinned {cid[:16]}... as {cid}")
return cid
except Exception as e:
logger.error(f"NFT.Storage pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""NFT.Storage doesn't support unpinning - data is stored permanently."""
logger.warning("NFT.Storage: Unpinning not supported (permanent storage)")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from NFT.Storage - would need CID mapping."""
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content is pinned - would need CID mapping."""
return False
@@ -459,13 +459,13 @@ class InfuraIPFSProvider(StorageProvider):
def _auth(self) -> tuple:
return (self.project_id, self.project_secret)
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Pin content to Infura IPFS."""
try:
import asyncio
def do_pin():
files = {"file": (filename or f"{content_hash[:16]}.bin", data)}
files = {"file": (filename or f"{cid[:16]}.bin", data)}
response = requests.post(
f"{self.base_url}/add",
files=files,
@@ -476,13 +476,13 @@ class InfuraIPFSProvider(StorageProvider):
return response.json().get("Hash")
cid = await asyncio.to_thread(do_pin)
logger.info(f"Infura IPFS: Pinned {content_hash[:16]}... as {cid}")
logger.info(f"Infura IPFS: Pinned {cid[:16]}... as {cid}")
return cid
except Exception as e:
logger.error(f"Infura IPFS pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""Unpin content from Infura IPFS."""
try:
import asyncio
@@ -490,7 +490,7 @@ class InfuraIPFSProvider(StorageProvider):
def do_unpin():
response = requests.post(
f"{self.base_url}/pin/rm",
params={"arg": content_hash},
params={"arg": cid},
auth=self._auth(),
timeout=30
)
@@ -502,7 +502,7 @@ class InfuraIPFSProvider(StorageProvider):
logger.error(f"Infura IPFS unpin failed: {e}")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from Infura IPFS gateway."""
try:
import asyncio
@@ -510,7 +510,7 @@ class InfuraIPFSProvider(StorageProvider):
def do_get():
response = requests.post(
f"{self.base_url}/cat",
params={"arg": content_hash},
params={"arg": cid},
auth=self._auth(),
timeout=120
)
@@ -522,7 +522,7 @@ class InfuraIPFSProvider(StorageProvider):
logger.error(f"Infura IPFS get failed: {e}")
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content is pinned on Infura IPFS."""
try:
import asyncio
@@ -530,7 +530,7 @@ class InfuraIPFSProvider(StorageProvider):
def do_check():
response = requests.post(
f"{self.base_url}/pin/ls",
params={"arg": content_hash},
params={"arg": cid},
auth=self._auth(),
timeout=30
)
@@ -579,7 +579,7 @@ class FilebaseProvider(StorageProvider):
self.capacity_bytes = capacity_gb * 1024**3
self.endpoint = "https://s3.filebase.com"
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Pin content to Filebase."""
try:
import asyncio
@@ -594,20 +594,20 @@ class FilebaseProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
key = filename or f"{content_hash[:16]}.bin"
key = filename or f"{cid[:16]}.bin"
s3.put_object(Bucket=self.bucket, Key=key, Body=data)
# Get CID from response headers
head = s3.head_object(Bucket=self.bucket, Key=key)
return head.get('Metadata', {}).get('cid', content_hash)
return head.get('Metadata', {}).get('cid', cid)
cid = await asyncio.to_thread(do_pin)
logger.info(f"Filebase: Pinned {content_hash[:16]}... as {cid}")
logger.info(f"Filebase: Pinned {cid[:16]}... as {cid}")
return cid
except Exception as e:
logger.error(f"Filebase pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""Remove content from Filebase."""
try:
import asyncio
@@ -622,7 +622,7 @@ class FilebaseProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
s3.delete_object(Bucket=self.bucket, Key=content_hash)
s3.delete_object(Bucket=self.bucket, Key=cid)
return True
return await asyncio.to_thread(do_unpin)
@@ -630,7 +630,7 @@ class FilebaseProvider(StorageProvider):
logger.error(f"Filebase unpin failed: {e}")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from Filebase."""
try:
import asyncio
@@ -645,7 +645,7 @@ class FilebaseProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
response = s3.get_object(Bucket=self.bucket, Key=content_hash)
response = s3.get_object(Bucket=self.bucket, Key=cid)
return response['Body'].read()
return await asyncio.to_thread(do_get)
@@ -653,7 +653,7 @@ class FilebaseProvider(StorageProvider):
logger.error(f"Filebase get failed: {e}")
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content exists in Filebase."""
try:
import asyncio
@@ -668,7 +668,7 @@ class FilebaseProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
s3.head_object(Bucket=self.bucket, Key=content_hash)
s3.head_object(Bucket=self.bucket, Key=cid)
return True
return await asyncio.to_thread(do_check)
@@ -718,7 +718,7 @@ class StorjProvider(StorageProvider):
self.capacity_bytes = capacity_gb * 1024**3
self.endpoint = "https://gateway.storjshare.io"
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Store content on Storj."""
try:
import asyncio
@@ -733,18 +733,18 @@ class StorjProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
key = filename or content_hash
key = filename or cid
s3.put_object(Bucket=self.bucket, Key=key, Body=data)
return content_hash
return cid
result = await asyncio.to_thread(do_pin)
logger.info(f"Storj: Stored {content_hash[:16]}...")
logger.info(f"Storj: Stored {cid[:16]}...")
return result
except Exception as e:
logger.error(f"Storj pin failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""Remove content from Storj."""
try:
import asyncio
@@ -759,7 +759,7 @@ class StorjProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
s3.delete_object(Bucket=self.bucket, Key=content_hash)
s3.delete_object(Bucket=self.bucket, Key=cid)
return True
return await asyncio.to_thread(do_unpin)
@@ -767,7 +767,7 @@ class StorjProvider(StorageProvider):
logger.error(f"Storj unpin failed: {e}")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from Storj."""
try:
import asyncio
@@ -782,7 +782,7 @@ class StorjProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
response = s3.get_object(Bucket=self.bucket, Key=content_hash)
response = s3.get_object(Bucket=self.bucket, Key=cid)
return response['Body'].read()
return await asyncio.to_thread(do_get)
@@ -790,7 +790,7 @@ class StorjProvider(StorageProvider):
logger.error(f"Storj get failed: {e}")
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content exists on Storj."""
try:
import asyncio
@@ -805,7 +805,7 @@ class StorjProvider(StorageProvider):
aws_secret_access_key=self.secret_key,
config=Config(signature_version='s3v4')
)
s3.head_object(Bucket=self.bucket, Key=content_hash)
s3.head_object(Bucket=self.bucket, Key=cid)
return True
return await asyncio.to_thread(do_check)
@@ -854,37 +854,37 @@ class LocalStorageProvider(StorageProvider):
# Create directory if it doesn't exist
self.base_path.mkdir(parents=True, exist_ok=True)
def _get_file_path(self, content_hash: str) -> Path:
def _get_file_path(self, cid: str) -> Path:
"""Get file path for a content hash (using subdirectories)."""
# Use first 2 chars as subdirectory for better filesystem performance
subdir = content_hash[:2]
return self.base_path / subdir / content_hash
subdir = cid[:2]
return self.base_path / subdir / cid
async def pin(self, content_hash: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
async def pin(self, cid: str, data: bytes, filename: Optional[str] = None) -> Optional[str]:
"""Store content locally."""
try:
import asyncio
def do_store():
file_path = self._get_file_path(content_hash)
file_path = self._get_file_path(cid)
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_bytes(data)
return content_hash # Use content_hash as ID for local storage
return cid # Use cid as ID for local storage
result = await asyncio.to_thread(do_store)
logger.info(f"Local: Stored {content_hash[:16]}...")
logger.info(f"Local: Stored {cid[:16]}...")
return result
except Exception as e:
logger.error(f"Local storage failed: {e}")
return None
async def unpin(self, content_hash: str) -> bool:
async def unpin(self, cid: str) -> bool:
"""Remove content from local storage."""
try:
import asyncio
def do_remove():
file_path = self._get_file_path(content_hash)
file_path = self._get_file_path(cid)
if file_path.exists():
file_path.unlink()
return True
@@ -895,13 +895,13 @@ class LocalStorageProvider(StorageProvider):
logger.error(f"Local unpin failed: {e}")
return False
async def get(self, content_hash: str) -> Optional[bytes]:
async def get(self, cid: str) -> Optional[bytes]:
"""Get content from local storage."""
try:
import asyncio
def do_get():
file_path = self._get_file_path(content_hash)
file_path = self._get_file_path(cid)
if file_path.exists():
return file_path.read_bytes()
return None
@@ -911,9 +911,9 @@ class LocalStorageProvider(StorageProvider):
logger.error(f"Local get failed: {e}")
return None
async def is_pinned(self, content_hash: str) -> bool:
async def is_pinned(self, cid: str) -> bool:
"""Check if content exists in local storage."""
return self._get_file_path(content_hash).exists()
return self._get_file_path(cid).exists()
async def test_connection(self) -> tuple[bool, str]:
"""Test local storage is writable."""