art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add PostgreSQL + IPFS backend, rename configs to recipes

Browse Source 
- Add PostgreSQL database for cache metadata storage with schema for
  cache_items, item_types, pin_reasons, and l2_shares tables
- Add IPFS integration as durable backing store (local cache as hot storage)
- Add postgres and ipfs services to docker-compose.yml
- Update cache_manager to upload to IPFS and track CIDs
- Rename all config references to recipe throughout server.py
- Update API endpoints: /configs/* -> /recipes/*
- Update models: ConfigStatus -> RecipeStatus, ConfigRunRequest -> RecipeRunRequest
- Update UI tabs and pages to show Recipes instead of Configs

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-08 14:58:29 +00:00
parent 4639a98231
commit ba244b9ebc
6 changed files with 938 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
cache_manager.py
View File

@@ -7,6 +7,7 @@ Integrates artdag's Cache, ActivityStore, and ActivityManager to provide:
- Activity tracking for runs (input/output/intermediate relationships)
- Deletion rules enforcement (shared items protected)
- L2 ActivityPub integration for "shared" status checks
- IPFS as durable backing store (local cache as hot storage)
"""
import hashlib
@@ -24,6 +25,8 @@ import requests
from artdag import Cache, CacheEntry, DAG, Node, NodeType
from artdag.activities import Activity, ActivityStore, ActivityManager, make_is_shared_fn
import ipfs_client
logger = logging.getLogger(__name__)
@@ -154,6 +157,10 @@ class L1CacheManager:
self._content_index: Dict[str, str] = {}
self._load_content_index()
# IPFS CID index: content_hash -> ipfs_cid
self._ipfs_cids: Dict[str, str] = {}
self._load_ipfs_index()
# Legacy files directory (for files uploaded directly by content_hash)
self.legacy_dir = self.cache_dir / "legacy"
self.legacy_dir.mkdir(parents=True, exist_ok=True)
@@ -181,6 +188,28 @@ class L1CacheManager:
with open(self._index_path(), "w") as f:
json.dump(self._content_index, f, indent=2)
def _ipfs_index_path(self) -> Path:
return self.cache_dir / "ipfs_index.json"
def _load_ipfs_index(self):
"""Load content_hash -> ipfs_cid index."""
if self._ipfs_index_path().exists():
try:
with open(self._ipfs_index_path()) as f:
self._ipfs_cids = json.load(f)
except (json.JSONDecodeError, IOError) as e:
logger.warning(f"Failed to load IPFS index: {e}")
self._ipfs_cids = {}
def _save_ipfs_index(self):
"""Save content_hash -> ipfs_cid index."""
with open(self._ipfs_index_path(), "w") as f:
json.dump(self._ipfs_cids, f, indent=2)
def get_ipfs_cid(self, content_hash: str) -> Optional[str]:
"""Get IPFS CID for a content hash."""
return self._ipfs_cids.get(content_hash)
def _is_shared_by_node_id(self, content_hash: str) -> bool:
"""Check if a content_hash is shared via L2."""
return self.l2_checker.is_shared(content_hash)
@@ -227,9 +256,9 @@ class L1CacheManager:
node_id: str = None,
execution_time: float = 0.0,
move: bool = False,
) -> CachedFile:
) -> tuple[CachedFile, Optional[str]]:
"""
Store a file in the cache.
Store a file in the cache and upload to IPFS.
Args:
source_path: Path to file to cache
@@ -239,7 +268,7 @@ class L1CacheManager:
move: If True, move instead of copy
Returns:
CachedFile with both node_id and content_hash
Tuple of (CachedFile with both node_id and content_hash, IPFS CID or None)
"""
# Compute content hash first
content_hash = file_hash(source_path)
@@ -252,9 +281,16 @@ class L1CacheManager:
# Check if already cached (by node_id)
existing = self.cache.get_entry(node_id)
if existing and existing.output_path.exists():
return CachedFile.from_cache_entry(existing)
# Already cached - still try to get IPFS CID if we don't have it
ipfs_cid = self._ipfs_cids.get(content_hash)
if not ipfs_cid:
ipfs_cid = ipfs_client.add_file(existing.output_path)
if ipfs_cid:
self._ipfs_cids[content_hash] = ipfs_cid
self._save_ipfs_index()
return CachedFile.from_cache_entry(existing), ipfs_cid
# Store in cache
# Store in local cache
self.cache.put(
node_id=node_id,
source_path=source_path,
@@ -269,27 +305,34 @@ class L1CacheManager:
self._content_index[entry.content_hash] = node_id
self._save_content_index()
return CachedFile.from_cache_entry(entry)
# Upload to IPFS (async in background would be better, but sync for now)
ipfs_cid = ipfs_client.add_file(entry.output_path)
if ipfs_cid:
self._ipfs_cids[entry.content_hash] = ipfs_cid
self._save_ipfs_index()
logger.info(f"Uploaded to IPFS: {entry.content_hash[:16]}... -> {ipfs_cid}")
return CachedFile.from_cache_entry(entry), ipfs_cid
def get_by_node_id(self, node_id: str) -> Optional[Path]:
"""Get cached file path by node_id."""
return self.cache.get(node_id)
def get_by_content_hash(self, content_hash: str) -> Optional[Path]:
"""Get cached file path by content_hash."""
"""Get cached file path by content_hash. Falls back to IPFS if not in local cache."""
# Check index first (new cache structure)
node_id = self._content_index.get(content_hash)
if node_id:
path = self.cache.get(node_id)
if path and path.exists():
logger.info(f" Found via index: {path}")
logger.debug(f" Found via index: {path}")
return path
# For uploads, node_id == content_hash, so try direct lookup
# This works even if cache index hasn't been reloaded
path = self.cache.get(content_hash)
logger.info(f" cache.get({content_hash[:16]}...) returned: {path}")
logger.debug(f" cache.get({content_hash[:16]}...) returned: {path}")
if path and path.exists():
self._content_index[content_hash] = content_hash
self._save_content_index()
@@ -298,7 +341,7 @@ class L1CacheManager:
# Scan cache entries (fallback for new structure)
entry = self.cache.find_by_content_hash(content_hash)
if entry and entry.output_path.exists():
logger.info(f" Found via scan: {entry.output_path}")
logger.debug(f" Found via scan: {entry.output_path}")
self._content_index[content_hash] = entry.node_id
self._save_content_index()
return entry.output_path
@@ -308,6 +351,15 @@ class L1CacheManager:
if legacy_path.exists() and legacy_path.is_file():
return legacy_path
# Try to recover from IPFS if we have a CID
ipfs_cid = self._ipfs_cids.get(content_hash)
if ipfs_cid:
logger.info(f"Recovering from IPFS: {content_hash[:16]}... ({ipfs_cid})")
recovery_path = self.legacy_dir / content_hash
if ipfs_client.get_file(ipfs_cid, recovery_path):
logger.info(f"Recovered from IPFS: {recovery_path}")
return recovery_path
return None
def has_content(self, content_hash: str) -> bool: