Add hybrid state manager for distributed L1 coordination

Implements HybridStateManager providing fast local Redis operations
with background IPNS sync for eventual consistency across L1 nodes.

- hybrid_state.py: Centralized state management (cache, claims, analysis, plans, runs)
- Updated execute_cid.py, analyze_cid.py, orchestrate_cid.py to use state manager
- Background IPNS sync (configurable interval, disabled by default)
- Atomic claiming with Redis SETNX for preventing duplicate work

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-11 09:35:50 +00:00
parent f11cec9d48
commit ca8bfd8705
4 changed files with 319 additions and 67 deletions

View File

@@ -2,6 +2,10 @@
IPFS-primary analysis tasks.
Fetches inputs from IPFS, stores analysis results on IPFS.
Uses HybridStateManager for:
- Fast local Redis operations
- Background IPNS sync with other L1 nodes
"""
import json
@@ -18,17 +22,7 @@ import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from celery_app import app
import ipfs_client
# Redis for caching analysis CIDs
import redis
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/5")
_redis: Optional[redis.Redis] = None
def get_redis() -> redis.Redis:
global _redis
if _redis is None:
_redis = redis.from_url(REDIS_URL, decode_responses=True)
return _redis
from hybrid_state import get_state_manager
# Import artdag analysis module
try:
@@ -39,26 +33,15 @@ except ImportError:
logger = logging.getLogger(__name__)
# Redis key for analysis cache
ANALYSIS_CACHE_KEY = "artdag:analysis_cid" # hash: input_hash:features → analysis CID
def get_analysis_cache_key(input_hash: str, features: List[str]) -> str:
"""Generate cache key for analysis results."""
features_str = ",".join(sorted(features))
return f"{input_hash}:{features_str}"
def get_cached_analysis_cid(input_hash: str, features: List[str]) -> Optional[str]:
"""Check if analysis is already cached."""
key = get_analysis_cache_key(input_hash, features)
return get_redis().hget(ANALYSIS_CACHE_KEY, key)
return get_state_manager().get_analysis_cid(input_hash, features)
def set_cached_analysis_cid(input_hash: str, features: List[str], cid: str) -> None:
"""Store analysis CID in cache."""
key = get_analysis_cache_key(input_hash, features)
get_redis().hset(ANALYSIS_CACHE_KEY, key, cid)
get_state_manager().set_analysis_cid(input_hash, features, cid)
@app.task(bind=True, name='tasks.analyze_input_cid')