Add hybrid state manager for distributed L1 coordination
Implements HybridStateManager providing fast local Redis operations with background IPNS sync for eventual consistency across L1 nodes. - hybrid_state.py: Centralized state management (cache, claims, analysis, plans, runs) - Updated execute_cid.py, analyze_cid.py, orchestrate_cid.py to use state manager - Background IPNS sync (configurable interval, disabled by default) - Atomic claiming with Redis SETNX for preventing duplicate work Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,10 @@
|
||||
IPFS-primary analysis tasks.
|
||||
|
||||
Fetches inputs from IPFS, stores analysis results on IPFS.
|
||||
|
||||
Uses HybridStateManager for:
|
||||
- Fast local Redis operations
|
||||
- Background IPNS sync with other L1 nodes
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -18,17 +22,7 @@ import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from celery_app import app
|
||||
import ipfs_client
|
||||
|
||||
# Redis for caching analysis CIDs
|
||||
import redis
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/5")
|
||||
_redis: Optional[redis.Redis] = None
|
||||
|
||||
def get_redis() -> redis.Redis:
|
||||
global _redis
|
||||
if _redis is None:
|
||||
_redis = redis.from_url(REDIS_URL, decode_responses=True)
|
||||
return _redis
|
||||
from hybrid_state import get_state_manager
|
||||
|
||||
# Import artdag analysis module
|
||||
try:
|
||||
@@ -39,26 +33,15 @@ except ImportError:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis key for analysis cache
|
||||
ANALYSIS_CACHE_KEY = "artdag:analysis_cid" # hash: input_hash:features → analysis CID
|
||||
|
||||
|
||||
def get_analysis_cache_key(input_hash: str, features: List[str]) -> str:
|
||||
"""Generate cache key for analysis results."""
|
||||
features_str = ",".join(sorted(features))
|
||||
return f"{input_hash}:{features_str}"
|
||||
|
||||
|
||||
def get_cached_analysis_cid(input_hash: str, features: List[str]) -> Optional[str]:
|
||||
"""Check if analysis is already cached."""
|
||||
key = get_analysis_cache_key(input_hash, features)
|
||||
return get_redis().hget(ANALYSIS_CACHE_KEY, key)
|
||||
return get_state_manager().get_analysis_cid(input_hash, features)
|
||||
|
||||
|
||||
def set_cached_analysis_cid(input_hash: str, features: List[str], cid: str) -> None:
|
||||
"""Store analysis CID in cache."""
|
||||
key = get_analysis_cache_key(input_hash, features)
|
||||
get_redis().hset(ANALYSIS_CACHE_KEY, key, cid)
|
||||
get_state_manager().set_analysis_cid(input_hash, features, cid)
|
||||
|
||||
|
||||
@app.task(bind=True, name='tasks.analyze_input_cid')
|
||||
|
||||
Reference in New Issue
Block a user