""" Path Registry - Maps human-friendly paths to content-addressed IDs. This module provides a bidirectional mapping between: - Human-friendly paths (e.g., "effects/ascii_fx_zone.sexp") - Content-addressed IDs (IPFS CIDs or SHA3-256 hashes) The registry is useful for: - Looking up effects by their friendly path name - Resolving cids back to the original path for debugging - Maintaining a stable naming scheme across cache updates Storage: - Uses the existing item_types table in the database (path column) - Caches in Redis for fast lookups across distributed workers The registry uses a system actor (@system@local) for global path mappings, allowing effects to be resolved by path without requiring user context. """ import logging import os from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional, Tuple from dataclasses import dataclass logger = logging.getLogger(__name__) # System actor for global path mappings (effects, recipes, analyzers) SYSTEM_ACTOR = "@system@local" @dataclass class PathEntry: """A registered path with its content-addressed ID.""" path: str # Human-friendly path (relative or normalized) cid: str # Content-addressed ID (IPFS CID or hash) content_type: str # Type: "effect", "recipe", "analyzer", etc. actor_id: str = SYSTEM_ACTOR # Owner (system for global) description: Optional[str] = None created_at: float = 0.0 class PathRegistry: """ Registry for mapping paths to content-addressed IDs. Uses the existing item_types table for persistence and Redis for fast lookups in distributed Celery workers. """ def __init__(self, redis_client=None): self._redis = redis_client self._redis_path_to_cid_key = "artdag:path_to_cid" self._redis_cid_to_path_key = "artdag:cid_to_path" def _run_async(self, coro): """Run async coroutine from sync context.""" import asyncio try: loop = asyncio.get_running_loop() import threading result = [None] error = [None] def run_in_thread(): try: new_loop = asyncio.new_event_loop() asyncio.set_event_loop(new_loop) try: result[0] = new_loop.run_until_complete(coro) finally: new_loop.close() except Exception as e: error[0] = e thread = threading.Thread(target=run_in_thread) thread.start() thread.join(timeout=30) if error[0]: raise error[0] return result[0] except RuntimeError: try: loop = asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) return loop.run_until_complete(coro) def _normalize_path(self, path: str) -> str: """Normalize a path for consistent storage.""" # Remove leading ./ or / path = path.lstrip('./') # Normalize separators path = path.replace('\\', '/') # Remove duplicate slashes while '//' in path: path = path.replace('//', '/') return path def register( self, path: str, cid: str, content_type: str = "effect", actor_id: str = SYSTEM_ACTOR, description: Optional[str] = None, ) -> PathEntry: """ Register a path -> cid mapping. Args: path: Human-friendly path (e.g., "effects/ascii_fx_zone.sexp") cid: Content-addressed ID (IPFS CID or hash) content_type: Type of content ("effect", "recipe", "analyzer") actor_id: Owner (default: system for global mappings) description: Optional description Returns: The created PathEntry """ norm_path = self._normalize_path(path) now = datetime.now(timezone.utc).timestamp() entry = PathEntry( path=norm_path, cid=cid, content_type=content_type, actor_id=actor_id, description=description, created_at=now, ) # Store in database (item_types table) self._save_to_db(entry) # Update Redis cache self._update_redis_cache(norm_path, cid) logger.info(f"Registered path '{norm_path}' -> {cid[:16]}...") return entry def _save_to_db(self, entry: PathEntry): """Save entry to database using item_types table.""" import database async def save(): import asyncpg conn = await asyncpg.connect(database.DATABASE_URL) try: # Ensure cache_item exists await conn.execute( "INSERT INTO cache_items (cid) VALUES ($1) ON CONFLICT DO NOTHING", entry.cid ) # Insert or update item_type with path await conn.execute( """ INSERT INTO item_types (cid, actor_id, type, path, description) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (cid, actor_id, type, path) DO UPDATE SET description = COALESCE(EXCLUDED.description, item_types.description) """, entry.cid, entry.actor_id, entry.content_type, entry.path, entry.description ) finally: await conn.close() try: self._run_async(save()) except Exception as e: logger.warning(f"Failed to save path registry to DB: {e}") def _update_redis_cache(self, path: str, cid: str): """Update Redis cache with mapping.""" if self._redis: try: self._redis.hset(self._redis_path_to_cid_key, path, cid) self._redis.hset(self._redis_cid_to_path_key, cid, path) except Exception as e: logger.warning(f"Failed to update Redis cache: {e}") def get_cid(self, path: str, content_type: str = None) -> Optional[str]: """ Get the cid for a path. Args: path: Human-friendly path content_type: Optional type filter Returns: The cid, or None if not found """ norm_path = self._normalize_path(path) # Try Redis first (fast path) if self._redis: try: val = self._redis.hget(self._redis_path_to_cid_key, norm_path) if val: return val.decode() if isinstance(val, bytes) else val except Exception as e: logger.warning(f"Redis lookup failed: {e}") # Fall back to database return self._get_cid_from_db(norm_path, content_type) def _get_cid_from_db(self, path: str, content_type: str = None) -> Optional[str]: """Get cid from database using item_types table.""" import database async def get(): import asyncpg conn = await asyncpg.connect(database.DATABASE_URL) try: if content_type: row = await conn.fetchrow( "SELECT cid FROM item_types WHERE path = $1 AND type = $2", path, content_type ) else: row = await conn.fetchrow( "SELECT cid FROM item_types WHERE path = $1", path ) return row["cid"] if row else None finally: await conn.close() try: result = self._run_async(get()) # Update Redis cache if found if result and self._redis: self._update_redis_cache(path, result) return result except Exception as e: logger.warning(f"Failed to get from DB: {e}") return None def get_path(self, cid: str) -> Optional[str]: """ Get the path for a cid. Args: cid: Content-addressed ID Returns: The path, or None if not found """ # Try Redis first if self._redis: try: val = self._redis.hget(self._redis_cid_to_path_key, cid) if val: return val.decode() if isinstance(val, bytes) else val except Exception as e: logger.warning(f"Redis lookup failed: {e}") # Fall back to database return self._get_path_from_db(cid) def _get_path_from_db(self, cid: str) -> Optional[str]: """Get path from database using item_types table.""" import database async def get(): import asyncpg conn = await asyncpg.connect(database.DATABASE_URL) try: row = await conn.fetchrow( "SELECT path FROM item_types WHERE cid = $1 AND path IS NOT NULL ORDER BY created_at LIMIT 1", cid ) return row["path"] if row else None finally: await conn.close() try: result = self._run_async(get()) # Update Redis cache if found if result and self._redis: self._update_redis_cache(result, cid) return result except Exception as e: logger.warning(f"Failed to get from DB: {e}") return None def list_by_type(self, content_type: str, actor_id: str = None) -> List[PathEntry]: """ List all entries of a given type. Args: content_type: Type to filter by ("effect", "recipe", etc.) actor_id: Optional actor filter (None = all, SYSTEM_ACTOR = global) Returns: List of PathEntry objects """ import database async def list_entries(): import asyncpg conn = await asyncpg.connect(database.DATABASE_URL) try: if actor_id: rows = await conn.fetch( """ SELECT cid, path, type, actor_id, description, EXTRACT(EPOCH FROM created_at) as created_at FROM item_types WHERE type = $1 AND actor_id = $2 AND path IS NOT NULL ORDER BY path """, content_type, actor_id ) else: rows = await conn.fetch( """ SELECT cid, path, type, actor_id, description, EXTRACT(EPOCH FROM created_at) as created_at FROM item_types WHERE type = $1 AND path IS NOT NULL ORDER BY path """, content_type ) return [ PathEntry( path=row["path"], cid=row["cid"], content_type=row["type"], actor_id=row["actor_id"], description=row["description"], created_at=row["created_at"] or 0, ) for row in rows ] finally: await conn.close() try: return self._run_async(list_entries()) except Exception as e: logger.warning(f"Failed to list from DB: {e}") return [] def delete(self, path: str, content_type: str = None) -> bool: """ Delete a path registration. Args: path: The path to delete content_type: Optional type filter Returns: True if deleted, False if not found """ norm_path = self._normalize_path(path) # Get cid for Redis cleanup cid = self.get_cid(norm_path, content_type) # Delete from database deleted = self._delete_from_db(norm_path, content_type) # Clean up Redis if deleted and cid and self._redis: try: self._redis.hdel(self._redis_path_to_cid_key, norm_path) self._redis.hdel(self._redis_cid_to_path_key, cid) except Exception as e: logger.warning(f"Failed to clean up Redis: {e}") return deleted def _delete_from_db(self, path: str, content_type: str = None) -> bool: """Delete from database.""" import database async def delete(): import asyncpg conn = await asyncpg.connect(database.DATABASE_URL) try: if content_type: result = await conn.execute( "DELETE FROM item_types WHERE path = $1 AND type = $2", path, content_type ) else: result = await conn.execute( "DELETE FROM item_types WHERE path = $1", path ) return "DELETE" in result finally: await conn.close() try: return self._run_async(delete()) except Exception as e: logger.warning(f"Failed to delete from DB: {e}") return False def register_effect( self, path: str, cid: str, description: Optional[str] = None, ) -> PathEntry: """ Convenience method to register an effect. Args: path: Effect path (e.g., "effects/ascii_fx_zone.sexp") cid: IPFS CID of the effect file description: Optional description Returns: The created PathEntry """ return self.register( path=path, cid=cid, content_type="effect", actor_id=SYSTEM_ACTOR, description=description, ) def get_effect_cid(self, path: str) -> Optional[str]: """ Get CID for an effect by path. Args: path: Effect path Returns: IPFS CID or None """ return self.get_cid(path, content_type="effect") def list_effects(self) -> List[PathEntry]: """List all registered effects.""" return self.list_by_type("effect", actor_id=SYSTEM_ACTOR) # Singleton instance _registry: Optional[PathRegistry] = None def get_path_registry() -> PathRegistry: """Get the singleton path registry instance.""" global _registry if _registry is None: import redis from urllib.parse import urlparse redis_url = os.environ.get('REDIS_URL', 'redis://localhost:6379/5') parsed = urlparse(redis_url) redis_client = redis.Redis( host=parsed.hostname or 'localhost', port=parsed.port or 6379, db=int(parsed.path.lstrip('/') or 0), socket_timeout=5, socket_connect_timeout=5 ) _registry = PathRegistry(redis_client=redis_client) return _registry def reset_path_registry(): """Reset the singleton (for testing).""" global _registry _registry = None