478 lines
15 KiB
Python
478 lines
15 KiB
Python
"""
|
|
Path Registry - Maps human-friendly paths to content-addressed IDs.
|
|
|
|
This module provides a bidirectional mapping between:
|
|
- Human-friendly paths (e.g., "effects/ascii_fx_zone.sexp")
|
|
- Content-addressed IDs (IPFS CIDs or SHA3-256 hashes)
|
|
|
|
The registry is useful for:
|
|
- Looking up effects by their friendly path name
|
|
- Resolving cids back to the original path for debugging
|
|
- Maintaining a stable naming scheme across cache updates
|
|
|
|
Storage:
|
|
- Uses the existing item_types table in the database (path column)
|
|
- Caches in Redis for fast lookups across distributed workers
|
|
|
|
The registry uses a system actor (@system@local) for global path mappings,
|
|
allowing effects to be resolved by path without requiring user context.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
from dataclasses import dataclass
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# System actor for global path mappings (effects, recipes, analyzers)
|
|
SYSTEM_ACTOR = "@system@local"
|
|
|
|
|
|
@dataclass
|
|
class PathEntry:
|
|
"""A registered path with its content-addressed ID."""
|
|
path: str # Human-friendly path (relative or normalized)
|
|
cid: str # Content-addressed ID (IPFS CID or hash)
|
|
content_type: str # Type: "effect", "recipe", "analyzer", etc.
|
|
actor_id: str = SYSTEM_ACTOR # Owner (system for global)
|
|
description: Optional[str] = None
|
|
created_at: float = 0.0
|
|
|
|
|
|
class PathRegistry:
|
|
"""
|
|
Registry for mapping paths to content-addressed IDs.
|
|
|
|
Uses the existing item_types table for persistence and Redis
|
|
for fast lookups in distributed Celery workers.
|
|
"""
|
|
|
|
def __init__(self, redis_client=None):
|
|
self._redis = redis_client
|
|
self._redis_path_to_cid_key = "artdag:path_to_cid"
|
|
self._redis_cid_to_path_key = "artdag:cid_to_path"
|
|
|
|
def _run_async(self, coro):
|
|
"""Run async coroutine from sync context."""
|
|
import asyncio
|
|
|
|
try:
|
|
loop = asyncio.get_running_loop()
|
|
import threading
|
|
result = [None]
|
|
error = [None]
|
|
|
|
def run_in_thread():
|
|
try:
|
|
new_loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(new_loop)
|
|
try:
|
|
result[0] = new_loop.run_until_complete(coro)
|
|
finally:
|
|
new_loop.close()
|
|
except Exception as e:
|
|
error[0] = e
|
|
|
|
thread = threading.Thread(target=run_in_thread)
|
|
thread.start()
|
|
thread.join(timeout=30)
|
|
if error[0]:
|
|
raise error[0]
|
|
return result[0]
|
|
except RuntimeError:
|
|
try:
|
|
loop = asyncio.get_event_loop()
|
|
except RuntimeError:
|
|
loop = asyncio.new_event_loop()
|
|
asyncio.set_event_loop(loop)
|
|
return loop.run_until_complete(coro)
|
|
|
|
def _normalize_path(self, path: str) -> str:
|
|
"""Normalize a path for consistent storage."""
|
|
# Remove leading ./ or /
|
|
path = path.lstrip('./')
|
|
# Normalize separators
|
|
path = path.replace('\\', '/')
|
|
# Remove duplicate slashes
|
|
while '//' in path:
|
|
path = path.replace('//', '/')
|
|
return path
|
|
|
|
def register(
|
|
self,
|
|
path: str,
|
|
cid: str,
|
|
content_type: str = "effect",
|
|
actor_id: str = SYSTEM_ACTOR,
|
|
description: Optional[str] = None,
|
|
) -> PathEntry:
|
|
"""
|
|
Register a path -> cid mapping.
|
|
|
|
Args:
|
|
path: Human-friendly path (e.g., "effects/ascii_fx_zone.sexp")
|
|
cid: Content-addressed ID (IPFS CID or hash)
|
|
content_type: Type of content ("effect", "recipe", "analyzer")
|
|
actor_id: Owner (default: system for global mappings)
|
|
description: Optional description
|
|
|
|
Returns:
|
|
The created PathEntry
|
|
"""
|
|
norm_path = self._normalize_path(path)
|
|
now = datetime.now(timezone.utc).timestamp()
|
|
|
|
entry = PathEntry(
|
|
path=norm_path,
|
|
cid=cid,
|
|
content_type=content_type,
|
|
actor_id=actor_id,
|
|
description=description,
|
|
created_at=now,
|
|
)
|
|
|
|
# Store in database (item_types table)
|
|
self._save_to_db(entry)
|
|
|
|
# Update Redis cache
|
|
self._update_redis_cache(norm_path, cid)
|
|
|
|
logger.info(f"Registered path '{norm_path}' -> {cid[:16]}...")
|
|
return entry
|
|
|
|
def _save_to_db(self, entry: PathEntry):
|
|
"""Save entry to database using item_types table."""
|
|
import database
|
|
|
|
async def save():
|
|
import asyncpg
|
|
conn = await asyncpg.connect(database.DATABASE_URL)
|
|
try:
|
|
# Ensure cache_item exists
|
|
await conn.execute(
|
|
"INSERT INTO cache_items (cid) VALUES ($1) ON CONFLICT DO NOTHING",
|
|
entry.cid
|
|
)
|
|
# Insert or update item_type with path
|
|
await conn.execute(
|
|
"""
|
|
INSERT INTO item_types (cid, actor_id, type, path, description)
|
|
VALUES ($1, $2, $3, $4, $5)
|
|
ON CONFLICT (cid, actor_id, type, path) DO UPDATE SET
|
|
description = COALESCE(EXCLUDED.description, item_types.description)
|
|
""",
|
|
entry.cid, entry.actor_id, entry.content_type, entry.path, entry.description
|
|
)
|
|
finally:
|
|
await conn.close()
|
|
|
|
try:
|
|
self._run_async(save())
|
|
except Exception as e:
|
|
logger.warning(f"Failed to save path registry to DB: {e}")
|
|
|
|
def _update_redis_cache(self, path: str, cid: str):
|
|
"""Update Redis cache with mapping."""
|
|
if self._redis:
|
|
try:
|
|
self._redis.hset(self._redis_path_to_cid_key, path, cid)
|
|
self._redis.hset(self._redis_cid_to_path_key, cid, path)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to update Redis cache: {e}")
|
|
|
|
def get_cid(self, path: str, content_type: str = None) -> Optional[str]:
|
|
"""
|
|
Get the cid for a path.
|
|
|
|
Args:
|
|
path: Human-friendly path
|
|
content_type: Optional type filter
|
|
|
|
Returns:
|
|
The cid, or None if not found
|
|
"""
|
|
norm_path = self._normalize_path(path)
|
|
|
|
# Try Redis first (fast path)
|
|
if self._redis:
|
|
try:
|
|
val = self._redis.hget(self._redis_path_to_cid_key, norm_path)
|
|
if val:
|
|
return val.decode() if isinstance(val, bytes) else val
|
|
except Exception as e:
|
|
logger.warning(f"Redis lookup failed: {e}")
|
|
|
|
# Fall back to database
|
|
return self._get_cid_from_db(norm_path, content_type)
|
|
|
|
def _get_cid_from_db(self, path: str, content_type: str = None) -> Optional[str]:
|
|
"""Get cid from database using item_types table."""
|
|
import database
|
|
|
|
async def get():
|
|
import asyncpg
|
|
conn = await asyncpg.connect(database.DATABASE_URL)
|
|
try:
|
|
if content_type:
|
|
row = await conn.fetchrow(
|
|
"SELECT cid FROM item_types WHERE path = $1 AND type = $2",
|
|
path, content_type
|
|
)
|
|
else:
|
|
row = await conn.fetchrow(
|
|
"SELECT cid FROM item_types WHERE path = $1",
|
|
path
|
|
)
|
|
return row["cid"] if row else None
|
|
finally:
|
|
await conn.close()
|
|
|
|
try:
|
|
result = self._run_async(get())
|
|
# Update Redis cache if found
|
|
if result and self._redis:
|
|
self._update_redis_cache(path, result)
|
|
return result
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get from DB: {e}")
|
|
return None
|
|
|
|
def get_path(self, cid: str) -> Optional[str]:
|
|
"""
|
|
Get the path for a cid.
|
|
|
|
Args:
|
|
cid: Content-addressed ID
|
|
|
|
Returns:
|
|
The path, or None if not found
|
|
"""
|
|
# Try Redis first
|
|
if self._redis:
|
|
try:
|
|
val = self._redis.hget(self._redis_cid_to_path_key, cid)
|
|
if val:
|
|
return val.decode() if isinstance(val, bytes) else val
|
|
except Exception as e:
|
|
logger.warning(f"Redis lookup failed: {e}")
|
|
|
|
# Fall back to database
|
|
return self._get_path_from_db(cid)
|
|
|
|
def _get_path_from_db(self, cid: str) -> Optional[str]:
|
|
"""Get path from database using item_types table."""
|
|
import database
|
|
|
|
async def get():
|
|
import asyncpg
|
|
conn = await asyncpg.connect(database.DATABASE_URL)
|
|
try:
|
|
row = await conn.fetchrow(
|
|
"SELECT path FROM item_types WHERE cid = $1 AND path IS NOT NULL ORDER BY created_at LIMIT 1",
|
|
cid
|
|
)
|
|
return row["path"] if row else None
|
|
finally:
|
|
await conn.close()
|
|
|
|
try:
|
|
result = self._run_async(get())
|
|
# Update Redis cache if found
|
|
if result and self._redis:
|
|
self._update_redis_cache(result, cid)
|
|
return result
|
|
except Exception as e:
|
|
logger.warning(f"Failed to get from DB: {e}")
|
|
return None
|
|
|
|
def list_by_type(self, content_type: str, actor_id: str = None) -> List[PathEntry]:
|
|
"""
|
|
List all entries of a given type.
|
|
|
|
Args:
|
|
content_type: Type to filter by ("effect", "recipe", etc.)
|
|
actor_id: Optional actor filter (None = all, SYSTEM_ACTOR = global)
|
|
|
|
Returns:
|
|
List of PathEntry objects
|
|
"""
|
|
import database
|
|
|
|
async def list_entries():
|
|
import asyncpg
|
|
conn = await asyncpg.connect(database.DATABASE_URL)
|
|
try:
|
|
if actor_id:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT cid, path, type, actor_id, description,
|
|
EXTRACT(EPOCH FROM created_at) as created_at
|
|
FROM item_types
|
|
WHERE type = $1 AND actor_id = $2 AND path IS NOT NULL
|
|
ORDER BY path
|
|
""",
|
|
content_type, actor_id
|
|
)
|
|
else:
|
|
rows = await conn.fetch(
|
|
"""
|
|
SELECT cid, path, type, actor_id, description,
|
|
EXTRACT(EPOCH FROM created_at) as created_at
|
|
FROM item_types
|
|
WHERE type = $1 AND path IS NOT NULL
|
|
ORDER BY path
|
|
""",
|
|
content_type
|
|
)
|
|
return [
|
|
PathEntry(
|
|
path=row["path"],
|
|
cid=row["cid"],
|
|
content_type=row["type"],
|
|
actor_id=row["actor_id"],
|
|
description=row["description"],
|
|
created_at=row["created_at"] or 0,
|
|
)
|
|
for row in rows
|
|
]
|
|
finally:
|
|
await conn.close()
|
|
|
|
try:
|
|
return self._run_async(list_entries())
|
|
except Exception as e:
|
|
logger.warning(f"Failed to list from DB: {e}")
|
|
return []
|
|
|
|
def delete(self, path: str, content_type: str = None) -> bool:
|
|
"""
|
|
Delete a path registration.
|
|
|
|
Args:
|
|
path: The path to delete
|
|
content_type: Optional type filter
|
|
|
|
Returns:
|
|
True if deleted, False if not found
|
|
"""
|
|
norm_path = self._normalize_path(path)
|
|
|
|
# Get cid for Redis cleanup
|
|
cid = self.get_cid(norm_path, content_type)
|
|
|
|
# Delete from database
|
|
deleted = self._delete_from_db(norm_path, content_type)
|
|
|
|
# Clean up Redis
|
|
if deleted and cid and self._redis:
|
|
try:
|
|
self._redis.hdel(self._redis_path_to_cid_key, norm_path)
|
|
self._redis.hdel(self._redis_cid_to_path_key, cid)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to clean up Redis: {e}")
|
|
|
|
return deleted
|
|
|
|
def _delete_from_db(self, path: str, content_type: str = None) -> bool:
|
|
"""Delete from database."""
|
|
import database
|
|
|
|
async def delete():
|
|
import asyncpg
|
|
conn = await asyncpg.connect(database.DATABASE_URL)
|
|
try:
|
|
if content_type:
|
|
result = await conn.execute(
|
|
"DELETE FROM item_types WHERE path = $1 AND type = $2",
|
|
path, content_type
|
|
)
|
|
else:
|
|
result = await conn.execute(
|
|
"DELETE FROM item_types WHERE path = $1",
|
|
path
|
|
)
|
|
return "DELETE" in result
|
|
finally:
|
|
await conn.close()
|
|
|
|
try:
|
|
return self._run_async(delete())
|
|
except Exception as e:
|
|
logger.warning(f"Failed to delete from DB: {e}")
|
|
return False
|
|
|
|
def register_effect(
|
|
self,
|
|
path: str,
|
|
cid: str,
|
|
description: Optional[str] = None,
|
|
) -> PathEntry:
|
|
"""
|
|
Convenience method to register an effect.
|
|
|
|
Args:
|
|
path: Effect path (e.g., "effects/ascii_fx_zone.sexp")
|
|
cid: IPFS CID of the effect file
|
|
description: Optional description
|
|
|
|
Returns:
|
|
The created PathEntry
|
|
"""
|
|
return self.register(
|
|
path=path,
|
|
cid=cid,
|
|
content_type="effect",
|
|
actor_id=SYSTEM_ACTOR,
|
|
description=description,
|
|
)
|
|
|
|
def get_effect_cid(self, path: str) -> Optional[str]:
|
|
"""
|
|
Get CID for an effect by path.
|
|
|
|
Args:
|
|
path: Effect path
|
|
|
|
Returns:
|
|
IPFS CID or None
|
|
"""
|
|
return self.get_cid(path, content_type="effect")
|
|
|
|
def list_effects(self) -> List[PathEntry]:
|
|
"""List all registered effects."""
|
|
return self.list_by_type("effect", actor_id=SYSTEM_ACTOR)
|
|
|
|
|
|
# Singleton instance
|
|
_registry: Optional[PathRegistry] = None
|
|
|
|
|
|
def get_path_registry() -> PathRegistry:
|
|
"""Get the singleton path registry instance."""
|
|
global _registry
|
|
if _registry is None:
|
|
import redis
|
|
from urllib.parse import urlparse
|
|
|
|
redis_url = os.environ.get('REDIS_URL', 'redis://localhost:6379/5')
|
|
parsed = urlparse(redis_url)
|
|
redis_client = redis.Redis(
|
|
host=parsed.hostname or 'localhost',
|
|
port=parsed.port or 6379,
|
|
db=int(parsed.path.lstrip('/') or 0),
|
|
socket_timeout=5,
|
|
socket_connect_timeout=5
|
|
)
|
|
|
|
_registry = PathRegistry(redis_client=redis_client)
|
|
return _registry
|
|
|
|
|
|
def reset_path_registry():
|
|
"""Reset the singleton (for testing)."""
|
|
global _registry
|
|
_registry = None
|