Add JAX typography, xector primitives, deferred effect chains, and GPU streaming
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 1m28s
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 1m28s
- Add JAX text rendering with font atlas, styled text placement, and typography primitives - Add xector (element-wise/reduction) operations library and sexp effects - Add deferred effect chain fusion for JIT-compiled effect pipelines - Expand drawing primitives with font management, alignment, shadow, and outline - Add interpreter support for function-style define and require - Add GPU persistence mode and hardware decode support to streaming - Add new sexp effects: cell_pattern, halftone, mosaic, and derived definitions - Add path registry for asset resolution - Add integration, primitives, and xector tests Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
gilesb
477
path_registry.py
Normal file
477
path_registry.py
Normal file
@@ -0,0 +1,477 @@
|
||||
"""
|
||||
Path Registry - Maps human-friendly paths to content-addressed IDs.
|
||||
|
||||
This module provides a bidirectional mapping between:
|
||||
- Human-friendly paths (e.g., "effects/ascii_fx_zone.sexp")
|
||||
- Content-addressed IDs (IPFS CIDs or SHA3-256 hashes)
|
||||
|
||||
The registry is useful for:
|
||||
- Looking up effects by their friendly path name
|
||||
- Resolving cids back to the original path for debugging
|
||||
- Maintaining a stable naming scheme across cache updates
|
||||
|
||||
Storage:
|
||||
- Uses the existing item_types table in the database (path column)
|
||||
- Caches in Redis for fast lookups across distributed workers
|
||||
|
||||
The registry uses a system actor (@system@local) for global path mappings,
|
||||
allowing effects to be resolved by path without requiring user context.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from dataclasses import dataclass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# System actor for global path mappings (effects, recipes, analyzers)
|
||||
SYSTEM_ACTOR = "@system@local"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PathEntry:
|
||||
"""A registered path with its content-addressed ID."""
|
||||
path: str # Human-friendly path (relative or normalized)
|
||||
cid: str # Content-addressed ID (IPFS CID or hash)
|
||||
content_type: str # Type: "effect", "recipe", "analyzer", etc.
|
||||
actor_id: str = SYSTEM_ACTOR # Owner (system for global)
|
||||
description: Optional[str] = None
|
||||
created_at: float = 0.0
|
||||
|
||||
|
||||
class PathRegistry:
|
||||
"""
|
||||
Registry for mapping paths to content-addressed IDs.
|
||||
|
||||
Uses the existing item_types table for persistence and Redis
|
||||
for fast lookups in distributed Celery workers.
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client=None):
|
||||
self._redis = redis_client
|
||||
self._redis_path_to_cid_key = "artdag:path_to_cid"
|
||||
self._redis_cid_to_path_key = "artdag:cid_to_path"
|
||||
|
||||
def _run_async(self, coro):
|
||||
"""Run async coroutine from sync context."""
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
import threading
|
||||
result = [None]
|
||||
error = [None]
|
||||
|
||||
def run_in_thread():
|
||||
try:
|
||||
new_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(new_loop)
|
||||
try:
|
||||
result[0] = new_loop.run_until_complete(coro)
|
||||
finally:
|
||||
new_loop.close()
|
||||
except Exception as e:
|
||||
error[0] = e
|
||||
|
||||
thread = threading.Thread(target=run_in_thread)
|
||||
thread.start()
|
||||
thread.join(timeout=30)
|
||||
if error[0]:
|
||||
raise error[0]
|
||||
return result[0]
|
||||
except RuntimeError:
|
||||
try:
|
||||
loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
return loop.run_until_complete(coro)
|
||||
|
||||
def _normalize_path(self, path: str) -> str:
|
||||
"""Normalize a path for consistent storage."""
|
||||
# Remove leading ./ or /
|
||||
path = path.lstrip('./')
|
||||
# Normalize separators
|
||||
path = path.replace('\\', '/')
|
||||
# Remove duplicate slashes
|
||||
while '//' in path:
|
||||
path = path.replace('//', '/')
|
||||
return path
|
||||
|
||||
def register(
|
||||
self,
|
||||
path: str,
|
||||
cid: str,
|
||||
content_type: str = "effect",
|
||||
actor_id: str = SYSTEM_ACTOR,
|
||||
description: Optional[str] = None,
|
||||
) -> PathEntry:
|
||||
"""
|
||||
Register a path -> cid mapping.
|
||||
|
||||
Args:
|
||||
path: Human-friendly path (e.g., "effects/ascii_fx_zone.sexp")
|
||||
cid: Content-addressed ID (IPFS CID or hash)
|
||||
content_type: Type of content ("effect", "recipe", "analyzer")
|
||||
actor_id: Owner (default: system for global mappings)
|
||||
description: Optional description
|
||||
|
||||
Returns:
|
||||
The created PathEntry
|
||||
"""
|
||||
norm_path = self._normalize_path(path)
|
||||
now = datetime.now(timezone.utc).timestamp()
|
||||
|
||||
entry = PathEntry(
|
||||
path=norm_path,
|
||||
cid=cid,
|
||||
content_type=content_type,
|
||||
actor_id=actor_id,
|
||||
description=description,
|
||||
created_at=now,
|
||||
)
|
||||
|
||||
# Store in database (item_types table)
|
||||
self._save_to_db(entry)
|
||||
|
||||
# Update Redis cache
|
||||
self._update_redis_cache(norm_path, cid)
|
||||
|
||||
logger.info(f"Registered path '{norm_path}' -> {cid[:16]}...")
|
||||
return entry
|
||||
|
||||
def _save_to_db(self, entry: PathEntry):
|
||||
"""Save entry to database using item_types table."""
|
||||
import database
|
||||
|
||||
async def save():
|
||||
import asyncpg
|
||||
conn = await asyncpg.connect(database.DATABASE_URL)
|
||||
try:
|
||||
# Ensure cache_item exists
|
||||
await conn.execute(
|
||||
"INSERT INTO cache_items (cid) VALUES ($1) ON CONFLICT DO NOTHING",
|
||||
entry.cid
|
||||
)
|
||||
# Insert or update item_type with path
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO item_types (cid, actor_id, type, path, description)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (cid, actor_id, type, path) DO UPDATE SET
|
||||
description = COALESCE(EXCLUDED.description, item_types.description)
|
||||
""",
|
||||
entry.cid, entry.actor_id, entry.content_type, entry.path, entry.description
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
try:
|
||||
self._run_async(save())
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save path registry to DB: {e}")
|
||||
|
||||
def _update_redis_cache(self, path: str, cid: str):
|
||||
"""Update Redis cache with mapping."""
|
||||
if self._redis:
|
||||
try:
|
||||
self._redis.hset(self._redis_path_to_cid_key, path, cid)
|
||||
self._redis.hset(self._redis_cid_to_path_key, cid, path)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to update Redis cache: {e}")
|
||||
|
||||
def get_cid(self, path: str, content_type: str = None) -> Optional[str]:
|
||||
"""
|
||||
Get the cid for a path.
|
||||
|
||||
Args:
|
||||
path: Human-friendly path
|
||||
content_type: Optional type filter
|
||||
|
||||
Returns:
|
||||
The cid, or None if not found
|
||||
"""
|
||||
norm_path = self._normalize_path(path)
|
||||
|
||||
# Try Redis first (fast path)
|
||||
if self._redis:
|
||||
try:
|
||||
val = self._redis.hget(self._redis_path_to_cid_key, norm_path)
|
||||
if val:
|
||||
return val.decode() if isinstance(val, bytes) else val
|
||||
except Exception as e:
|
||||
logger.warning(f"Redis lookup failed: {e}")
|
||||
|
||||
# Fall back to database
|
||||
return self._get_cid_from_db(norm_path, content_type)
|
||||
|
||||
def _get_cid_from_db(self, path: str, content_type: str = None) -> Optional[str]:
|
||||
"""Get cid from database using item_types table."""
|
||||
import database
|
||||
|
||||
async def get():
|
||||
import asyncpg
|
||||
conn = await asyncpg.connect(database.DATABASE_URL)
|
||||
try:
|
||||
if content_type:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT cid FROM item_types WHERE path = $1 AND type = $2",
|
||||
path, content_type
|
||||
)
|
||||
else:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT cid FROM item_types WHERE path = $1",
|
||||
path
|
||||
)
|
||||
return row["cid"] if row else None
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
try:
|
||||
result = self._run_async(get())
|
||||
# Update Redis cache if found
|
||||
if result and self._redis:
|
||||
self._update_redis_cache(path, result)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get from DB: {e}")
|
||||
return None
|
||||
|
||||
def get_path(self, cid: str) -> Optional[str]:
|
||||
"""
|
||||
Get the path for a cid.
|
||||
|
||||
Args:
|
||||
cid: Content-addressed ID
|
||||
|
||||
Returns:
|
||||
The path, or None if not found
|
||||
"""
|
||||
# Try Redis first
|
||||
if self._redis:
|
||||
try:
|
||||
val = self._redis.hget(self._redis_cid_to_path_key, cid)
|
||||
if val:
|
||||
return val.decode() if isinstance(val, bytes) else val
|
||||
except Exception as e:
|
||||
logger.warning(f"Redis lookup failed: {e}")
|
||||
|
||||
# Fall back to database
|
||||
return self._get_path_from_db(cid)
|
||||
|
||||
def _get_path_from_db(self, cid: str) -> Optional[str]:
|
||||
"""Get path from database using item_types table."""
|
||||
import database
|
||||
|
||||
async def get():
|
||||
import asyncpg
|
||||
conn = await asyncpg.connect(database.DATABASE_URL)
|
||||
try:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT path FROM item_types WHERE cid = $1 AND path IS NOT NULL ORDER BY created_at LIMIT 1",
|
||||
cid
|
||||
)
|
||||
return row["path"] if row else None
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
try:
|
||||
result = self._run_async(get())
|
||||
# Update Redis cache if found
|
||||
if result and self._redis:
|
||||
self._update_redis_cache(result, cid)
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to get from DB: {e}")
|
||||
return None
|
||||
|
||||
def list_by_type(self, content_type: str, actor_id: str = None) -> List[PathEntry]:
|
||||
"""
|
||||
List all entries of a given type.
|
||||
|
||||
Args:
|
||||
content_type: Type to filter by ("effect", "recipe", etc.)
|
||||
actor_id: Optional actor filter (None = all, SYSTEM_ACTOR = global)
|
||||
|
||||
Returns:
|
||||
List of PathEntry objects
|
||||
"""
|
||||
import database
|
||||
|
||||
async def list_entries():
|
||||
import asyncpg
|
||||
conn = await asyncpg.connect(database.DATABASE_URL)
|
||||
try:
|
||||
if actor_id:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT cid, path, type, actor_id, description,
|
||||
EXTRACT(EPOCH FROM created_at) as created_at
|
||||
FROM item_types
|
||||
WHERE type = $1 AND actor_id = $2 AND path IS NOT NULL
|
||||
ORDER BY path
|
||||
""",
|
||||
content_type, actor_id
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT cid, path, type, actor_id, description,
|
||||
EXTRACT(EPOCH FROM created_at) as created_at
|
||||
FROM item_types
|
||||
WHERE type = $1 AND path IS NOT NULL
|
||||
ORDER BY path
|
||||
""",
|
||||
content_type
|
||||
)
|
||||
return [
|
||||
PathEntry(
|
||||
path=row["path"],
|
||||
cid=row["cid"],
|
||||
content_type=row["type"],
|
||||
actor_id=row["actor_id"],
|
||||
description=row["description"],
|
||||
created_at=row["created_at"] or 0,
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
try:
|
||||
return self._run_async(list_entries())
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to list from DB: {e}")
|
||||
return []
|
||||
|
||||
def delete(self, path: str, content_type: str = None) -> bool:
|
||||
"""
|
||||
Delete a path registration.
|
||||
|
||||
Args:
|
||||
path: The path to delete
|
||||
content_type: Optional type filter
|
||||
|
||||
Returns:
|
||||
True if deleted, False if not found
|
||||
"""
|
||||
norm_path = self._normalize_path(path)
|
||||
|
||||
# Get cid for Redis cleanup
|
||||
cid = self.get_cid(norm_path, content_type)
|
||||
|
||||
# Delete from database
|
||||
deleted = self._delete_from_db(norm_path, content_type)
|
||||
|
||||
# Clean up Redis
|
||||
if deleted and cid and self._redis:
|
||||
try:
|
||||
self._redis.hdel(self._redis_path_to_cid_key, norm_path)
|
||||
self._redis.hdel(self._redis_cid_to_path_key, cid)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up Redis: {e}")
|
||||
|
||||
return deleted
|
||||
|
||||
def _delete_from_db(self, path: str, content_type: str = None) -> bool:
|
||||
"""Delete from database."""
|
||||
import database
|
||||
|
||||
async def delete():
|
||||
import asyncpg
|
||||
conn = await asyncpg.connect(database.DATABASE_URL)
|
||||
try:
|
||||
if content_type:
|
||||
result = await conn.execute(
|
||||
"DELETE FROM item_types WHERE path = $1 AND type = $2",
|
||||
path, content_type
|
||||
)
|
||||
else:
|
||||
result = await conn.execute(
|
||||
"DELETE FROM item_types WHERE path = $1",
|
||||
path
|
||||
)
|
||||
return "DELETE" in result
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
try:
|
||||
return self._run_async(delete())
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to delete from DB: {e}")
|
||||
return False
|
||||
|
||||
def register_effect(
|
||||
self,
|
||||
path: str,
|
||||
cid: str,
|
||||
description: Optional[str] = None,
|
||||
) -> PathEntry:
|
||||
"""
|
||||
Convenience method to register an effect.
|
||||
|
||||
Args:
|
||||
path: Effect path (e.g., "effects/ascii_fx_zone.sexp")
|
||||
cid: IPFS CID of the effect file
|
||||
description: Optional description
|
||||
|
||||
Returns:
|
||||
The created PathEntry
|
||||
"""
|
||||
return self.register(
|
||||
path=path,
|
||||
cid=cid,
|
||||
content_type="effect",
|
||||
actor_id=SYSTEM_ACTOR,
|
||||
description=description,
|
||||
)
|
||||
|
||||
def get_effect_cid(self, path: str) -> Optional[str]:
|
||||
"""
|
||||
Get CID for an effect by path.
|
||||
|
||||
Args:
|
||||
path: Effect path
|
||||
|
||||
Returns:
|
||||
IPFS CID or None
|
||||
"""
|
||||
return self.get_cid(path, content_type="effect")
|
||||
|
||||
def list_effects(self) -> List[PathEntry]:
|
||||
"""List all registered effects."""
|
||||
return self.list_by_type("effect", actor_id=SYSTEM_ACTOR)
|
||||
|
||||
|
||||
# Singleton instance
|
||||
_registry: Optional[PathRegistry] = None
|
||||
|
||||
|
||||
def get_path_registry() -> PathRegistry:
|
||||
"""Get the singleton path registry instance."""
|
||||
global _registry
|
||||
if _registry is None:
|
||||
import redis
|
||||
from urllib.parse import urlparse
|
||||
|
||||
redis_url = os.environ.get('REDIS_URL', 'redis://localhost:6379/5')
|
||||
parsed = urlparse(redis_url)
|
||||
redis_client = redis.Redis(
|
||||
host=parsed.hostname or 'localhost',
|
||||
port=parsed.port or 6379,
|
||||
db=int(parsed.path.lstrip('/') or 0),
|
||||
socket_timeout=5,
|
||||
socket_connect_timeout=5
|
||||
)
|
||||
|
||||
_registry = PathRegistry(redis_client=redis_client)
|
||||
return _registry
|
||||
|
||||
|
||||
def reset_path_registry():
|
||||
"""Reset the singleton (for testing)."""
|
||||
global _registry
|
||||
_registry = None
|
||||
Reference in New Issue
Block a user