Integrate artdag cache with deletion rules
- Add cache_manager.py with L1CacheManager wrapping artdag Cache
- Add L2SharedChecker for checking published status via L2 API
- Update server.py to use cache_manager for storage
- Update DELETE /cache/{content_hash} to enforce deletion rules
- Add DELETE /runs/{run_id} endpoint for discarding runs
- Record activities when runs complete for deletion tracking
- Add comprehensive tests for cache manager
Deletion rules enforced:
- Cannot delete items published to L2
- Cannot delete inputs/outputs of runs
- Can delete orphaned items
- Runs can only be discarded if no items are shared
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
126
server.py
126
server.py
@@ -27,6 +27,7 @@ from urllib.parse import urlparse
|
||||
|
||||
from celery_app import app as celery_app
|
||||
from tasks import render_effect
|
||||
from cache_manager import L1CacheManager, get_cache_manager
|
||||
|
||||
# L2 server for auth verification
|
||||
L2_SERVER = os.environ.get("L2_SERVER", "http://localhost:8200")
|
||||
@@ -37,6 +38,9 @@ L1_PUBLIC_URL = os.environ.get("L1_PUBLIC_URL", "http://localhost:8100")
|
||||
CACHE_DIR = Path(os.environ.get("CACHE_DIR", str(Path.home() / ".artdag" / "cache")))
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Initialize L1 cache manager with artdag integration
|
||||
cache_manager = L1CacheManager(cache_dir=CACHE_DIR, l2_server=L2_SERVER)
|
||||
|
||||
# Redis for persistent run storage
|
||||
REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379/5')
|
||||
parsed = urlparse(REDIS_URL)
|
||||
@@ -152,14 +156,14 @@ def file_hash(path: Path) -> str:
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def cache_file(source: Path) -> str:
|
||||
"""Copy file to cache, return content hash."""
|
||||
content_hash = file_hash(source)
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if not cache_path.exists():
|
||||
import shutil
|
||||
shutil.copy2(source, cache_path)
|
||||
return content_hash
|
||||
def cache_file(source: Path, node_type: str = "output") -> str:
|
||||
"""
|
||||
Copy file to cache using L1CacheManager, return content hash.
|
||||
|
||||
Uses artdag's Cache internally for proper tracking.
|
||||
"""
|
||||
cached = cache_manager.put(source, node_type=node_type)
|
||||
return cached.content_hash
|
||||
|
||||
|
||||
@app.get("/api")
|
||||
@@ -321,7 +325,15 @@ async def get_run(run_id: str):
|
||||
# Cache the output
|
||||
output_path = Path(result.get("output", {}).get("local_path", ""))
|
||||
if output_path.exists():
|
||||
cache_file(output_path)
|
||||
cache_file(output_path, node_type="effect_output")
|
||||
|
||||
# Record activity for deletion tracking
|
||||
if run.output_hash and run.inputs:
|
||||
cache_manager.record_simple_activity(
|
||||
input_hashes=run.inputs,
|
||||
output_hash=run.output_hash,
|
||||
run_id=run.run_id,
|
||||
)
|
||||
else:
|
||||
run.status = "failed"
|
||||
run.error = str(task.result)
|
||||
@@ -332,6 +344,42 @@ async def get_run(run_id: str):
|
||||
return run
|
||||
|
||||
|
||||
@app.delete("/runs/{run_id}")
|
||||
async def discard_run(run_id: str, username: str = Depends(get_required_user)):
|
||||
"""
|
||||
Discard (delete) a run and its intermediate cache entries.
|
||||
|
||||
Enforces deletion rules:
|
||||
- Cannot discard if any item (input, output) is published to L2
|
||||
- Deletes intermediate cache entries
|
||||
- Keeps inputs (may be used by other runs)
|
||||
- Deletes orphaned outputs
|
||||
"""
|
||||
run = load_run(run_id)
|
||||
if not run:
|
||||
raise HTTPException(404, f"Run {run_id} not found")
|
||||
|
||||
# Check ownership
|
||||
actor_id = f"@{username}@{L2_DOMAIN}"
|
||||
if run.username not in (username, actor_id):
|
||||
raise HTTPException(403, "Access denied")
|
||||
|
||||
# Check if run can be discarded
|
||||
can_discard, reason = cache_manager.can_discard_activity(run_id)
|
||||
if not can_discard:
|
||||
raise HTTPException(400, f"Cannot discard run: {reason}")
|
||||
|
||||
# Discard the activity (cleans up cache entries)
|
||||
success, msg = cache_manager.discard_activity(run_id)
|
||||
if not success:
|
||||
raise HTTPException(500, f"Failed to discard: {msg}")
|
||||
|
||||
# Remove from Redis
|
||||
redis_client.delete(f"{RUNS_KEY_PREFIX}{run_id}")
|
||||
|
||||
return {"discarded": True, "run_id": run_id}
|
||||
|
||||
|
||||
@app.get("/run/{run_id}")
|
||||
async def run_detail(run_id: str, request: Request):
|
||||
"""Run detail. HTML for browsers, JSON for APIs."""
|
||||
@@ -1428,31 +1476,45 @@ async def discard_cache(content_hash: str, username: str = Depends(get_required_
|
||||
"""
|
||||
Discard (delete) a cached item.
|
||||
|
||||
Refuses to delete pinned items. Pinned items include:
|
||||
- Published items
|
||||
- Inputs to published items
|
||||
Enforces deletion rules:
|
||||
- Cannot delete items published to L2 (shared)
|
||||
- Cannot delete inputs/outputs of activities (runs)
|
||||
- Cannot delete pinned items
|
||||
"""
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if not cache_path.exists():
|
||||
raise HTTPException(404, "Content not found")
|
||||
# Check if content exists (in cache_manager or legacy location)
|
||||
if not cache_manager.has_content(content_hash):
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if not cache_path.exists():
|
||||
raise HTTPException(404, "Content not found")
|
||||
|
||||
# Check ownership
|
||||
user_hashes = get_user_cache_hashes(username)
|
||||
if content_hash not in user_hashes:
|
||||
raise HTTPException(403, "Access denied")
|
||||
|
||||
# Check if pinned
|
||||
# Check if pinned (legacy metadata)
|
||||
meta = load_cache_meta(content_hash)
|
||||
if meta.get("pinned"):
|
||||
pin_reason = meta.get("pin_reason", "unknown")
|
||||
raise HTTPException(400, f"Cannot discard pinned item (reason: {pin_reason})")
|
||||
|
||||
# Delete the file and metadata
|
||||
cache_path.unlink()
|
||||
# Check deletion rules via cache_manager
|
||||
can_delete, reason = cache_manager.can_delete(content_hash)
|
||||
if not can_delete:
|
||||
raise HTTPException(400, f"Cannot discard: {reason}")
|
||||
|
||||
# Delete via cache_manager
|
||||
success, msg = cache_manager.delete_by_content_hash(content_hash)
|
||||
if not success:
|
||||
# Fallback to legacy deletion
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if cache_path.exists():
|
||||
cache_path.unlink()
|
||||
|
||||
# Clean up legacy metadata files
|
||||
meta_path = CACHE_DIR / f"{content_hash}.meta.json"
|
||||
if meta_path.exists():
|
||||
meta_path.unlink()
|
||||
# Also delete transcoded mp4 if exists
|
||||
mp4_path = CACHE_DIR / f"{content_hash}.mp4"
|
||||
if mp4_path.exists():
|
||||
mp4_path.unlink()
|
||||
@@ -1472,18 +1534,32 @@ async def ui_discard_cache(content_hash: str, request: Request):
|
||||
if content_hash not in user_hashes:
|
||||
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Access denied</div>'
|
||||
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if not cache_path.exists():
|
||||
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Content not found</div>'
|
||||
# Check if content exists
|
||||
if not cache_manager.has_content(content_hash):
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if not cache_path.exists():
|
||||
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Content not found</div>'
|
||||
|
||||
# Check if pinned
|
||||
# Check if pinned (legacy metadata)
|
||||
meta = load_cache_meta(content_hash)
|
||||
if meta.get("pinned"):
|
||||
pin_reason = meta.get("pin_reason", "unknown")
|
||||
return f'<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Cannot discard: item is pinned ({pin_reason})</div>'
|
||||
|
||||
# Delete the file and metadata
|
||||
cache_path.unlink()
|
||||
# Check deletion rules via cache_manager
|
||||
can_delete, reason = cache_manager.can_delete(content_hash)
|
||||
if not can_delete:
|
||||
return f'<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Cannot discard: {reason}</div>'
|
||||
|
||||
# Delete via cache_manager
|
||||
success, msg = cache_manager.delete_by_content_hash(content_hash)
|
||||
if not success:
|
||||
# Fallback to legacy deletion
|
||||
cache_path = CACHE_DIR / content_hash
|
||||
if cache_path.exists():
|
||||
cache_path.unlink()
|
||||
|
||||
# Clean up legacy metadata files
|
||||
meta_path = CACHE_DIR / f"{content_hash}.meta.json"
|
||||
if meta_path.exists():
|
||||
meta_path.unlink()
|
||||
|
||||
Reference in New Issue
Block a user