Refactor storage: remove Redis duplication, use proper data tiers

- Recipes: Now content-addressed only (cache + IPFS), removed Redis storage
- Runs: Completed runs stored in PostgreSQL, Redis only for task_id mapping
- Add list_runs_by_actor() to database.py for paginated run queries
- Add list_by_type() to cache_manager for filtering by node_type
- Fix upload endpoint to return size and filename fields
- Fix recipe run endpoint with proper DAG input binding
- Fix get_run_service() dependency to pass database module

Storage architecture:
- Redis: Ephemeral only (sessions, task mappings with TTL)
- PostgreSQL: Permanent records (completed runs, metadata)
- Cache: Content-addressed files (recipes, media, outputs)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
giles
2026-01-11 14:05:31 +00:00
parent 8591faf0fc
commit 854396680f
8 changed files with 965 additions and 264 deletions

View File

@@ -219,7 +219,12 @@ async def upload_content(
if error:
raise HTTPException(400, error)
return {"content_hash": content_hash, "uploaded": True}
return {
"content_hash": content_hash,
"filename": file.filename,
"size": len(content),
"uploaded": True,
}
# Media listing endpoint

View File

@@ -172,24 +172,56 @@ async def run_recipe(
if not recipe:
raise HTTPException(404, "Recipe not found")
# Create run using run service
run_service = RunService(database, get_redis_client(), get_cache_manager())
run, error = await run_service.create_run(
recipe=recipe.get("name", recipe_id),
inputs=req.inputs,
use_dag=True,
actor_id=ctx.actor_id,
l2_server=ctx.l2_server,
)
try:
import json
if error:
raise HTTPException(400, error)
# Create run using run service
run_service = RunService(database, get_redis_client(), get_cache_manager())
return {
"run_id": run.run_id,
"status": run.status,
"message": "Recipe execution started",
}
# If recipe has a DAG definition, bind inputs and convert to JSON
recipe_dag = recipe.get("dag")
dag_json = None
if recipe_dag and isinstance(recipe_dag, dict):
# Bind inputs to the DAG's source nodes
dag_copy = json.loads(json.dumps(recipe_dag)) # Deep copy
nodes = dag_copy.get("nodes", {})
# Map input names to content hashes
for input_name, content_hash in req.inputs.items():
if input_name in nodes:
node = nodes[input_name]
if node.get("type") == "SOURCE":
if "config" not in node:
node["config"] = {}
node["config"]["content_hash"] = content_hash
dag_json = json.dumps(dag_copy)
run, error = await run_service.create_run(
recipe=recipe.get("name", recipe_id),
inputs=req.inputs,
use_dag=True,
dag_json=dag_json,
actor_id=ctx.actor_id,
l2_server=ctx.l2_server,
)
if error:
raise HTTPException(400, error)
if not run:
raise HTTPException(500, "Run creation returned no result")
return {
"run_id": run.run_id,
"status": run.status,
"message": "Recipe execution started",
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error running recipe {recipe_id}")
raise HTTPException(500, f"Run failed: {e}")
@router.get("/{recipe_id}/dag")