Refactor storage: remove Redis duplication, use proper data tiers
- Recipes: Now content-addressed only (cache + IPFS), removed Redis storage - Runs: Completed runs stored in PostgreSQL, Redis only for task_id mapping - Add list_runs_by_actor() to database.py for paginated run queries - Add list_by_type() to cache_manager for filtering by node_type - Fix upload endpoint to return size and filename fields - Fix recipe run endpoint with proper DAG input binding - Fix get_run_service() dependency to pass database module Storage architecture: - Redis: Ephemeral only (sessions, task mappings with TTL) - PostgreSQL: Permanent records (completed runs, metadata) - Cache: Content-addressed files (recipes, media, outputs) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -219,7 +219,12 @@ async def upload_content(
|
||||
if error:
|
||||
raise HTTPException(400, error)
|
||||
|
||||
return {"content_hash": content_hash, "uploaded": True}
|
||||
return {
|
||||
"content_hash": content_hash,
|
||||
"filename": file.filename,
|
||||
"size": len(content),
|
||||
"uploaded": True,
|
||||
}
|
||||
|
||||
|
||||
# Media listing endpoint
|
||||
|
||||
@@ -172,24 +172,56 @@ async def run_recipe(
|
||||
if not recipe:
|
||||
raise HTTPException(404, "Recipe not found")
|
||||
|
||||
# Create run using run service
|
||||
run_service = RunService(database, get_redis_client(), get_cache_manager())
|
||||
run, error = await run_service.create_run(
|
||||
recipe=recipe.get("name", recipe_id),
|
||||
inputs=req.inputs,
|
||||
use_dag=True,
|
||||
actor_id=ctx.actor_id,
|
||||
l2_server=ctx.l2_server,
|
||||
)
|
||||
try:
|
||||
import json
|
||||
|
||||
if error:
|
||||
raise HTTPException(400, error)
|
||||
# Create run using run service
|
||||
run_service = RunService(database, get_redis_client(), get_cache_manager())
|
||||
|
||||
return {
|
||||
"run_id": run.run_id,
|
||||
"status": run.status,
|
||||
"message": "Recipe execution started",
|
||||
}
|
||||
# If recipe has a DAG definition, bind inputs and convert to JSON
|
||||
recipe_dag = recipe.get("dag")
|
||||
dag_json = None
|
||||
if recipe_dag and isinstance(recipe_dag, dict):
|
||||
# Bind inputs to the DAG's source nodes
|
||||
dag_copy = json.loads(json.dumps(recipe_dag)) # Deep copy
|
||||
nodes = dag_copy.get("nodes", {})
|
||||
|
||||
# Map input names to content hashes
|
||||
for input_name, content_hash in req.inputs.items():
|
||||
if input_name in nodes:
|
||||
node = nodes[input_name]
|
||||
if node.get("type") == "SOURCE":
|
||||
if "config" not in node:
|
||||
node["config"] = {}
|
||||
node["config"]["content_hash"] = content_hash
|
||||
|
||||
dag_json = json.dumps(dag_copy)
|
||||
|
||||
run, error = await run_service.create_run(
|
||||
recipe=recipe.get("name", recipe_id),
|
||||
inputs=req.inputs,
|
||||
use_dag=True,
|
||||
dag_json=dag_json,
|
||||
actor_id=ctx.actor_id,
|
||||
l2_server=ctx.l2_server,
|
||||
)
|
||||
|
||||
if error:
|
||||
raise HTTPException(400, error)
|
||||
|
||||
if not run:
|
||||
raise HTTPException(500, "Run creation returned no result")
|
||||
|
||||
return {
|
||||
"run_id": run.run_id,
|
||||
"status": run.status,
|
||||
"message": "Recipe execution started",
|
||||
}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception(f"Error running recipe {recipe_id}")
|
||||
raise HTTPException(500, f"Run failed: {e}")
|
||||
|
||||
|
||||
@router.get("/{recipe_id}/dag")
|
||||
|
||||
Reference in New Issue
Block a user