Files
celery/app/routers/recipes.py
giles 854396680f Refactor storage: remove Redis duplication, use proper data tiers
- Recipes: Now content-addressed only (cache + IPFS), removed Redis storage
- Runs: Completed runs stored in PostgreSQL, Redis only for task_id mapping
- Add list_runs_by_actor() to database.py for paginated run queries
- Add list_by_type() to cache_manager for filtering by node_type
- Fix upload endpoint to return size and filename fields
- Fix recipe run endpoint with proper DAG input binding
- Fix get_run_service() dependency to pass database module

Storage architecture:
- Redis: Ephemeral only (sessions, task mappings with TTL)
- PostgreSQL: Permanent records (completed runs, metadata)
- Cache: Content-addressed files (recipes, media, outputs)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-11 14:05:31 +00:00

286 lines
8.5 KiB
Python

"""
Recipe management routes for L1 server.
Handles recipe upload, listing, viewing, and execution.
"""
import logging
from typing import List, Optional
from fastapi import APIRouter, Request, Depends, HTTPException, UploadFile, File
from fastapi.responses import HTMLResponse
from pydantic import BaseModel
from artdag_common import render
from artdag_common.middleware import wants_html, wants_json
from ..dependencies import require_auth, get_templates, get_redis_client, get_cache_manager
from ..services.auth_service import UserContext, AuthService
from ..services.recipe_service import RecipeService
router = APIRouter()
logger = logging.getLogger(__name__)
class RecipeUploadRequest(BaseModel):
yaml_content: str
name: Optional[str] = None
description: Optional[str] = None
class RecipeRunRequest(BaseModel):
inputs: dict = {}
def get_recipe_service():
"""Get recipe service instance."""
return RecipeService(get_redis_client(), get_cache_manager())
@router.post("/upload")
async def upload_recipe(
req: RecipeUploadRequest,
ctx: UserContext = Depends(require_auth),
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Upload a new recipe from YAML."""
recipe_id, error = await recipe_service.upload_recipe(
yaml_content=req.yaml_content,
uploader=ctx.actor_id,
name=req.name,
description=req.description,
)
if error:
raise HTTPException(400, error)
return {"recipe_id": recipe_id, "message": "Recipe uploaded successfully"}
@router.get("")
async def list_recipes(
request: Request,
offset: int = 0,
limit: int = 20,
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""List available recipes."""
auth_service = AuthService(get_redis_client())
ctx = auth_service.get_user_from_cookie(request)
if not ctx:
if wants_json(request):
raise HTTPException(401, "Authentication required")
from fastapi.responses import RedirectResponse
return RedirectResponse(url="/auth", status_code=302)
recipes = await recipe_service.list_recipes(ctx.actor_id, offset=offset, limit=limit)
if wants_json(request):
return {"recipes": recipes, "offset": offset, "limit": limit}
templates = get_templates(request)
return render(templates, "recipes/list.html", request,
recipes=recipes,
user=ctx,
active_tab="recipes",
)
@router.get("/{recipe_id}")
async def get_recipe(
recipe_id: str,
request: Request,
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Get recipe details."""
auth_service = AuthService(get_redis_client())
ctx = auth_service.get_user_from_cookie(request)
if not ctx:
if wants_json(request):
raise HTTPException(401, "Authentication required")
from fastapi.responses import RedirectResponse
return RedirectResponse(url="/auth", status_code=302)
recipe = await recipe_service.get_recipe(recipe_id)
if not recipe:
raise HTTPException(404, "Recipe not found")
if wants_json(request):
return recipe
# Build DAG elements for visualization
dag_elements = []
node_colors = {
"input": "#3b82f6",
"effect": "#8b5cf6",
"analyze": "#ec4899",
"transform": "#10b981",
"output": "#f59e0b",
}
for i, step in enumerate(recipe.get("steps", [])):
step_id = step.get("id", f"step-{i}")
dag_elements.append({
"data": {
"id": step_id,
"label": step.get("name", f"Step {i+1}"),
"color": node_colors.get(step.get("type", "effect"), "#6b7280"),
}
})
for inp in step.get("inputs", []):
dag_elements.append({
"data": {"source": inp, "target": step_id}
})
templates = get_templates(request)
return render(templates, "recipes/detail.html", request,
recipe=recipe,
dag_elements=dag_elements,
user=ctx,
active_tab="recipes",
)
@router.delete("/{recipe_id}")
async def delete_recipe(
recipe_id: str,
ctx: UserContext = Depends(require_auth),
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Delete a recipe."""
success, error = await recipe_service.delete_recipe(recipe_id, ctx.actor_id)
if error:
raise HTTPException(400 if "Cannot" in error else 404, error)
return {"deleted": True, "recipe_id": recipe_id}
@router.post("/{recipe_id}/run")
async def run_recipe(
recipe_id: str,
req: RecipeRunRequest,
ctx: UserContext = Depends(require_auth),
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Run a recipe with given inputs."""
from ..services.run_service import RunService
from ..dependencies import get_cache_manager
import database
recipe = await recipe_service.get_recipe(recipe_id)
if not recipe:
raise HTTPException(404, "Recipe not found")
try:
import json
# Create run using run service
run_service = RunService(database, get_redis_client(), get_cache_manager())
# If recipe has a DAG definition, bind inputs and convert to JSON
recipe_dag = recipe.get("dag")
dag_json = None
if recipe_dag and isinstance(recipe_dag, dict):
# Bind inputs to the DAG's source nodes
dag_copy = json.loads(json.dumps(recipe_dag)) # Deep copy
nodes = dag_copy.get("nodes", {})
# Map input names to content hashes
for input_name, content_hash in req.inputs.items():
if input_name in nodes:
node = nodes[input_name]
if node.get("type") == "SOURCE":
if "config" not in node:
node["config"] = {}
node["config"]["content_hash"] = content_hash
dag_json = json.dumps(dag_copy)
run, error = await run_service.create_run(
recipe=recipe.get("name", recipe_id),
inputs=req.inputs,
use_dag=True,
dag_json=dag_json,
actor_id=ctx.actor_id,
l2_server=ctx.l2_server,
)
if error:
raise HTTPException(400, error)
if not run:
raise HTTPException(500, "Run creation returned no result")
return {
"run_id": run.run_id,
"status": run.status,
"message": "Recipe execution started",
}
except HTTPException:
raise
except Exception as e:
logger.exception(f"Error running recipe {recipe_id}")
raise HTTPException(500, f"Run failed: {e}")
@router.get("/{recipe_id}/dag")
async def recipe_dag(
recipe_id: str,
request: Request,
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Get recipe DAG visualization data."""
recipe = await recipe_service.get_recipe(recipe_id)
if not recipe:
raise HTTPException(404, "Recipe not found")
dag_elements = []
node_colors = {
"input": "#3b82f6",
"effect": "#8b5cf6",
"analyze": "#ec4899",
"transform": "#10b981",
"output": "#f59e0b",
}
for i, step in enumerate(recipe.get("steps", [])):
step_id = step.get("id", f"step-{i}")
dag_elements.append({
"data": {
"id": step_id,
"label": step.get("name", f"Step {i+1}"),
"color": node_colors.get(step.get("type", "effect"), "#6b7280"),
}
})
for inp in step.get("inputs", []):
dag_elements.append({
"data": {"source": inp, "target": step_id}
})
return {"elements": dag_elements}
@router.delete("/{recipe_id}/ui", response_class=HTMLResponse)
async def ui_discard_recipe(
recipe_id: str,
request: Request,
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""HTMX handler: discard a recipe."""
auth_service = AuthService(get_redis_client())
ctx = auth_service.get_user_from_cookie(request)
if not ctx:
return HTMLResponse('<div class="text-red-400">Login required</div>', status_code=401)
success, error = await recipe_service.delete_recipe(recipe_id, ctx.actor_id)
if error:
return HTMLResponse(f'<div class="text-red-400">{error}</div>')
return HTMLResponse(
'<div class="text-green-400">Recipe deleted</div>'
'<script>setTimeout(() => window.location.href = "/recipes", 1500);</script>'
)