Add S-expression recipe support

- Add format detection that correctly handles ; comments
- Import artdag.sexp parser/compiler with YAML fallback
- Add execute_step_sexp and run_plan_sexp Celery tasks
- Update recipe upload to handle both S-expr and YAML formats

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

app/routers/recipes.py

@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
 
 
 class RecipeUploadRequest(BaseModel):
-    sexp_content: str
+    content: str  # S-expression or YAML
     name: Optional[str] = None
     description: Optional[str] = None
 
@@ -44,50 +44,87 @@ async def upload_recipe(
     ctx: UserContext = Depends(require_auth),
     recipe_service: RecipeService = Depends(get_recipe_service),
 ):
-    """Upload a new recipe from S-expression file."""
-    from artdag.sexp import compile_string, ParseError, CompileError
+    """Upload a new recipe from S-expression or YAML file."""
+    import yaml
 
-    # Read the S-expression content from the uploaded file
-    sexp_content = (await file.read()).decode("utf-8")
+    # Read content from the uploaded file
+    content = (await file.read()).decode("utf-8")
+
+    # Detect format (skip comments starting with ;)
+    def is_sexp_format(text):
+        for line in text.split('\n'):
+            stripped = line.strip()
+            if not stripped or stripped.startswith(';'):
+                continue
+            return stripped.startswith('(')
+        return False
+
+    is_sexp = is_sexp_format(content)
 
-    # Parse and compile S-expression to extract recipe info
     try:
-        compiled = compile_string(sexp_content)
-    except ParseError as e:
-        raise HTTPException(400, f"Parse error: {e}")
-    except CompileError as e:
-        raise HTTPException(400, f"Compile error: {e}")
+        from artdag.sexp import compile_string, ParseError, CompileError
+        SEXP_AVAILABLE = True
+    except ImportError:
+        SEXP_AVAILABLE = False
 
-    # Use filename (without extension) as recipe name if not specified
-    recipe_name = compiled.name
+    recipe_name = None
+    recipe_version = "1.0"
+    recipe_description = None
+    variable_inputs = []
+    fixed_inputs = []
+
+    if is_sexp and SEXP_AVAILABLE:
+        # Parse S-expression
+        try:
+            compiled = compile_string(content)
+            recipe_name = compiled.name
+            recipe_version = compiled.version
+            recipe_description = compiled.description
+
+            for node in compiled.nodes:
+                if node.get("type") == "SOURCE":
+                    config = node.get("config", {})
+                    if config.get("input"):
+                        variable_inputs.append(config.get("name", node.get("id")))
+                    elif config.get("asset"):
+                        fixed_inputs.append(config.get("asset"))
+        except Exception as e:
+            raise HTTPException(400, f"Parse error: {e}")
+    else:
+        # Parse YAML
+        try:
+            recipe_data = yaml.safe_load(content)
+            recipe_name = recipe_data.get("name")
+            recipe_version = recipe_data.get("version", "1.0")
+            recipe_description = recipe_data.get("description")
+
+            inputs = recipe_data.get("inputs", {})
+            for input_name, input_def in inputs.items():
+                if isinstance(input_def, dict) and input_def.get("fixed"):
+                    fixed_inputs.append(input_name)
+                else:
+                    variable_inputs.append(input_name)
+        except yaml.YAMLError as e:
+            raise HTTPException(400, f"Invalid YAML: {e}")
+
+    # Use filename as recipe name if not specified
     if not recipe_name and file.filename:
         recipe_name = file.filename.rsplit(".", 1)[0]
 
     recipe_id, error = await recipe_service.upload_recipe(
-        sexp_content=sexp_content,
+        content=content,
         uploader=ctx.actor_id,
         name=recipe_name,
-        description=compiled.description,
+        description=recipe_description,
     )
 
     if error:
         raise HTTPException(400, error)
 
-    # Extract input info from compiled nodes
-    variable_inputs = []
-    fixed_inputs = []
-    for node in compiled.nodes:
-        if node.get("type") == "SOURCE":
-            config = node.get("config", {})
-            if config.get("input"):
-                variable_inputs.append(config.get("name", node.get("id")))
-            elif config.get("asset"):
-                fixed_inputs.append(config.get("asset"))
-
     return {
         "recipe_id": recipe_id,
         "name": recipe_name or "unnamed",
-        "version": compiled.version,
+        "version": recipe_version,
         "variable_inputs": variable_inputs,
         "fixed_inputs": fixed_inputs,
         "message": "Recipe uploaded successfully",
@@ -351,12 +388,13 @@ async def run_recipe(
             dag_json = json.dumps(dag_copy)
 
         run, error = await run_service.create_run(
-            recipe=recipe.get("name", recipe_id),
+            recipe=recipe_id,  # Use recipe hash as primary identifier
             inputs=req.inputs,
             use_dag=True,
             dag_json=dag_json,
             actor_id=ctx.actor_id,
             l2_server=ctx.l2_server,
+            recipe_name=recipe.get("name"),  # Store name for display
         )
 
         if error: