art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add S-expression recipe support

Browse Source 
- Add format detection that correctly handles ; comments
- Import artdag.sexp parser/compiler with YAML fallback
- Add execute_step_sexp and run_plan_sexp Celery tasks
- Update recipe upload to handle both S-expr and YAML formats

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-11 23:08:53 +00:00
parent 9df78f771d
commit e59a50c000
5 changed files with 637 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
app/routers/recipes.py
View File

@@ -24,7 +24,7 @@ logger = logging.getLogger(__name__)
class RecipeUploadRequest(BaseModel):
sexp_content: str
content: str # S-expression or YAML
name: Optional[str] = None
description: Optional[str] = None
@@ -44,50 +44,87 @@ async def upload_recipe(
ctx: UserContext = Depends(require_auth),
recipe_service: RecipeService = Depends(get_recipe_service),
):
"""Upload a new recipe from S-expression file."""
from artdag.sexp import compile_string, ParseError, CompileError
"""Upload a new recipe from S-expression or YAML file."""
import yaml
# Read the S-expression content from the uploaded file
sexp_content = (await file.read()).decode("utf-8")
# Read content from the uploaded file
content = (await file.read()).decode("utf-8")
# Detect format (skip comments starting with ;)
def is_sexp_format(text):
for line in text.split('\n'):
stripped = line.strip()
if not stripped or stripped.startswith(';'):
continue
return stripped.startswith('(')
return False
is_sexp = is_sexp_format(content)
# Parse and compile S-expression to extract recipe info
try:
compiled = compile_string(sexp_content)
except ParseError as e:
raise HTTPException(400, f"Parse error: {e}")
except CompileError as e:
raise HTTPException(400, f"Compile error: {e}")
from artdag.sexp import compile_string, ParseError, CompileError
SEXP_AVAILABLE = True
except ImportError:
SEXP_AVAILABLE = False
# Use filename (without extension) as recipe name if not specified
recipe_name = compiled.name
recipe_name = None
recipe_version = "1.0"
recipe_description = None
variable_inputs = []
fixed_inputs = []
if is_sexp and SEXP_AVAILABLE:
# Parse S-expression
try:
compiled = compile_string(content)
recipe_name = compiled.name
recipe_version = compiled.version
recipe_description = compiled.description
for node in compiled.nodes:
if node.get("type") == "SOURCE":
config = node.get("config", {})
if config.get("input"):
variable_inputs.append(config.get("name", node.get("id")))
elif config.get("asset"):
fixed_inputs.append(config.get("asset"))
except Exception as e:
raise HTTPException(400, f"Parse error: {e}")
else:
# Parse YAML
try:
recipe_data = yaml.safe_load(content)
recipe_name = recipe_data.get("name")
recipe_version = recipe_data.get("version", "1.0")
recipe_description = recipe_data.get("description")
inputs = recipe_data.get("inputs", {})
for input_name, input_def in inputs.items():
if isinstance(input_def, dict) and input_def.get("fixed"):
fixed_inputs.append(input_name)
else:
variable_inputs.append(input_name)
except yaml.YAMLError as e:
raise HTTPException(400, f"Invalid YAML: {e}")
# Use filename as recipe name if not specified
if not recipe_name and file.filename:
recipe_name = file.filename.rsplit(".", 1)[0]
recipe_id, error = await recipe_service.upload_recipe(
sexp_content=sexp_content,
content=content,
uploader=ctx.actor_id,
name=recipe_name,
description=compiled.description,
description=recipe_description,
)
if error:
raise HTTPException(400, error)
# Extract input info from compiled nodes
variable_inputs = []
fixed_inputs = []
for node in compiled.nodes:
if node.get("type") == "SOURCE":
config = node.get("config", {})
if config.get("input"):
variable_inputs.append(config.get("name", node.get("id")))
elif config.get("asset"):
fixed_inputs.append(config.get("asset"))
return {
"recipe_id": recipe_id,
"name": recipe_name or "unnamed",
"version": compiled.version,
"version": recipe_version,
"variable_inputs": variable_inputs,
"fixed_inputs": fixed_inputs,
"message": "Recipe uploaded successfully",
@@ -351,12 +388,13 @@ async def run_recipe(
dag_json = json.dumps(dag_copy)
run, error = await run_service.create_run(
recipe=recipe.get("name", recipe_id),
recipe=recipe_id, # Use recipe hash as primary identifier
inputs=req.inputs,
use_dag=True,
dag_json=dag_json,
actor_id=ctx.actor_id,
l2_server=ctx.l2_server,
recipe_name=recipe.get("name"), # Store name for display
)
if error:

133
app/services/recipe_service.py
View File

@@ -1,15 +1,45 @@
"""
Recipe Service - business logic for recipe management.
Recipes are content-addressed S-expression files stored in the cache (and IPFS).
The recipe ID is the content hash of the S-expression file.
Recipes are content-addressed files stored in the cache (and IPFS).
Supports both S-expression (.sexp) and YAML (.yaml) formats.
The recipe ID is the content hash of the file.
"""
import tempfile
from pathlib import Path
from typing import Optional, List, Dict, Any, Tuple
from artdag.sexp import compile_string, parse, serialize, CompileError, ParseError
# Try to import S-expression support, fall back to YAML
try:
from artdag.sexp import compile_string, parse, serialize, CompileError, ParseError
SEXP_AVAILABLE = True
except ImportError:
SEXP_AVAILABLE = False
compile_string = None
parse = None
serialize = None
CompileError = Exception
ParseError = Exception
import yaml
def _is_sexp_format(content: str) -> bool:
"""
Detect if content is S-expression format.
Skips leading comments (lines starting with ;) and whitespace.
Returns True if the first non-comment content starts with (.
"""
for line in content.split('\n'):
stripped = line.strip()
if not stripped:
continue # Skip empty lines
if stripped.startswith(';'):
continue # Skip comments
return stripped.startswith('(')
return False
class RecipeService:
@@ -32,19 +62,31 @@ class RecipeService:
return None
with open(path) as f:
sexp_content = f.read()
content = f.read()
# Compile S-expression recipe
try:
compiled = compile_string(sexp_content)
recipe_data = compiled.to_dict()
except (ParseError, CompileError) as e:
# Return basic error info
return {"error": str(e), "recipe_id": recipe_id}
# Try to detect format and parse
recipe_data = None
is_sexp = _is_sexp_format(content)
if is_sexp and SEXP_AVAILABLE:
# Parse as S-expression
try:
compiled = compile_string(content)
recipe_data = compiled.to_dict()
recipe_data["sexp"] = content
except (ParseError, CompileError) as e:
return {"error": str(e), "recipe_id": recipe_id}
else:
# Parse as YAML
try:
recipe_data = yaml.safe_load(content)
if not isinstance(recipe_data, dict):
return {"error": "Invalid recipe format", "recipe_id": recipe_id}
except yaml.YAMLError as e:
return {"error": str(e), "recipe_id": recipe_id}
# Add the recipe_id to the data for convenience
recipe_data["recipe_id"] = recipe_id
recipe_data["sexp"] = sexp_content # Keep original S-expression
# Get IPFS CID if available
ipfs_cid = self.cache.get_ipfs_cid(recipe_id)
@@ -53,7 +95,11 @@ class RecipeService:
# Compute step_count from nodes
nodes = recipe_data.get("dag", {}).get("nodes", [])
recipe_data["step_count"] = len(nodes) if isinstance(nodes, list) else 0
if not nodes:
nodes = recipe_data.get("nodes", [])
if not nodes:
nodes = recipe_data.get("pipeline", [])
recipe_data["step_count"] = len(nodes) if isinstance(nodes, (list, dict)) else 0
return recipe_data
@@ -93,33 +139,53 @@ class RecipeService:
async def upload_recipe(
self,
sexp_content: str,
content: str,
uploader: str,
name: str = None,
description: str = None,
) -> Tuple[Optional[str], Optional[str]]:
"""
Upload a recipe from S-expression content.
Upload a recipe from S-expression or YAML content.
The recipe is stored in the cache and optionally pinned to IPFS.
Returns (recipe_id, error_message).
"""
# Validate and compile S-expression
try:
compiled = compile_string(sexp_content)
except ParseError as e:
return None, f"Parse error: {e}"
except CompileError as e:
return None, f"Compile error: {e}"
# Detect format
is_sexp = _is_sexp_format(content)
# For now, store the original S-expression content
# The uploader info is not embedded in the S-expression (kept in metadata)
# In a full implementation, we might add a :uploader keyword
if is_sexp and SEXP_AVAILABLE:
# Validate S-expression
try:
compiled = compile_string(content)
except ParseError as e:
return None, f"Parse error: {e}"
except CompileError as e:
return None, f"Compile error: {e}"
suffix = ".sexp"
else:
# Validate YAML
try:
recipe_data = yaml.safe_load(content)
if not isinstance(recipe_data, dict):
return None, "Recipe must be a YAML dictionary"
# Add uploader info
recipe_data["uploader"] = uploader
if name:
recipe_data["name"] = name
if description:
recipe_data["description"] = description
# Serialize back
content = yaml.dump(recipe_data, default_flow_style=False)
except yaml.YAMLError as e:
return None, f"Invalid YAML: {e}"
suffix = ".yaml"
# Write to temp file for caching
try:
with tempfile.NamedTemporaryFile(delete=False, suffix=".sexp", mode="w") as tmp:
tmp.write(sexp_content)
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix, mode="w") as tmp:
tmp.write(content)
tmp_path = Path(tmp.name)
# Store in cache (content-addressed, auto-pins to IPFS)
@@ -165,10 +231,15 @@ class RecipeService:
except Exception as e:
return False, f"Failed to delete: {e}"
def parse_recipe(self, sexp_content: str) -> Dict[str, Any]:
"""Parse and compile recipe S-expression content."""
compiled = compile_string(sexp_content)
return compiled.to_dict()
def parse_recipe(self, content: str) -> Dict[str, Any]:
"""Parse recipe content (S-expression or YAML)."""
is_sexp = _is_sexp_format(content)
if is_sexp and SEXP_AVAILABLE:
compiled = compile_string(content)
return compiled.to_dict()
else:
return yaml.safe_load(content)
def build_dag(self, recipe: Dict[str, Any]) -> Dict[str, Any]:
"""