art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add S-expression recipe support

Browse Source 
- Add format detection that correctly handles ; comments
- Import artdag.sexp parser/compiler with YAML fallback
- Add execute_step_sexp and run_plan_sexp Celery tasks
- Update recipe upload to handle both S-expr and YAML formats

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-11 23:08:53 +00:00
parent 9df78f771d
commit e59a50c000
5 changed files with 637 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
app/services/recipe_service.py
View File

@@ -1,15 +1,45 @@
"""
Recipe Service - business logic for recipe management.
Recipes are content-addressed S-expression files stored in the cache (and IPFS).
The recipe ID is the content hash of the S-expression file.
Recipes are content-addressed files stored in the cache (and IPFS).
Supports both S-expression (.sexp) and YAML (.yaml) formats.
The recipe ID is the content hash of the file.
"""
import tempfile
from pathlib import Path
from typing import Optional, List, Dict, Any, Tuple
from artdag.sexp import compile_string, parse, serialize, CompileError, ParseError
# Try to import S-expression support, fall back to YAML
try:
from artdag.sexp import compile_string, parse, serialize, CompileError, ParseError
SEXP_AVAILABLE = True
except ImportError:
SEXP_AVAILABLE = False
compile_string = None
parse = None
serialize = None
CompileError = Exception
ParseError = Exception
import yaml
def _is_sexp_format(content: str) -> bool:
"""
Detect if content is S-expression format.
Skips leading comments (lines starting with ;) and whitespace.
Returns True if the first non-comment content starts with (.
"""
for line in content.split('\n'):
stripped = line.strip()
if not stripped:
continue # Skip empty lines
if stripped.startswith(';'):
continue # Skip comments
return stripped.startswith('(')
return False
class RecipeService:
@@ -32,19 +62,31 @@ class RecipeService:
return None
with open(path) as f:
sexp_content = f.read()
content = f.read()
# Compile S-expression recipe
try:
compiled = compile_string(sexp_content)
recipe_data = compiled.to_dict()
except (ParseError, CompileError) as e:
# Return basic error info
return {"error": str(e), "recipe_id": recipe_id}
# Try to detect format and parse
recipe_data = None
is_sexp = _is_sexp_format(content)
if is_sexp and SEXP_AVAILABLE:
# Parse as S-expression
try:
compiled = compile_string(content)
recipe_data = compiled.to_dict()
recipe_data["sexp"] = content
except (ParseError, CompileError) as e:
return {"error": str(e), "recipe_id": recipe_id}
else:
# Parse as YAML
try:
recipe_data = yaml.safe_load(content)
if not isinstance(recipe_data, dict):
return {"error": "Invalid recipe format", "recipe_id": recipe_id}
except yaml.YAMLError as e:
return {"error": str(e), "recipe_id": recipe_id}
# Add the recipe_id to the data for convenience
recipe_data["recipe_id"] = recipe_id
recipe_data["sexp"] = sexp_content # Keep original S-expression
# Get IPFS CID if available
ipfs_cid = self.cache.get_ipfs_cid(recipe_id)
@@ -53,7 +95,11 @@ class RecipeService:
# Compute step_count from nodes
nodes = recipe_data.get("dag", {}).get("nodes", [])
recipe_data["step_count"] = len(nodes) if isinstance(nodes, list) else 0
if not nodes:
nodes = recipe_data.get("nodes", [])
if not nodes:
nodes = recipe_data.get("pipeline", [])
recipe_data["step_count"] = len(nodes) if isinstance(nodes, (list, dict)) else 0
return recipe_data
@@ -93,33 +139,53 @@ class RecipeService:
async def upload_recipe(
self,
sexp_content: str,
content: str,
uploader: str,
name: str = None,
description: str = None,
) -> Tuple[Optional[str], Optional[str]]:
"""
Upload a recipe from S-expression content.
Upload a recipe from S-expression or YAML content.
The recipe is stored in the cache and optionally pinned to IPFS.
Returns (recipe_id, error_message).
"""
# Validate and compile S-expression
try:
compiled = compile_string(sexp_content)
except ParseError as e:
return None, f"Parse error: {e}"
except CompileError as e:
return None, f"Compile error: {e}"
# Detect format
is_sexp = _is_sexp_format(content)
# For now, store the original S-expression content
# The uploader info is not embedded in the S-expression (kept in metadata)
# In a full implementation, we might add a :uploader keyword
if is_sexp and SEXP_AVAILABLE:
# Validate S-expression
try:
compiled = compile_string(content)
except ParseError as e:
return None, f"Parse error: {e}"
except CompileError as e:
return None, f"Compile error: {e}"
suffix = ".sexp"
else:
# Validate YAML
try:
recipe_data = yaml.safe_load(content)
if not isinstance(recipe_data, dict):
return None, "Recipe must be a YAML dictionary"
# Add uploader info
recipe_data["uploader"] = uploader
if name:
recipe_data["name"] = name
if description:
recipe_data["description"] = description
# Serialize back
content = yaml.dump(recipe_data, default_flow_style=False)
except yaml.YAMLError as e:
return None, f"Invalid YAML: {e}"
suffix = ".yaml"
# Write to temp file for caching
try:
with tempfile.NamedTemporaryFile(delete=False, suffix=".sexp", mode="w") as tmp:
tmp.write(sexp_content)
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix, mode="w") as tmp:
tmp.write(content)
tmp_path = Path(tmp.name)
# Store in cache (content-addressed, auto-pins to IPFS)
@@ -165,10 +231,15 @@ class RecipeService:
except Exception as e:
return False, f"Failed to delete: {e}"
def parse_recipe(self, sexp_content: str) -> Dict[str, Any]:
"""Parse and compile recipe S-expression content."""
compiled = compile_string(sexp_content)
return compiled.to_dict()
def parse_recipe(self, content: str) -> Dict[str, Any]:
"""Parse recipe content (S-expression or YAML)."""
is_sexp = _is_sexp_format(content)
if is_sexp and SEXP_AVAILABLE:
compiled = compile_string(content)
return compiled.to_dict()
else:
return yaml.safe_load(content)
def build_dag(self, recipe: Dict[str, Any]) -> Dict[str, Any]:
"""