art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to S-expression based execution with code-addressed cache IDs

Browse Source 
Major changes:
- Add execute_recipe task that uses S-expression planner
- Recipe S-expression unfolds into plan S-expression with code-addressed cache IDs
- Cache IDs computed from Merkle tree of plan structure (before execution)
- Add ipfs_client.add_string() for storing S-expression plans
- Update run_service.create_run() to use execute_recipe when recipe_sexp available
- Add _sexp_to_steps() to parse S-expression plans for UI visualization
- Plan endpoint now returns both sexp content and parsed steps

The code-addressed hashing means each plan step's cache_id is:
  sha3_256({node_type, config, sorted(input_cache_ids)})

This creates deterministic "buckets" for computation results computed
entirely from the plan structure, enabling automatic cache reuse.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-13 00:27:24 +00:00
parent 2c27eacb12
commit d603485d40
4 changed files with 529 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/routers/recipes.py
View File

@@ -569,6 +569,7 @@ async def run_recipe(
actor_id=ctx.actor_id,
l2_server=ctx.l2_server,
recipe_name=recipe.get("name"), # Store name for display
recipe_sexp=recipe.get("sexp"), # S-expression for code-addressed execution
)
if error:

135
app/services/run_service.py
View File

@@ -312,15 +312,19 @@ class RunService:
actor_id: Optional[str] = None,
l2_server: Optional[str] = None,
recipe_name: Optional[str] = None,
recipe_sexp: Optional[str] = None,
) -> Tuple[Optional[RunResult], Optional[str]]:
"""
Create a new rendering run. Checks cache before executing.
If recipe_sexp is provided, uses the new S-expression execution path
which generates code-addressed cache IDs before execution.
Returns (run_dict, error_message).
"""
import httpx
try:
from legacy_tasks import render_effect, execute_dag, build_effect_dag
from legacy_tasks import render_effect, execute_dag, build_effect_dag, execute_recipe
except ImportError as e:
return None, f"Celery tasks not available: {e}"
@@ -401,7 +405,17 @@ class RunService:
# Not cached - submit to Celery
try:
if use_dag or recipe == "dag":
# Prefer S-expression execution path (code-addressed cache IDs)
if recipe_sexp:
# Convert inputs to dict if needed
if isinstance(inputs, dict):
input_hashes = inputs
else:
# Legacy list format - use positional names
input_hashes = {f"input_{i}": cid for i, cid in enumerate(input_list)}
task = execute_recipe.delay(recipe_sexp, input_hashes, run_id)
elif use_dag or recipe == "dag":
if dag_json:
dag_data = dag_json
else:
@@ -562,6 +576,118 @@ class RunService:
"format": "json",
}
def _sexp_to_steps(self, sexp_content: str) -> Dict[str, Any]:
"""Convert S-expression plan to steps list format for UI.
Parses the S-expression plan format:
(plan :id <id> :recipe <name> :recipe-hash <hash>
(inputs (input_name hash) ...)
(step step_id :cache-id <hash> :level <int> (node-type :key val ...))
...
:output <output_step_id>)
Returns steps list compatible with UI visualization.
"""
try:
from artdag.sexp import parse, Symbol, Keyword
except ImportError:
return {"sexp": sexp_content, "steps": [], "format": "sexp"}
try:
parsed = parse(sexp_content)
except Exception:
return {"sexp": sexp_content, "steps": [], "format": "sexp"}
if not isinstance(parsed, list) or not parsed:
return {"sexp": sexp_content, "steps": [], "format": "sexp"}
steps = []
output_step_id = None
plan_id = None
recipe_name = None
# Parse plan structure
i = 0
while i < len(parsed):
item = parsed[i]
if isinstance(item, Keyword):
key = item.name
if i + 1 < len(parsed):
value = parsed[i + 1]
if key == "id":
plan_id = value
elif key == "recipe":
recipe_name = value
elif key == "output":
output_step_id = value
i += 2
continue
if isinstance(item, list) and item:
first = item[0]
if isinstance(first, Symbol) and first.name == "step":
# Parse step: (step step_id :cache-id <hash> :level <int> (node-expr))
step_id = item[1] if len(item) > 1 else None
cache_id = None
level = 0
node_type = "EFFECT"
config = {}
inputs = []
j = 2
while j < len(item):
part = item[j]
if isinstance(part, Keyword):
key = part.name
if j + 1 < len(item):
val = item[j + 1]
if key == "cache-id":
cache_id = val
elif key == "level":
level = val
j += 2
continue
elif isinstance(part, list) and part:
# Node expression: (node-type :key val ...)
if isinstance(part[0], Symbol):
node_type = part[0].name.upper()
k = 1
while k < len(part):
if isinstance(part[k], Keyword):
kname = part[k].name
if k + 1 < len(part):
kval = part[k + 1]
if kname == "inputs":
inputs = kval if isinstance(kval, list) else [kval]
else:
config[kname] = kval
k += 2
continue
k += 1
j += 1
steps.append({
"id": step_id,
"step_id": step_id,
"type": node_type,
"config": config,
"inputs": inputs,
"cache_id": cache_id or step_id,
"level": level,
})
i += 1
return {
"sexp": sexp_content,
"steps": steps,
"output_id": output_step_id,
"plan_id": plan_id,
"recipe": recipe_name,
"format": "sexp",
}
async def get_run_plan(self, run_id: str) -> Optional[Dict[str, Any]]:
"""Get execution plan for a run.
@@ -582,7 +708,8 @@ class RunService:
content = f.read()
# Detect format
if content.strip().startswith("("):
return {"sexp": content, "format": "sexp"}
# S-expression format - parse for UI
return self._sexp_to_steps(content)
else:
plan = json.loads(content)
return self._dag_to_steps(plan)
@@ -591,7 +718,7 @@ class RunService:
sexp_path = self.cache_dir / "plans" / f"{run_id}.sexp"
if sexp_path.exists():
with open(sexp_path) as f:
return {"sexp": f.read(), "format": "sexp"}
return self._sexp_to_steps(f.read())
json_path = self.cache_dir / "plans" / f"{run_id}.json"
if json_path.exists():