Add config/recipe support for DAG-based jobs
- Add PyYAML dependency for parsing config files - Add Pydantic models: VariableInput, FixedInput, ConfigStatus, ConfigRunRequest - Add Redis storage functions for configs - Add config YAML parsing with variable and fixed input detection - Add config API endpoints: upload, list, get, delete, run - Add config UI: Configs tab, list page, detail page with run form - Add HTMX endpoints for config operations - Add pinning on publish: configs and their fixed inputs are pinned when runs from configs are published to L2 - Clean up debug logging in cache_manager Config YAML format supports: - Fixed inputs: resolve asset hashes from registry - Variable inputs: marked with `input: true`, filled at run time - DAG definition with nodes and edges - Registry of assets and effects Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -277,31 +277,25 @@ class L1CacheManager:
|
|||||||
|
|
||||||
def get_by_content_hash(self, content_hash: str) -> Optional[Path]:
|
def get_by_content_hash(self, content_hash: str) -> Optional[Path]:
|
||||||
"""Get cached file path by content_hash."""
|
"""Get cached file path by content_hash."""
|
||||||
logger.info(f"get_by_content_hash({content_hash[:16]}...) - cache_dir={self.cache_dir}, nodes_dir={self.cache.cache_dir}")
|
|
||||||
|
|
||||||
# Check index first (new cache structure)
|
# Check index first (new cache structure)
|
||||||
node_id = self._content_index.get(content_hash)
|
node_id = self._content_index.get(content_hash)
|
||||||
if node_id:
|
if node_id:
|
||||||
logger.info(f" Found in content_index: node_id={node_id[:16]}...")
|
|
||||||
path = self.cache.get(node_id)
|
path = self.cache.get(node_id)
|
||||||
if path and path.exists():
|
if path and path.exists():
|
||||||
logger.info(f" Found via index: {path}")
|
logger.info(f" Found via index: {path}")
|
||||||
return path
|
return path
|
||||||
logger.info(f" Index entry but path not found: {path}")
|
|
||||||
|
|
||||||
# For uploads, node_id == content_hash, so try direct lookup
|
# For uploads, node_id == content_hash, so try direct lookup
|
||||||
# This works even if cache index hasn't been reloaded
|
# This works even if cache index hasn't been reloaded
|
||||||
logger.info(f" Trying direct lookup with content_hash as node_id...")
|
|
||||||
path = self.cache.get(content_hash)
|
path = self.cache.get(content_hash)
|
||||||
logger.info(f" cache.get({content_hash[:16]}...) returned: {path}")
|
logger.info(f" cache.get({content_hash[:16]}...) returned: {path}")
|
||||||
if path and path.exists():
|
if path and path.exists():
|
||||||
logger.info(f" Found via direct lookup: {path}")
|
|
||||||
self._content_index[content_hash] = content_hash
|
self._content_index[content_hash] = content_hash
|
||||||
self._save_content_index()
|
self._save_content_index()
|
||||||
return path
|
return path
|
||||||
|
|
||||||
# Scan cache entries (fallback for new structure)
|
# Scan cache entries (fallback for new structure)
|
||||||
logger.info(f" Trying find_by_content_hash...")
|
|
||||||
entry = self.cache.find_by_content_hash(content_hash)
|
entry = self.cache.find_by_content_hash(content_hash)
|
||||||
if entry and entry.output_path.exists():
|
if entry and entry.output_path.exists():
|
||||||
logger.info(f" Found via scan: {entry.output_path}")
|
logger.info(f" Found via scan: {entry.output_path}")
|
||||||
@@ -311,12 +305,9 @@ class L1CacheManager:
|
|||||||
|
|
||||||
# Check legacy location (files stored directly as CACHE_DIR/{content_hash})
|
# Check legacy location (files stored directly as CACHE_DIR/{content_hash})
|
||||||
legacy_path = self.cache_dir / content_hash
|
legacy_path = self.cache_dir / content_hash
|
||||||
logger.info(f" Checking legacy path: {legacy_path}, exists={legacy_path.exists()}")
|
|
||||||
if legacy_path.exists() and legacy_path.is_file():
|
if legacy_path.exists() and legacy_path.is_file():
|
||||||
logger.info(f" Found at legacy location: {legacy_path}")
|
|
||||||
return legacy_path
|
return legacy_path
|
||||||
|
|
||||||
logger.info(f" NOT FOUND anywhere")
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def has_content(self, content_hash: str) -> bool:
|
def has_content(self, content_hash: str) -> bool:
|
||||||
|
|||||||
@@ -4,5 +4,6 @@ requests>=2.31.0
|
|||||||
fastapi>=0.109.0
|
fastapi>=0.109.0
|
||||||
uvicorn>=0.27.0
|
uvicorn>=0.27.0
|
||||||
python-multipart>=0.0.6
|
python-multipart>=0.0.6
|
||||||
|
PyYAML>=6.0
|
||||||
# Core artdag from GitHub
|
# Core artdag from GitHub
|
||||||
git+https://github.com/gilesbradshaw/art-dag.git
|
git+https://github.com/gilesbradshaw/art-dag.git
|
||||||
|
|||||||
711
server.py
711
server.py
@@ -24,6 +24,7 @@ from pydantic import BaseModel
|
|||||||
import redis
|
import redis
|
||||||
import requests as http_requests
|
import requests as http_requests
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
import yaml
|
||||||
|
|
||||||
from celery_app import app as celery_app
|
from celery_app import app as celery_app
|
||||||
from tasks import render_effect, execute_dag, build_effect_dag
|
from tasks import render_effect, execute_dag, build_effect_dag
|
||||||
@@ -50,6 +51,7 @@ redis_client = redis.Redis(
|
|||||||
db=int(parsed.path.lstrip('/') or 0)
|
db=int(parsed.path.lstrip('/') or 0)
|
||||||
)
|
)
|
||||||
RUNS_KEY_PREFIX = "artdag:run:"
|
RUNS_KEY_PREFIX = "artdag:run:"
|
||||||
|
CONFIGS_KEY_PREFIX = "artdag:config:"
|
||||||
|
|
||||||
|
|
||||||
def save_run(run: "RunStatus"):
|
def save_run(run: "RunStatus"):
|
||||||
@@ -123,6 +125,130 @@ class RunStatus(BaseModel):
|
|||||||
infrastructure: Optional[dict] = None # Hardware/software used for rendering
|
infrastructure: Optional[dict] = None # Hardware/software used for rendering
|
||||||
|
|
||||||
|
|
||||||
|
# ============ Config Models ============
|
||||||
|
|
||||||
|
class VariableInput(BaseModel):
|
||||||
|
"""A variable input that must be filled at run time."""
|
||||||
|
node_id: str
|
||||||
|
name: str
|
||||||
|
description: Optional[str] = None
|
||||||
|
required: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
class FixedInput(BaseModel):
|
||||||
|
"""A fixed input resolved from the registry."""
|
||||||
|
node_id: str
|
||||||
|
asset: str
|
||||||
|
content_hash: str
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigStatus(BaseModel):
|
||||||
|
"""Status/metadata of a config."""
|
||||||
|
config_id: str # Content hash of the YAML file
|
||||||
|
name: str
|
||||||
|
version: str
|
||||||
|
description: Optional[str] = None
|
||||||
|
variable_inputs: list[VariableInput]
|
||||||
|
fixed_inputs: list[FixedInput]
|
||||||
|
output_node: str
|
||||||
|
owner: Optional[str] = None
|
||||||
|
uploaded_at: str
|
||||||
|
uploader: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigRunRequest(BaseModel):
|
||||||
|
"""Request to run a config with variable inputs."""
|
||||||
|
inputs: dict[str, str] # node_id -> content_hash
|
||||||
|
|
||||||
|
|
||||||
|
def save_config(config: ConfigStatus):
|
||||||
|
"""Save config to Redis."""
|
||||||
|
redis_client.set(f"{CONFIGS_KEY_PREFIX}{config.config_id}", config.model_dump_json())
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(config_id: str) -> Optional[ConfigStatus]:
|
||||||
|
"""Load config from Redis."""
|
||||||
|
data = redis_client.get(f"{CONFIGS_KEY_PREFIX}{config_id}")
|
||||||
|
if data:
|
||||||
|
return ConfigStatus.model_validate_json(data)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def list_all_configs() -> list[ConfigStatus]:
|
||||||
|
"""List all configs from Redis."""
|
||||||
|
configs = []
|
||||||
|
for key in redis_client.scan_iter(f"{CONFIGS_KEY_PREFIX}*"):
|
||||||
|
data = redis_client.get(key)
|
||||||
|
if data:
|
||||||
|
configs.append(ConfigStatus.model_validate_json(data))
|
||||||
|
return sorted(configs, key=lambda c: c.uploaded_at, reverse=True)
|
||||||
|
|
||||||
|
|
||||||
|
def delete_config_from_redis(config_id: str) -> bool:
|
||||||
|
"""Delete config from Redis."""
|
||||||
|
return redis_client.delete(f"{CONFIGS_KEY_PREFIX}{config_id}") > 0
|
||||||
|
|
||||||
|
|
||||||
|
def parse_config_yaml(yaml_content: str, config_hash: str, uploader: str) -> ConfigStatus:
|
||||||
|
"""Parse a config YAML file and extract metadata."""
|
||||||
|
config = yaml.safe_load(yaml_content)
|
||||||
|
|
||||||
|
# Extract basic info
|
||||||
|
name = config.get("name", "unnamed")
|
||||||
|
version = config.get("version", "1.0")
|
||||||
|
description = config.get("description")
|
||||||
|
owner = config.get("owner")
|
||||||
|
|
||||||
|
# Parse registry
|
||||||
|
registry = config.get("registry", {})
|
||||||
|
assets = registry.get("assets", {})
|
||||||
|
|
||||||
|
# Parse DAG nodes
|
||||||
|
dag = config.get("dag", {})
|
||||||
|
nodes = dag.get("nodes", [])
|
||||||
|
output_node = dag.get("output")
|
||||||
|
|
||||||
|
variable_inputs = []
|
||||||
|
fixed_inputs = []
|
||||||
|
|
||||||
|
for node in nodes:
|
||||||
|
node_id = node.get("id")
|
||||||
|
node_type = node.get("type")
|
||||||
|
node_config = node.get("config", {})
|
||||||
|
|
||||||
|
if node_type == "SOURCE":
|
||||||
|
if node_config.get("input"):
|
||||||
|
# Variable input
|
||||||
|
variable_inputs.append(VariableInput(
|
||||||
|
node_id=node_id,
|
||||||
|
name=node_config.get("name", node_id),
|
||||||
|
description=node_config.get("description"),
|
||||||
|
required=node_config.get("required", True)
|
||||||
|
))
|
||||||
|
elif "asset" in node_config:
|
||||||
|
# Fixed input - resolve from registry
|
||||||
|
asset_name = node_config["asset"]
|
||||||
|
asset_info = assets.get(asset_name, {})
|
||||||
|
fixed_inputs.append(FixedInput(
|
||||||
|
node_id=node_id,
|
||||||
|
asset=asset_name,
|
||||||
|
content_hash=asset_info.get("hash", "")
|
||||||
|
))
|
||||||
|
|
||||||
|
return ConfigStatus(
|
||||||
|
config_id=config_hash,
|
||||||
|
name=name,
|
||||||
|
version=version,
|
||||||
|
description=description,
|
||||||
|
variable_inputs=variable_inputs,
|
||||||
|
fixed_inputs=fixed_inputs,
|
||||||
|
output_node=output_node or "",
|
||||||
|
owner=owner,
|
||||||
|
uploaded_at=datetime.now(timezone.utc).isoformat(),
|
||||||
|
uploader=uploader
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ============ Auth ============
|
# ============ Auth ============
|
||||||
|
|
||||||
security = HTTPBearer(auto_error=False)
|
security = HTTPBearer(auto_error=False)
|
||||||
@@ -280,42 +406,6 @@ async def root():
|
|||||||
return HOME_HTML
|
return HOME_HTML
|
||||||
|
|
||||||
|
|
||||||
@app.get("/debug/cache/{content_hash}")
|
|
||||||
async def debug_cache(content_hash: str):
|
|
||||||
"""Debug endpoint to check cache status for a content hash."""
|
|
||||||
import os
|
|
||||||
|
|
||||||
result = {
|
|
||||||
"content_hash": content_hash,
|
|
||||||
"cache_dir": str(cache_manager.cache_dir),
|
|
||||||
"nodes_dir": str(cache_manager.cache.cache_dir),
|
|
||||||
"in_content_index": content_hash in cache_manager._content_index,
|
|
||||||
"node_id_from_index": cache_manager._content_index.get(content_hash),
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check various locations
|
|
||||||
locations = {
|
|
||||||
"legacy_direct": cache_manager.cache_dir / content_hash,
|
|
||||||
"nodes_dir": cache_manager.cache.cache_dir / content_hash,
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, path in locations.items():
|
|
||||||
result[f"{name}_path"] = str(path)
|
|
||||||
result[f"{name}_exists"] = path.exists()
|
|
||||||
if path.exists() and path.is_dir():
|
|
||||||
result[f"{name}_contents"] = [f.name for f in path.iterdir()]
|
|
||||||
|
|
||||||
# Check if artdag cache has it
|
|
||||||
result["artdag_cache_get"] = str(cache_manager.cache.get(content_hash))
|
|
||||||
|
|
||||||
# Check via cache_manager
|
|
||||||
found_path = cache_manager.get_by_content_hash(content_hash)
|
|
||||||
result["cache_manager_path"] = str(found_path) if found_path else None
|
|
||||||
result["has_content"] = cache_manager.has_content(content_hash)
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/runs", response_model=RunStatus)
|
@app.post("/runs", response_model=RunStatus)
|
||||||
async def create_run(request: RunRequest, username: str = Depends(get_required_user)):
|
async def create_run(request: RunRequest, username: str = Depends(get_required_user)):
|
||||||
"""Start a new rendering run. Requires authentication."""
|
"""Start a new rendering run. Requires authentication."""
|
||||||
@@ -914,6 +1004,531 @@ async def list_runs(request: Request, page: int = 1, limit: int = 20):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ============ Config Endpoints ============
|
||||||
|
|
||||||
|
@app.post("/configs/upload")
|
||||||
|
async def upload_config(file: UploadFile = File(...), username: str = Depends(get_required_user)):
|
||||||
|
"""Upload a config YAML file. Requires authentication."""
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
# Read file content
|
||||||
|
content = await file.read()
|
||||||
|
try:
|
||||||
|
yaml_content = content.decode('utf-8')
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
raise HTTPException(400, "Config file must be valid UTF-8 text")
|
||||||
|
|
||||||
|
# Validate YAML
|
||||||
|
try:
|
||||||
|
yaml.safe_load(yaml_content)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
raise HTTPException(400, f"Invalid YAML: {e}")
|
||||||
|
|
||||||
|
# Store YAML file in cache
|
||||||
|
with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as tmp:
|
||||||
|
tmp.write(content)
|
||||||
|
tmp_path = Path(tmp.name)
|
||||||
|
|
||||||
|
cached = cache_manager.put(tmp_path, node_type="config", move=True)
|
||||||
|
config_hash = cached.content_hash
|
||||||
|
|
||||||
|
# Parse and save metadata
|
||||||
|
actor_id = f"@{username}@{L2_DOMAIN}"
|
||||||
|
try:
|
||||||
|
config_status = parse_config_yaml(yaml_content, config_hash, actor_id)
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(400, f"Failed to parse config: {e}")
|
||||||
|
|
||||||
|
save_config(config_status)
|
||||||
|
|
||||||
|
# Save cache metadata
|
||||||
|
save_cache_meta(config_hash, actor_id, file.filename, type="config", config_name=config_status.name)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"config_id": config_hash,
|
||||||
|
"name": config_status.name,
|
||||||
|
"version": config_status.version,
|
||||||
|
"variable_inputs": len(config_status.variable_inputs),
|
||||||
|
"fixed_inputs": len(config_status.fixed_inputs)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/configs")
|
||||||
|
async def list_configs_api(request: Request, page: int = 1, limit: int = 20):
|
||||||
|
"""List configs. HTML for browsers, JSON for APIs."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
|
||||||
|
all_configs = list_all_configs()
|
||||||
|
total = len(all_configs)
|
||||||
|
|
||||||
|
# Pagination
|
||||||
|
start = (page - 1) * limit
|
||||||
|
end = start + limit
|
||||||
|
configs_page = all_configs[start:end]
|
||||||
|
has_more = end < total
|
||||||
|
|
||||||
|
if wants_html(request):
|
||||||
|
# HTML response - redirect to /configs page with proper UI
|
||||||
|
return RedirectResponse(f"/configs?page={page}")
|
||||||
|
|
||||||
|
# JSON response for APIs
|
||||||
|
return {
|
||||||
|
"configs": [c.model_dump() for c in configs_page],
|
||||||
|
"pagination": {
|
||||||
|
"page": page,
|
||||||
|
"limit": limit,
|
||||||
|
"total": total,
|
||||||
|
"has_more": has_more
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/configs/{config_id}")
|
||||||
|
async def get_config_api(config_id: str):
|
||||||
|
"""Get config details."""
|
||||||
|
config = load_config(config_id)
|
||||||
|
if not config:
|
||||||
|
raise HTTPException(404, f"Config {config_id} not found")
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/configs/{config_id}")
|
||||||
|
async def remove_config(config_id: str, username: str = Depends(get_required_user)):
|
||||||
|
"""Delete a config. Requires authentication."""
|
||||||
|
config = load_config(config_id)
|
||||||
|
if not config:
|
||||||
|
raise HTTPException(404, f"Config {config_id} not found")
|
||||||
|
|
||||||
|
# Check ownership
|
||||||
|
actor_id = f"@{username}@{L2_DOMAIN}"
|
||||||
|
if config.uploader not in (username, actor_id):
|
||||||
|
raise HTTPException(403, "Access denied")
|
||||||
|
|
||||||
|
# Check if pinned
|
||||||
|
pinned, reason = cache_manager.is_pinned(config_id)
|
||||||
|
if pinned:
|
||||||
|
raise HTTPException(400, f"Cannot delete pinned config: {reason}")
|
||||||
|
|
||||||
|
# Delete from Redis and cache
|
||||||
|
delete_config_from_redis(config_id)
|
||||||
|
cache_manager.delete_by_content_hash(config_id)
|
||||||
|
|
||||||
|
return {"deleted": True, "config_id": config_id}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/configs/{config_id}/run")
|
||||||
|
async def run_config(config_id: str, request: ConfigRunRequest, username: str = Depends(get_required_user)):
|
||||||
|
"""Run a config with provided variable inputs. Requires authentication."""
|
||||||
|
config = load_config(config_id)
|
||||||
|
if not config:
|
||||||
|
raise HTTPException(404, f"Config {config_id} not found")
|
||||||
|
|
||||||
|
# Validate all required inputs are provided
|
||||||
|
for var_input in config.variable_inputs:
|
||||||
|
if var_input.required and var_input.node_id not in request.inputs:
|
||||||
|
raise HTTPException(400, f"Missing required input: {var_input.name}")
|
||||||
|
|
||||||
|
# Load config YAML
|
||||||
|
config_path = cache_manager.get_by_content_hash(config_id)
|
||||||
|
if not config_path:
|
||||||
|
raise HTTPException(500, "Config YAML not found in cache")
|
||||||
|
|
||||||
|
with open(config_path) as f:
|
||||||
|
yaml_config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
# Build DAG from config
|
||||||
|
dag = build_dag_from_config(yaml_config, request.inputs, config)
|
||||||
|
|
||||||
|
# Create run
|
||||||
|
run_id = str(uuid.uuid4())
|
||||||
|
actor_id = f"@{username}@{L2_DOMAIN}"
|
||||||
|
|
||||||
|
# Collect all input hashes
|
||||||
|
all_inputs = list(request.inputs.values())
|
||||||
|
for fixed in config.fixed_inputs:
|
||||||
|
if fixed.content_hash:
|
||||||
|
all_inputs.append(fixed.content_hash)
|
||||||
|
|
||||||
|
run = RunStatus(
|
||||||
|
run_id=run_id,
|
||||||
|
status="pending",
|
||||||
|
recipe=f"config:{config.name}",
|
||||||
|
inputs=all_inputs,
|
||||||
|
output_name=f"{config.name}-{run_id[:8]}",
|
||||||
|
created_at=datetime.now(timezone.utc).isoformat(),
|
||||||
|
username=actor_id
|
||||||
|
)
|
||||||
|
|
||||||
|
# Submit to Celery
|
||||||
|
dag_json = dag.to_json()
|
||||||
|
task = execute_dag.delay(dag_json, run.run_id)
|
||||||
|
run.celery_task_id = task.id
|
||||||
|
run.status = "running"
|
||||||
|
|
||||||
|
save_run(run)
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
def build_dag_from_config(yaml_config: dict, user_inputs: dict[str, str], config: ConfigStatus):
|
||||||
|
"""Build a DAG from config YAML with user-provided inputs."""
|
||||||
|
from artdag import DAG, Node
|
||||||
|
|
||||||
|
dag = DAG()
|
||||||
|
node_map = {} # node_id -> Node
|
||||||
|
|
||||||
|
registry = yaml_config.get("registry", {})
|
||||||
|
assets = registry.get("assets", {})
|
||||||
|
effects = registry.get("effects", {})
|
||||||
|
dag_config = yaml_config.get("dag", {})
|
||||||
|
nodes = dag_config.get("nodes", [])
|
||||||
|
|
||||||
|
for node_def in nodes:
|
||||||
|
node_id = node_def.get("id")
|
||||||
|
node_type = node_def.get("type")
|
||||||
|
node_config = node_def.get("config", {})
|
||||||
|
input_ids = node_def.get("inputs", [])
|
||||||
|
|
||||||
|
if node_type == "SOURCE":
|
||||||
|
if node_config.get("input"):
|
||||||
|
# Variable input - use user-provided hash
|
||||||
|
content_hash = user_inputs.get(node_id)
|
||||||
|
if not content_hash:
|
||||||
|
raise HTTPException(400, f"Missing input for node {node_id}")
|
||||||
|
node = Node(node_id, "SOURCE", {"content_hash": content_hash})
|
||||||
|
else:
|
||||||
|
# Fixed input - use registry hash
|
||||||
|
asset_name = node_config.get("asset")
|
||||||
|
asset_info = assets.get(asset_name, {})
|
||||||
|
content_hash = asset_info.get("hash")
|
||||||
|
if not content_hash:
|
||||||
|
raise HTTPException(400, f"Asset {asset_name} not found in registry")
|
||||||
|
node = Node(node_id, "SOURCE", {"content_hash": content_hash})
|
||||||
|
elif node_type == "EFFECT":
|
||||||
|
effect_name = node_config.get("effect")
|
||||||
|
effect_info = effects.get(effect_name, {})
|
||||||
|
effect_hash = effect_info.get("hash")
|
||||||
|
node = Node(node_id, "EFFECT", {"effect": effect_name, "effect_hash": effect_hash})
|
||||||
|
else:
|
||||||
|
node = Node(node_id, node_type, node_config)
|
||||||
|
|
||||||
|
node_map[node_id] = node
|
||||||
|
dag.add_node(node)
|
||||||
|
|
||||||
|
# Connect edges
|
||||||
|
for node_def in nodes:
|
||||||
|
node_id = node_def.get("id")
|
||||||
|
input_ids = node_def.get("inputs", [])
|
||||||
|
for input_id in input_ids:
|
||||||
|
dag.add_edge(input_id, node_id)
|
||||||
|
|
||||||
|
return dag
|
||||||
|
|
||||||
|
|
||||||
|
# ============ Config UI Pages ============
|
||||||
|
|
||||||
|
@app.get("/configs", response_class=HTMLResponse)
|
||||||
|
async def configs_page(request: Request, page: int = 1):
|
||||||
|
"""Configs list page (HTML)."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
|
||||||
|
if not current_user:
|
||||||
|
return HTMLResponse(render_page(
|
||||||
|
"Configs",
|
||||||
|
'<p class="text-gray-400 py-8 text-center"><a href="/login" class="text-blue-400 hover:text-blue-300">Login</a> to see configs.</p>',
|
||||||
|
None,
|
||||||
|
active_tab="configs"
|
||||||
|
))
|
||||||
|
|
||||||
|
all_configs = list_all_configs()
|
||||||
|
|
||||||
|
# Filter to user's configs
|
||||||
|
actor_id = f"@{current_user}@{L2_DOMAIN}"
|
||||||
|
user_configs = [c for c in all_configs if c.uploader in (current_user, actor_id)]
|
||||||
|
total = len(user_configs)
|
||||||
|
|
||||||
|
if not user_configs:
|
||||||
|
content = '''
|
||||||
|
<h2 class="text-xl font-semibold text-white mb-6">Configs (0)</h2>
|
||||||
|
<p class="text-gray-400 py-8 text-center">No configs yet. Upload a config YAML file to get started.</p>
|
||||||
|
'''
|
||||||
|
return HTMLResponse(render_page("Configs", content, current_user, active_tab="configs"))
|
||||||
|
|
||||||
|
html_parts = []
|
||||||
|
for config in user_configs:
|
||||||
|
var_count = len(config.variable_inputs)
|
||||||
|
fixed_count = len(config.fixed_inputs)
|
||||||
|
input_info = []
|
||||||
|
if var_count:
|
||||||
|
input_info.append(f"{var_count} variable")
|
||||||
|
if fixed_count:
|
||||||
|
input_info.append(f"{fixed_count} fixed")
|
||||||
|
inputs_str = ", ".join(input_info) if input_info else "no inputs"
|
||||||
|
|
||||||
|
html_parts.append(f'''
|
||||||
|
<a href="/config/{config.config_id}" class="block">
|
||||||
|
<div class="bg-dark-700 rounded-lg p-4 hover:bg-dark-600 transition-colors">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-3 mb-3">
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<span class="px-3 py-1 bg-purple-600 text-white text-sm font-medium rounded-full">{config.name}</span>
|
||||||
|
<span class="text-gray-400 text-xs">v{config.version}</span>
|
||||||
|
</div>
|
||||||
|
<span class="text-xs text-gray-400">{inputs_str}</span>
|
||||||
|
</div>
|
||||||
|
<div class="text-sm text-gray-400 mb-2">
|
||||||
|
{config.description or "No description"}
|
||||||
|
</div>
|
||||||
|
<div class="text-xs text-gray-500 font-mono truncate">
|
||||||
|
{config.config_id[:24]}...
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
''')
|
||||||
|
|
||||||
|
content = f'''
|
||||||
|
<h2 class="text-xl font-semibold text-white mb-6">Configs ({total})</h2>
|
||||||
|
<div class="space-y-4">
|
||||||
|
{''.join(html_parts)}
|
||||||
|
</div>
|
||||||
|
'''
|
||||||
|
|
||||||
|
return HTMLResponse(render_page("Configs", content, current_user, active_tab="configs"))
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/config/{config_id}", response_class=HTMLResponse)
|
||||||
|
async def config_detail_page(config_id: str, request: Request):
|
||||||
|
"""Config detail page with run form."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
config = load_config(config_id)
|
||||||
|
|
||||||
|
if not config:
|
||||||
|
return HTMLResponse(render_page(
|
||||||
|
"Config Not Found",
|
||||||
|
f'<p class="text-red-400">Config {config_id} not found.</p>',
|
||||||
|
current_user,
|
||||||
|
active_tab="configs"
|
||||||
|
), status_code=404)
|
||||||
|
|
||||||
|
# Build variable inputs form
|
||||||
|
var_inputs_html = ""
|
||||||
|
if config.variable_inputs:
|
||||||
|
var_inputs_html = '<div class="space-y-4 mb-6">'
|
||||||
|
for var_input in config.variable_inputs:
|
||||||
|
required = "required" if var_input.required else ""
|
||||||
|
var_inputs_html += f'''
|
||||||
|
<div>
|
||||||
|
<label class="block text-sm font-medium text-gray-300 mb-2">
|
||||||
|
{var_input.name} {'*' if var_input.required else ''}
|
||||||
|
</label>
|
||||||
|
<input type="text" name="{var_input.node_id}" {required}
|
||||||
|
placeholder="Content hash..."
|
||||||
|
class="w-full px-4 py-2 bg-dark-600 border border-dark-500 rounded-lg text-white focus:ring-2 focus:ring-blue-500 focus:border-transparent">
|
||||||
|
<p class="text-xs text-gray-400 mt-1">{var_input.description or 'Enter a content hash from your cache'}</p>
|
||||||
|
</div>
|
||||||
|
'''
|
||||||
|
var_inputs_html += '</div>'
|
||||||
|
else:
|
||||||
|
var_inputs_html = '<p class="text-gray-400 mb-4">This config has no variable inputs - it uses fixed assets only.</p>'
|
||||||
|
|
||||||
|
# Build fixed inputs display
|
||||||
|
fixed_inputs_html = ""
|
||||||
|
if config.fixed_inputs:
|
||||||
|
fixed_inputs_html = '<div class="mt-4"><h4 class="text-sm font-medium text-gray-300 mb-2">Fixed Inputs</h4><ul class="text-sm text-gray-400 space-y-1">'
|
||||||
|
for fixed in config.fixed_inputs:
|
||||||
|
fixed_inputs_html += f'<li><span class="text-gray-500">{fixed.asset}:</span> <span class="font-mono text-xs">{fixed.content_hash[:16]}...</span></li>'
|
||||||
|
fixed_inputs_html += '</ul></div>'
|
||||||
|
|
||||||
|
# Check if pinned
|
||||||
|
pinned, pin_reason = cache_manager.is_pinned(config_id)
|
||||||
|
pinned_badge = ""
|
||||||
|
if pinned:
|
||||||
|
pinned_badge = f'<span class="px-2 py-1 bg-yellow-600 text-white text-xs rounded-full ml-2">Pinned: {pin_reason}</span>'
|
||||||
|
|
||||||
|
content = f'''
|
||||||
|
<div class="mb-6">
|
||||||
|
<a href="/configs" class="text-blue-400 hover:text-blue-300 text-sm">← Back to configs</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="bg-dark-700 rounded-lg p-6 mb-6">
|
||||||
|
<div class="flex items-center gap-3 mb-4">
|
||||||
|
<h2 class="text-2xl font-bold text-white">{config.name}</h2>
|
||||||
|
<span class="px-2 py-1 bg-gray-600 text-white text-xs rounded-full">v{config.version}</span>
|
||||||
|
{pinned_badge}
|
||||||
|
</div>
|
||||||
|
<p class="text-gray-400 mb-4">{config.description or 'No description'}</p>
|
||||||
|
<div class="text-xs text-gray-500 font-mono">{config.config_id}</div>
|
||||||
|
{fixed_inputs_html}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="bg-dark-700 rounded-lg p-6">
|
||||||
|
<h3 class="text-lg font-semibold text-white mb-4">Run this Config</h3>
|
||||||
|
<form hx-post="/ui/configs/{config_id}/run" hx-target="#run-result" hx-swap="innerHTML">
|
||||||
|
{var_inputs_html}
|
||||||
|
<div id="run-result"></div>
|
||||||
|
<button type="submit"
|
||||||
|
class="px-6 py-2 bg-green-600 hover:bg-green-700 text-white font-medium rounded-lg transition-colors">
|
||||||
|
Run Config
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
'''
|
||||||
|
|
||||||
|
return HTMLResponse(render_page(f"Config: {config.name}", content, current_user, active_tab="configs"))
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/ui/configs/{config_id}/run", response_class=HTMLResponse)
|
||||||
|
async def ui_run_config(config_id: str, request: Request):
|
||||||
|
"""HTMX handler: run a config with form inputs."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
if not current_user:
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Login required</div>'
|
||||||
|
|
||||||
|
config = load_config(config_id)
|
||||||
|
if not config:
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Config not found</div>'
|
||||||
|
|
||||||
|
# Parse form data
|
||||||
|
form_data = await request.form()
|
||||||
|
inputs = {}
|
||||||
|
for var_input in config.variable_inputs:
|
||||||
|
value = form_data.get(var_input.node_id, "").strip()
|
||||||
|
if var_input.required and not value:
|
||||||
|
return f'<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Missing required input: {var_input.name}</div>'
|
||||||
|
if value:
|
||||||
|
inputs[var_input.node_id] = value
|
||||||
|
|
||||||
|
# Load config YAML
|
||||||
|
config_path = cache_manager.get_by_content_hash(config_id)
|
||||||
|
if not config_path:
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Config YAML not found in cache</div>'
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(config_path) as f:
|
||||||
|
yaml_config = yaml.safe_load(f)
|
||||||
|
|
||||||
|
# Build DAG from config
|
||||||
|
dag = build_dag_from_config(yaml_config, inputs, config)
|
||||||
|
|
||||||
|
# Create run
|
||||||
|
run_id = str(uuid.uuid4())
|
||||||
|
actor_id = f"@{current_user}@{L2_DOMAIN}"
|
||||||
|
|
||||||
|
# Collect all input hashes
|
||||||
|
all_inputs = list(inputs.values())
|
||||||
|
for fixed in config.fixed_inputs:
|
||||||
|
if fixed.content_hash:
|
||||||
|
all_inputs.append(fixed.content_hash)
|
||||||
|
|
||||||
|
run = RunStatus(
|
||||||
|
run_id=run_id,
|
||||||
|
status="pending",
|
||||||
|
recipe=f"config:{config.name}",
|
||||||
|
inputs=all_inputs,
|
||||||
|
output_name=f"{config.name}-{run_id[:8]}",
|
||||||
|
created_at=datetime.now(timezone.utc).isoformat(),
|
||||||
|
username=actor_id
|
||||||
|
)
|
||||||
|
|
||||||
|
# Submit to Celery
|
||||||
|
dag_json = dag.to_json()
|
||||||
|
task = execute_dag.delay(dag_json, run.run_id)
|
||||||
|
run.celery_task_id = task.id
|
||||||
|
run.status = "running"
|
||||||
|
|
||||||
|
save_run(run)
|
||||||
|
|
||||||
|
return f'''
|
||||||
|
<div class="bg-green-900/50 border border-green-700 text-green-300 px-4 py-3 rounded-lg mb-4">
|
||||||
|
Run started! <a href="/run/{run_id}" class="underline">View run</a>
|
||||||
|
</div>
|
||||||
|
'''
|
||||||
|
except Exception as e:
|
||||||
|
return f'<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Error: {str(e)}</div>'
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/ui/configs-list", response_class=HTMLResponse)
|
||||||
|
async def ui_configs_list(request: Request):
|
||||||
|
"""HTMX partial: list of configs."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
|
||||||
|
if not current_user:
|
||||||
|
return '<p class="text-gray-400 py-8 text-center"><a href="/ui/login" class="text-blue-400 hover:text-blue-300">Login</a> to see configs.</p>'
|
||||||
|
|
||||||
|
all_configs = list_all_configs()
|
||||||
|
|
||||||
|
# Filter to user's configs
|
||||||
|
actor_id = f"@{current_user}@{L2_DOMAIN}"
|
||||||
|
user_configs = [c for c in all_configs if c.uploader in (current_user, actor_id)]
|
||||||
|
|
||||||
|
if not user_configs:
|
||||||
|
return '<p class="text-gray-400 py-8 text-center">No configs yet. Upload a config YAML file to get started.</p>'
|
||||||
|
|
||||||
|
html_parts = ['<div class="space-y-4">']
|
||||||
|
for config in user_configs:
|
||||||
|
var_count = len(config.variable_inputs)
|
||||||
|
fixed_count = len(config.fixed_inputs)
|
||||||
|
input_info = []
|
||||||
|
if var_count:
|
||||||
|
input_info.append(f"{var_count} variable")
|
||||||
|
if fixed_count:
|
||||||
|
input_info.append(f"{fixed_count} fixed")
|
||||||
|
inputs_str = ", ".join(input_info) if input_info else "no inputs"
|
||||||
|
|
||||||
|
html_parts.append(f'''
|
||||||
|
<a href="/config/{config.config_id}" class="block">
|
||||||
|
<div class="bg-dark-700 rounded-lg p-4 hover:bg-dark-600 transition-colors">
|
||||||
|
<div class="flex flex-wrap items-center justify-between gap-3 mb-3">
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<span class="px-3 py-1 bg-purple-600 text-white text-sm font-medium rounded-full">{config.name}</span>
|
||||||
|
<span class="text-gray-400 text-xs">v{config.version}</span>
|
||||||
|
</div>
|
||||||
|
<span class="text-xs text-gray-400">{inputs_str}</span>
|
||||||
|
</div>
|
||||||
|
<div class="text-sm text-gray-400 mb-2">
|
||||||
|
{config.description or "No description"}
|
||||||
|
</div>
|
||||||
|
<div class="text-xs text-gray-500 font-mono truncate">
|
||||||
|
{config.config_id[:24]}...
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
''')
|
||||||
|
|
||||||
|
html_parts.append('</div>')
|
||||||
|
return '\n'.join(html_parts)
|
||||||
|
|
||||||
|
|
||||||
|
@app.delete("/ui/configs/{config_id}/discard", response_class=HTMLResponse)
|
||||||
|
async def ui_discard_config(config_id: str, request: Request):
|
||||||
|
"""HTMX handler: discard a config."""
|
||||||
|
current_user = get_user_from_cookie(request)
|
||||||
|
if not current_user:
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Login required</div>'
|
||||||
|
|
||||||
|
config = load_config(config_id)
|
||||||
|
if not config:
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Config not found</div>'
|
||||||
|
|
||||||
|
# Check ownership
|
||||||
|
actor_id = f"@{current_user}@{L2_DOMAIN}"
|
||||||
|
if config.uploader not in (current_user, actor_id):
|
||||||
|
return '<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Access denied</div>'
|
||||||
|
|
||||||
|
# Check if pinned
|
||||||
|
pinned, reason = cache_manager.is_pinned(config_id)
|
||||||
|
if pinned:
|
||||||
|
return f'<div class="bg-red-900/50 border border-red-700 text-red-300 px-4 py-3 rounded-lg mb-4">Cannot delete: config is pinned ({reason})</div>'
|
||||||
|
|
||||||
|
# Delete from Redis and cache
|
||||||
|
delete_config_from_redis(config_id)
|
||||||
|
cache_manager.delete_by_content_hash(config_id)
|
||||||
|
|
||||||
|
return '''
|
||||||
|
<div class="bg-green-900/50 border border-green-700 text-green-300 px-4 py-3 rounded-lg mb-4">
|
||||||
|
Config deleted. <a href="/configs" class="underline">Back to configs</a>
|
||||||
|
</div>
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
@app.get("/cache/{content_hash}")
|
@app.get("/cache/{content_hash}")
|
||||||
async def get_cached(content_hash: str):
|
async def get_cached(content_hash: str):
|
||||||
"""Get cached content by hash."""
|
"""Get cached content by hash."""
|
||||||
@@ -2317,6 +2932,7 @@ def render_page(title: str, content: str, username: Optional[str] = None, active
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
runs_active = "border-b-2 border-blue-500 text-white" if active_tab == "runs" else "text-gray-400 hover:text-white"
|
runs_active = "border-b-2 border-blue-500 text-white" if active_tab == "runs" else "text-gray-400 hover:text-white"
|
||||||
|
configs_active = "border-b-2 border-blue-500 text-white" if active_tab == "configs" else "text-gray-400 hover:text-white"
|
||||||
cache_active = "border-b-2 border-blue-500 text-white" if active_tab == "cache" else "text-gray-400 hover:text-white"
|
cache_active = "border-b-2 border-blue-500 text-white" if active_tab == "cache" else "text-gray-400 hover:text-white"
|
||||||
|
|
||||||
return f"""
|
return f"""
|
||||||
@@ -2339,6 +2955,7 @@ def render_page(title: str, content: str, username: Optional[str] = None, active
|
|||||||
|
|
||||||
<nav class="flex gap-6 mb-6 border-b border-dark-500 pb-0">
|
<nav class="flex gap-6 mb-6 border-b border-dark-500 pb-0">
|
||||||
<a href="/runs" class="pb-3 px-1 font-medium transition-colors {runs_active}">Runs</a>
|
<a href="/runs" class="pb-3 px-1 font-medium transition-colors {runs_active}">Runs</a>
|
||||||
|
<a href="/configs" class="pb-3 px-1 font-medium transition-colors {configs_active}">Configs</a>
|
||||||
<a href="/cache" class="pb-3 px-1 font-medium transition-colors {cache_active}">Cache</a>
|
<a href="/cache" class="pb-3 px-1 font-medium transition-colors {cache_active}">Cache</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
@@ -2369,9 +2986,15 @@ def render_ui_html(username: Optional[str] = None, tab: str = "runs") -> str:
|
|||||||
'''
|
'''
|
||||||
|
|
||||||
runs_active = "border-b-2 border-blue-500 text-white" if tab == "runs" else "text-gray-400 hover:text-white"
|
runs_active = "border-b-2 border-blue-500 text-white" if tab == "runs" else "text-gray-400 hover:text-white"
|
||||||
|
configs_active = "border-b-2 border-blue-500 text-white" if tab == "configs" else "text-gray-400 hover:text-white"
|
||||||
cache_active = "border-b-2 border-blue-500 text-white" if tab == "cache" else "text-gray-400 hover:text-white"
|
cache_active = "border-b-2 border-blue-500 text-white" if tab == "cache" else "text-gray-400 hover:text-white"
|
||||||
|
|
||||||
content_url = "/ui/runs" if tab == "runs" else "/ui/cache-list"
|
if tab == "runs":
|
||||||
|
content_url = "/ui/runs"
|
||||||
|
elif tab == "configs":
|
||||||
|
content_url = "/ui/configs-list"
|
||||||
|
else:
|
||||||
|
content_url = "/ui/cache-list"
|
||||||
|
|
||||||
return f"""
|
return f"""
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
@@ -2393,6 +3016,7 @@ def render_ui_html(username: Optional[str] = None, tab: str = "runs") -> str:
|
|||||||
|
|
||||||
<nav class="flex gap-6 mb-6 border-b border-dark-500 pb-0">
|
<nav class="flex gap-6 mb-6 border-b border-dark-500 pb-0">
|
||||||
<a href="/ui" class="pb-3 px-1 font-medium transition-colors {runs_active}">Runs</a>
|
<a href="/ui" class="pb-3 px-1 font-medium transition-colors {runs_active}">Runs</a>
|
||||||
|
<a href="/ui?tab=configs" class="pb-3 px-1 font-medium transition-colors {configs_active}">Configs</a>
|
||||||
<a href="/ui?tab=cache" class="pb-3 px-1 font-medium transition-colors {cache_active}">Cache</a>
|
<a href="/ui?tab=cache" class="pb-3 px-1 font-medium transition-colors {cache_active}">Cache</a>
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
@@ -2621,6 +3245,19 @@ async def ui_publish_run(run_id: str, request: Request, output_name: str = Form(
|
|||||||
for input_hash in run.inputs:
|
for input_hash in run.inputs:
|
||||||
save_cache_meta(input_hash, pinned=True, pin_reason="input_to_published")
|
save_cache_meta(input_hash, pinned=True, pin_reason="input_to_published")
|
||||||
|
|
||||||
|
# If this was a config-based run, pin the config and its fixed inputs
|
||||||
|
if run.recipe.startswith("config:"):
|
||||||
|
config_name = run.recipe.replace("config:", "")
|
||||||
|
for config in list_all_configs():
|
||||||
|
if config.name == config_name:
|
||||||
|
# Pin the config YAML
|
||||||
|
cache_manager.pin(config.config_id, reason="config_for_published")
|
||||||
|
# Pin all fixed inputs referenced by the config
|
||||||
|
for fixed in config.fixed_inputs:
|
||||||
|
if fixed.content_hash:
|
||||||
|
cache_manager.pin(fixed.content_hash, reason="fixed_input_in_published_config")
|
||||||
|
break
|
||||||
|
|
||||||
return HTMLResponse(f'''
|
return HTMLResponse(f'''
|
||||||
<div class="bg-green-900/50 border border-green-700 text-green-300 px-4 py-3 rounded-lg mb-4">
|
<div class="bg-green-900/50 border border-green-700 text-green-300 px-4 py-3 rounded-lg mb-4">
|
||||||
Published to L2 as <strong>{result["asset"]["name"]}</strong>
|
Published to L2 as <strong>{result["asset"]["name"]}</strong>
|
||||||
|
|||||||
Reference in New Issue
Block a user