Replace batch DAG system with streaming architecture
- Remove legacy_tasks.py, hybrid_state.py, render.py - Remove old task modules (analyze, execute, execute_sexp, orchestrate) - Add streaming interpreter from test repo - Add sexp_effects with primitives and video effects - Add streaming Celery task with CID-based asset resolution - Support both CID and friendly name references for assets - Add .dockerignore to prevent local clones from conflicting Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
giles
308
streaming/backends.py
Normal file
308
streaming/backends.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
Effect processing backends.
|
||||
|
||||
Provides abstraction over different rendering backends:
|
||||
- numpy: CPU-based, works everywhere, ~3-5 fps
|
||||
- glsl: GPU-based, requires OpenGL, 30+ fps (future)
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Backend(ABC):
|
||||
"""Abstract base class for effect processing backends."""
|
||||
|
||||
@abstractmethod
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Process multiple input frames through effects and composite.
|
||||
|
||||
Args:
|
||||
frames: List of input frames (one per source)
|
||||
effects_per_frame: List of effect chains (one per source)
|
||||
compositor_config: How to blend the layers
|
||||
t: Current time in seconds
|
||||
analysis_data: Analysis data for binding resolution
|
||||
|
||||
Returns:
|
||||
Composited output frame
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
"""Load an effect definition."""
|
||||
pass
|
||||
|
||||
|
||||
class NumpyBackend(Backend):
|
||||
"""
|
||||
CPU-based effect processing using NumPy.
|
||||
|
||||
Uses existing sexp_effects interpreter for effect execution.
|
||||
Works on any system, but limited to ~3-5 fps for complex effects.
|
||||
"""
|
||||
|
||||
def __init__(self, recipe_dir: Path = None, minimal_primitives: bool = True):
|
||||
self.recipe_dir = recipe_dir or Path(".")
|
||||
self.minimal_primitives = minimal_primitives
|
||||
self._interpreter = None
|
||||
self._loaded_effects = {}
|
||||
|
||||
def _get_interpreter(self):
|
||||
"""Lazy-load the sexp interpreter."""
|
||||
if self._interpreter is None:
|
||||
from sexp_effects import get_interpreter
|
||||
self._interpreter = get_interpreter(minimal_primitives=self.minimal_primitives)
|
||||
return self._interpreter
|
||||
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
"""Load an effect from sexp file."""
|
||||
effect_key = str(effect_path)
|
||||
if effect_key not in self._loaded_effects:
|
||||
interp = self._get_interpreter()
|
||||
interp.load_effect(str(effect_path))
|
||||
self._loaded_effects[effect_key] = effect_path.stem
|
||||
return self._loaded_effects[effect_key]
|
||||
|
||||
def _resolve_binding(self, value: Any, t: float, analysis_data: Dict) -> Any:
|
||||
"""Resolve a parameter binding to its value at time t."""
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
|
||||
if "_binding" in value or "_bind" in value:
|
||||
source = value.get("source") or value.get("_bind")
|
||||
feature = value.get("feature", "values")
|
||||
range_map = value.get("range")
|
||||
|
||||
track = analysis_data.get(source, {})
|
||||
times = track.get("times", [])
|
||||
values = track.get("values", [])
|
||||
|
||||
if not times or not values:
|
||||
return 0.0
|
||||
|
||||
# Find value at time t (linear interpolation)
|
||||
if t <= times[0]:
|
||||
val = values[0]
|
||||
elif t >= times[-1]:
|
||||
val = values[-1]
|
||||
else:
|
||||
# Binary search for bracket
|
||||
for i in range(len(times) - 1):
|
||||
if times[i] <= t <= times[i + 1]:
|
||||
alpha = (t - times[i]) / (times[i + 1] - times[i])
|
||||
val = values[i] * (1 - alpha) + values[i + 1] * alpha
|
||||
break
|
||||
else:
|
||||
val = values[-1]
|
||||
|
||||
# Apply range mapping
|
||||
if range_map and len(range_map) == 2:
|
||||
val = range_map[0] + val * (range_map[1] - range_map[0])
|
||||
|
||||
return val
|
||||
|
||||
return value
|
||||
|
||||
def _apply_effect(
|
||||
self,
|
||||
frame: np.ndarray,
|
||||
effect_name: str,
|
||||
params: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""Apply a single effect to a frame."""
|
||||
# Resolve bindings in params
|
||||
resolved_params = {"_time": t}
|
||||
for key, value in params.items():
|
||||
if key in ("effect", "effect_path", "cid", "analysis_refs"):
|
||||
continue
|
||||
resolved_params[key] = self._resolve_binding(value, t, analysis_data)
|
||||
|
||||
# Try fast native effects first
|
||||
result = self._apply_native_effect(frame, effect_name, resolved_params)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Fall back to sexp interpreter for complex effects
|
||||
interp = self._get_interpreter()
|
||||
if effect_name in interp.effects:
|
||||
result, _ = interp.run_effect(effect_name, frame, resolved_params, {})
|
||||
return result
|
||||
|
||||
# Unknown effect - pass through
|
||||
return frame
|
||||
|
||||
def _apply_native_effect(
|
||||
self,
|
||||
frame: np.ndarray,
|
||||
effect_name: str,
|
||||
params: Dict,
|
||||
) -> Optional[np.ndarray]:
|
||||
"""Fast native numpy effects for real-time streaming."""
|
||||
import cv2
|
||||
|
||||
if effect_name == "zoom":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
if abs(amount - 1.0) < 0.01:
|
||||
return frame
|
||||
h, w = frame.shape[:2]
|
||||
# Crop center and resize
|
||||
new_w, new_h = int(w / amount), int(h / amount)
|
||||
x1, y1 = (w - new_w) // 2, (h - new_h) // 2
|
||||
cropped = frame[y1:y1+new_h, x1:x1+new_w]
|
||||
return cv2.resize(cropped, (w, h))
|
||||
|
||||
elif effect_name == "rotate":
|
||||
angle = float(params.get("angle", 0))
|
||||
if abs(angle) < 0.5:
|
||||
return frame
|
||||
h, w = frame.shape[:2]
|
||||
center = (w // 2, h // 2)
|
||||
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
||||
return cv2.warpAffine(frame, matrix, (w, h))
|
||||
|
||||
elif effect_name == "brightness":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
return np.clip(frame * amount, 0, 255).astype(np.uint8)
|
||||
|
||||
elif effect_name == "invert":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
if amount < 0.5:
|
||||
return frame
|
||||
return 255 - frame
|
||||
|
||||
# Not a native effect
|
||||
return None
|
||||
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Process frames through effects and composite.
|
||||
"""
|
||||
if not frames:
|
||||
return np.zeros((720, 1280, 3), dtype=np.uint8)
|
||||
|
||||
processed = []
|
||||
|
||||
# Apply effects to each input frame
|
||||
for i, (frame, effects) in enumerate(zip(frames, effects_per_frame)):
|
||||
result = frame.copy()
|
||||
for effect_config in effects:
|
||||
effect_name = effect_config.get("effect", "")
|
||||
if effect_name:
|
||||
result = self._apply_effect(
|
||||
result, effect_name, effect_config, t, analysis_data
|
||||
)
|
||||
processed.append(result)
|
||||
|
||||
# Composite layers
|
||||
if len(processed) == 1:
|
||||
return processed[0]
|
||||
|
||||
return self._composite(processed, compositor_config, t, analysis_data)
|
||||
|
||||
def _composite(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""Composite multiple frames into one."""
|
||||
mode = config.get("mode", "alpha")
|
||||
weights = config.get("weights", [1.0 / len(frames)] * len(frames))
|
||||
|
||||
# Resolve weight bindings
|
||||
resolved_weights = []
|
||||
for w in weights:
|
||||
resolved_weights.append(self._resolve_binding(w, t, analysis_data))
|
||||
|
||||
# Normalize weights
|
||||
total = sum(resolved_weights)
|
||||
if total > 0:
|
||||
resolved_weights = [w / total for w in resolved_weights]
|
||||
else:
|
||||
resolved_weights = [1.0 / len(frames)] * len(frames)
|
||||
|
||||
# Resize frames to match first frame
|
||||
target_h, target_w = frames[0].shape[:2]
|
||||
resized = []
|
||||
for frame in frames:
|
||||
if frame.shape[:2] != (target_h, target_w):
|
||||
import cv2
|
||||
frame = cv2.resize(frame, (target_w, target_h))
|
||||
resized.append(frame.astype(np.float32))
|
||||
|
||||
# Weighted blend
|
||||
result = np.zeros_like(resized[0])
|
||||
for frame, weight in zip(resized, resolved_weights):
|
||||
result += frame * weight
|
||||
|
||||
return np.clip(result, 0, 255).astype(np.uint8)
|
||||
|
||||
|
||||
class GLSLBackend(Backend):
|
||||
"""
|
||||
GPU-based effect processing using OpenGL/GLSL.
|
||||
|
||||
Requires GPU with OpenGL 3.3+ support (or Mesa software renderer).
|
||||
Achieves 30+ fps real-time processing.
|
||||
|
||||
TODO: Implement when ready for GPU acceleration.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise NotImplementedError(
|
||||
"GLSL backend not yet implemented. Use NumpyBackend for now."
|
||||
)
|
||||
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
pass
|
||||
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
pass
|
||||
|
||||
|
||||
def get_backend(name: str = "numpy", **kwargs) -> Backend:
|
||||
"""
|
||||
Get a backend by name.
|
||||
|
||||
Args:
|
||||
name: "numpy" or "glsl"
|
||||
**kwargs: Backend-specific options
|
||||
|
||||
Returns:
|
||||
Backend instance
|
||||
"""
|
||||
if name == "numpy":
|
||||
return NumpyBackend(**kwargs)
|
||||
elif name == "glsl":
|
||||
return GLSLBackend(**kwargs)
|
||||
else:
|
||||
raise ValueError(f"Unknown backend: {name}")
|
||||
Reference in New Issue
Block a user