Add streaming video compositor with sexp interpreter
- New streaming/ module for real-time video processing: - compositor.py: Main streaming compositor with cycle-crossfade - sexp_executor.py: Executes compiled sexp recipes in real-time - sexp_interp.py: Full S-expression interpreter for SLICE_ON Lambda - recipe_adapter.py: Bridges recipes to streaming compositor - sources.py: Video source with ffmpeg streaming - audio.py: Real-time audio analysis (energy, beats) - output.py: Preview (mpv) and file output with audio muxing - New templates/: - cycle-crossfade.sexp: Smooth zoom-based video cycling - process-pair.sexp: Dual-clip processing with effects - Key features: - Videos cycle in input-videos order (not definition order) - Cumulative whole-spin rotation - Zero-weight sources skip processing - Live audio-reactive effects - New effects: blend_multi for weighted layer compositing - Updated primitives and interpreter for streaming compatibility Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
415
streaming/recipe_executor.py
Normal file
415
streaming/recipe_executor.py
Normal file
@@ -0,0 +1,415 @@
|
||||
"""
|
||||
Streaming recipe executor.
|
||||
|
||||
Implements the full recipe logic for real-time streaming:
|
||||
- Scans (state machines that evolve on beats)
|
||||
- Process-pair template (two clips with sporadic effects, blended)
|
||||
- Cycle-crossfade (dynamic composition cycling through video pairs)
|
||||
"""
|
||||
|
||||
import random
|
||||
import numpy as np
|
||||
from typing import Dict, List, Any, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanState:
|
||||
"""State for a scan (beat-driven state machine)."""
|
||||
value: Any = 0
|
||||
rng: random.Random = field(default_factory=random.Random)
|
||||
|
||||
|
||||
class StreamingScans:
|
||||
"""
|
||||
Real-time scan executor.
|
||||
|
||||
Scans are state machines that evolve on each beat.
|
||||
They drive effect parameters like invert triggers, hue shifts, etc.
|
||||
"""
|
||||
|
||||
def __init__(self, seed: int = 42, n_sources: int = 4):
|
||||
self.master_seed = seed
|
||||
self.n_sources = n_sources
|
||||
self.scans: Dict[str, ScanState] = {}
|
||||
self.beat_count = 0
|
||||
self.current_time = 0.0
|
||||
self.last_beat_time = 0.0
|
||||
self._init_scans()
|
||||
|
||||
def _init_scans(self):
|
||||
"""Initialize all scans with their own RNG seeds."""
|
||||
scan_names = []
|
||||
|
||||
# Per-pair scans (dynamic based on n_sources)
|
||||
for i in range(self.n_sources):
|
||||
scan_names.extend([
|
||||
f"inv_a_{i}", f"inv_b_{i}", f"hue_a_{i}", f"hue_b_{i}",
|
||||
f"ascii_a_{i}", f"ascii_b_{i}", f"pair_mix_{i}", f"pair_rot_{i}",
|
||||
])
|
||||
|
||||
# Global scans
|
||||
scan_names.extend(["whole_spin", "ripple_gate", "cycle"])
|
||||
|
||||
for i, name in enumerate(scan_names):
|
||||
rng = random.Random(self.master_seed + i)
|
||||
self.scans[name] = ScanState(value=self._init_value(name), rng=rng)
|
||||
|
||||
def _init_value(self, name: str) -> Any:
|
||||
"""Get initial value for a scan."""
|
||||
if name.startswith("inv_") or name.startswith("ascii_"):
|
||||
return 0 # Counter for remaining beats
|
||||
elif name.startswith("hue_"):
|
||||
return {"rem": 0, "hue": 0}
|
||||
elif name.startswith("pair_mix"):
|
||||
return {"rem": 0, "opacity": 0.5}
|
||||
elif name.startswith("pair_rot"):
|
||||
pair_idx = int(name.split("_")[-1])
|
||||
rot_dir = 1 if pair_idx % 2 == 0 else -1
|
||||
return {"beat": 0, "clen": 25, "dir": rot_dir, "angle": 0}
|
||||
elif name == "whole_spin":
|
||||
return {
|
||||
"phase": 0, # 0 = waiting, 1 = spinning
|
||||
"beat": 0, # beats into current phase
|
||||
"plen": 20, # beats in this phase
|
||||
"dir": 1, # spin direction
|
||||
"total_angle": 0.0, # cumulative angle after all spins
|
||||
"spin_start_angle": 0.0, # angle when current spin started
|
||||
"spin_start_time": 0.0, # time when current spin started
|
||||
"spin_end_time": 0.0, # estimated time when spin ends
|
||||
}
|
||||
elif name == "ripple_gate":
|
||||
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
||||
elif name == "cycle":
|
||||
return {"cycle": 0, "beat": 0, "clen": 60}
|
||||
return 0
|
||||
|
||||
def on_beat(self):
|
||||
"""Update all scans on a beat."""
|
||||
self.beat_count += 1
|
||||
# Estimate beat interval from last two beats
|
||||
beat_interval = self.current_time - self.last_beat_time if self.last_beat_time > 0 else 0.5
|
||||
self.last_beat_time = self.current_time
|
||||
|
||||
for name, state in self.scans.items():
|
||||
state.value = self._step_scan(name, state.value, state.rng, beat_interval)
|
||||
|
||||
def _step_scan(self, name: str, value: Any, rng: random.Random, beat_interval: float = 0.5) -> Any:
|
||||
"""Step a scan forward by one beat."""
|
||||
|
||||
# Invert scan: 10% chance, lasts 1-5 beats
|
||||
if name.startswith("inv_"):
|
||||
if value > 0:
|
||||
return value - 1
|
||||
elif rng.random() < 0.1:
|
||||
return rng.randint(1, 5)
|
||||
return 0
|
||||
|
||||
# Hue scan: 10% chance, random hue 30-330, lasts 1-5 beats
|
||||
elif name.startswith("hue_"):
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "hue": value["hue"]}
|
||||
elif rng.random() < 0.1:
|
||||
return {"rem": rng.randint(1, 5), "hue": rng.uniform(30, 330)}
|
||||
return {"rem": 0, "hue": 0}
|
||||
|
||||
# ASCII scan: 5% chance, lasts 1-3 beats
|
||||
elif name.startswith("ascii_"):
|
||||
if value > 0:
|
||||
return value - 1
|
||||
elif rng.random() < 0.05:
|
||||
return rng.randint(1, 3)
|
||||
return 0
|
||||
|
||||
# Pair mix: changes every 1-11 beats
|
||||
elif name.startswith("pair_mix"):
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "opacity": value["opacity"]}
|
||||
return {"rem": rng.randint(1, 11), "opacity": rng.choice([0, 0.5, 1.0])}
|
||||
|
||||
# Pair rotation: full rotation every 20-30 beats
|
||||
elif name.startswith("pair_rot"):
|
||||
beat = value["beat"]
|
||||
clen = value["clen"]
|
||||
dir_ = value["dir"]
|
||||
angle = value["angle"]
|
||||
|
||||
if beat + 1 < clen:
|
||||
new_angle = angle + dir_ * (360 / clen)
|
||||
return {"beat": beat + 1, "clen": clen, "dir": dir_, "angle": new_angle}
|
||||
else:
|
||||
return {"beat": 0, "clen": rng.randint(20, 30), "dir": -dir_, "angle": angle}
|
||||
|
||||
# Whole spin: sporadic 720 degree spins (cumulative - stays rotated)
|
||||
elif name == "whole_spin":
|
||||
phase = value["phase"]
|
||||
beat = value["beat"]
|
||||
plen = value["plen"]
|
||||
dir_ = value["dir"]
|
||||
total_angle = value.get("total_angle", 0.0)
|
||||
spin_start_angle = value.get("spin_start_angle", 0.0)
|
||||
spin_start_time = value.get("spin_start_time", 0.0)
|
||||
spin_end_time = value.get("spin_end_time", 0.0)
|
||||
|
||||
if phase == 1:
|
||||
# Currently spinning
|
||||
if beat + 1 < plen:
|
||||
return {
|
||||
"phase": 1, "beat": beat + 1, "plen": plen, "dir": dir_,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": spin_start_angle,
|
||||
"spin_start_time": spin_start_time,
|
||||
"spin_end_time": spin_end_time,
|
||||
}
|
||||
else:
|
||||
# Spin complete - update total_angle with final spin
|
||||
new_total = spin_start_angle + dir_ * 720.0
|
||||
return {
|
||||
"phase": 0, "beat": 0, "plen": rng.randint(20, 40), "dir": dir_,
|
||||
"total_angle": new_total,
|
||||
"spin_start_angle": new_total,
|
||||
"spin_start_time": self.current_time,
|
||||
"spin_end_time": self.current_time,
|
||||
}
|
||||
else:
|
||||
# Waiting phase
|
||||
if beat + 1 < plen:
|
||||
return {
|
||||
"phase": 0, "beat": beat + 1, "plen": plen, "dir": dir_,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": spin_start_angle,
|
||||
"spin_start_time": spin_start_time,
|
||||
"spin_end_time": spin_end_time,
|
||||
}
|
||||
else:
|
||||
# Start new spin
|
||||
new_dir = 1 if rng.random() < 0.5 else -1
|
||||
new_plen = rng.randint(10, 25)
|
||||
spin_duration = new_plen * beat_interval
|
||||
return {
|
||||
"phase": 1, "beat": 0, "plen": new_plen, "dir": new_dir,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": total_angle,
|
||||
"spin_start_time": self.current_time,
|
||||
"spin_end_time": self.current_time + spin_duration,
|
||||
}
|
||||
|
||||
# Ripple gate: 5% chance, lasts 1-20 beats
|
||||
elif name == "ripple_gate":
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "cx": value["cx"], "cy": value["cy"]}
|
||||
elif rng.random() < 0.05:
|
||||
return {"rem": rng.randint(1, 20),
|
||||
"cx": rng.uniform(0.1, 0.9),
|
||||
"cy": rng.uniform(0.1, 0.9)}
|
||||
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
||||
|
||||
# Cycle: track which video pair is active
|
||||
elif name == "cycle":
|
||||
beat = value["beat"]
|
||||
clen = value["clen"]
|
||||
cycle = value["cycle"]
|
||||
|
||||
if beat + 1 < clen:
|
||||
return {"cycle": cycle, "beat": beat + 1, "clen": clen}
|
||||
else:
|
||||
# Move to next pair, vary cycle length
|
||||
return {"cycle": (cycle + 1) % 4, "beat": 0,
|
||||
"clen": 40 + (self.beat_count * 7) % 41}
|
||||
|
||||
return value
|
||||
|
||||
def get_emit(self, name: str) -> float:
|
||||
"""Get emitted value for a scan."""
|
||||
value = self.scans[name].value
|
||||
|
||||
if name.startswith("inv_") or name.startswith("ascii_"):
|
||||
return 1.0 if value > 0 else 0.0
|
||||
|
||||
elif name.startswith("hue_"):
|
||||
return value["hue"] if value["rem"] > 0 else 0.0
|
||||
|
||||
elif name.startswith("pair_mix"):
|
||||
return value["opacity"]
|
||||
|
||||
elif name.startswith("pair_rot"):
|
||||
return value["angle"]
|
||||
|
||||
elif name == "whole_spin":
|
||||
# Smooth time-based interpolation during spin
|
||||
phase = value.get("phase", 0)
|
||||
if phase == 1:
|
||||
# Currently spinning - interpolate based on time
|
||||
spin_start_time = value.get("spin_start_time", 0.0)
|
||||
spin_end_time = value.get("spin_end_time", spin_start_time + 1.0)
|
||||
spin_start_angle = value.get("spin_start_angle", 0.0)
|
||||
dir_ = value.get("dir", 1)
|
||||
|
||||
duration = spin_end_time - spin_start_time
|
||||
if duration > 0:
|
||||
progress = (self.current_time - spin_start_time) / duration
|
||||
progress = max(0.0, min(1.0, progress)) # clamp to 0-1
|
||||
else:
|
||||
progress = 1.0
|
||||
|
||||
return spin_start_angle + progress * 720.0 * dir_
|
||||
else:
|
||||
# Not spinning - return cumulative angle
|
||||
return value.get("total_angle", 0.0)
|
||||
|
||||
elif name == "ripple_gate":
|
||||
return 1.0 if value["rem"] > 0 else 0.0
|
||||
|
||||
elif name == "cycle":
|
||||
return value
|
||||
|
||||
return 0.0
|
||||
|
||||
|
||||
class StreamingRecipeExecutor:
|
||||
"""
|
||||
Executes a recipe in streaming mode.
|
||||
|
||||
Implements:
|
||||
- process-pair: two video clips with opposite effects, blended
|
||||
- cycle-crossfade: dynamic cycling through video pairs
|
||||
- Final effects: whole-spin rotation, ripple
|
||||
"""
|
||||
|
||||
def __init__(self, n_sources: int = 4, seed: int = 42):
|
||||
self.n_sources = n_sources
|
||||
self.scans = StreamingScans(seed, n_sources=n_sources)
|
||||
self.last_beat_detected = False
|
||||
self.current_time = 0.0
|
||||
|
||||
def on_frame(self, energy: float, is_beat: bool, t: float = 0.0):
|
||||
"""Called each frame with current audio analysis."""
|
||||
self.current_time = t
|
||||
self.scans.current_time = t
|
||||
# Update scans on beat
|
||||
if is_beat and not self.last_beat_detected:
|
||||
self.scans.on_beat()
|
||||
self.last_beat_detected = is_beat
|
||||
|
||||
def get_effect_params(self, source_idx: int, clip: str, energy: float) -> Dict:
|
||||
"""
|
||||
Get effect parameters for a source clip.
|
||||
|
||||
Args:
|
||||
source_idx: Which video source (0-3)
|
||||
clip: "a" or "b" (each source has two clips)
|
||||
energy: Current audio energy (0-1)
|
||||
"""
|
||||
suffix = f"_{source_idx}"
|
||||
|
||||
# Rotation ranges alternate
|
||||
if source_idx % 2 == 0:
|
||||
rot_range = [0, 45] if clip == "a" else [0, -45]
|
||||
zoom_range = [1, 1.5] if clip == "a" else [1, 0.5]
|
||||
else:
|
||||
rot_range = [0, -45] if clip == "a" else [0, 45]
|
||||
zoom_range = [1, 0.5] if clip == "a" else [1, 1.5]
|
||||
|
||||
return {
|
||||
"rotate_angle": rot_range[0] + energy * (rot_range[1] - rot_range[0]),
|
||||
"zoom_amount": zoom_range[0] + energy * (zoom_range[1] - zoom_range[0]),
|
||||
"invert_amount": self.scans.get_emit(f"inv_{clip}{suffix}"),
|
||||
"hue_degrees": self.scans.get_emit(f"hue_{clip}{suffix}"),
|
||||
"ascii_mix": 0, # Disabled - too slow without GPU
|
||||
"ascii_char_size": 4 + energy * 28, # 4-32
|
||||
}
|
||||
|
||||
def get_pair_params(self, source_idx: int) -> Dict:
|
||||
"""Get blend and rotation params for a video pair."""
|
||||
suffix = f"_{source_idx}"
|
||||
return {
|
||||
"blend_opacity": self.scans.get_emit(f"pair_mix{suffix}"),
|
||||
"pair_rotation": self.scans.get_emit(f"pair_rot{suffix}"),
|
||||
}
|
||||
|
||||
def get_cycle_weights(self) -> List[float]:
|
||||
"""Get blend weights for cycle-crossfade composition."""
|
||||
cycle_state = self.scans.get_emit("cycle")
|
||||
active = cycle_state["cycle"]
|
||||
beat = cycle_state["beat"]
|
||||
clen = cycle_state["clen"]
|
||||
n = self.n_sources
|
||||
|
||||
phase3 = beat * 3
|
||||
weights = []
|
||||
|
||||
for p in range(n):
|
||||
prev = (p + n - 1) % n
|
||||
|
||||
if active == p:
|
||||
if phase3 < clen:
|
||||
w = 0.9
|
||||
elif phase3 < clen * 2:
|
||||
w = 0.9 - ((phase3 - clen) / clen) * 0.85
|
||||
else:
|
||||
w = 0.05
|
||||
elif active == prev:
|
||||
if phase3 < clen:
|
||||
w = 0.05
|
||||
elif phase3 < clen * 2:
|
||||
w = 0.05 + ((phase3 - clen) / clen) * 0.85
|
||||
else:
|
||||
w = 0.9
|
||||
else:
|
||||
w = 0.05
|
||||
|
||||
weights.append(w)
|
||||
|
||||
# Normalize
|
||||
total = sum(weights)
|
||||
if total > 0:
|
||||
weights = [w / total for w in weights]
|
||||
|
||||
return weights
|
||||
|
||||
def get_cycle_zooms(self) -> List[float]:
|
||||
"""Get zoom amounts for cycle-crossfade."""
|
||||
cycle_state = self.scans.get_emit("cycle")
|
||||
active = cycle_state["cycle"]
|
||||
beat = cycle_state["beat"]
|
||||
clen = cycle_state["clen"]
|
||||
n = self.n_sources
|
||||
|
||||
phase3 = beat * 3
|
||||
zooms = []
|
||||
|
||||
for p in range(n):
|
||||
prev = (p + n - 1) % n
|
||||
|
||||
if active == p:
|
||||
if phase3 < clen:
|
||||
z = 1.0
|
||||
elif phase3 < clen * 2:
|
||||
z = 1.0 + ((phase3 - clen) / clen) * 1.0
|
||||
else:
|
||||
z = 0.1
|
||||
elif active == prev:
|
||||
if phase3 < clen:
|
||||
z = 3.0 # Start big
|
||||
elif phase3 < clen * 2:
|
||||
z = 3.0 - ((phase3 - clen) / clen) * 2.0 # Shrink to 1.0
|
||||
else:
|
||||
z = 1.0
|
||||
else:
|
||||
z = 0.1
|
||||
|
||||
zooms.append(z)
|
||||
|
||||
return zooms
|
||||
|
||||
def get_final_effects(self, energy: float) -> Dict:
|
||||
"""Get final composition effects (whole-spin, ripple)."""
|
||||
ripple_gate = self.scans.get_emit("ripple_gate")
|
||||
ripple_state = self.scans.scans["ripple_gate"].value
|
||||
|
||||
return {
|
||||
"whole_spin_angle": self.scans.get_emit("whole_spin"),
|
||||
"ripple_amplitude": ripple_gate * (5 + energy * 45), # 5-50
|
||||
"ripple_cx": ripple_state["cx"],
|
||||
"ripple_cy": ripple_state["cy"],
|
||||
}
|
||||
Reference in New Issue
Block a user