416 lines
15 KiB
Python
416 lines
15 KiB
Python
"""
|
|
Streaming recipe executor.
|
|
|
|
Implements the full recipe logic for real-time streaming:
|
|
- Scans (state machines that evolve on beats)
|
|
- Process-pair template (two clips with sporadic effects, blended)
|
|
- Cycle-crossfade (dynamic composition cycling through video pairs)
|
|
"""
|
|
|
|
import random
|
|
import numpy as np
|
|
from typing import Dict, List, Any, Optional
|
|
from dataclasses import dataclass, field
|
|
|
|
|
|
@dataclass
|
|
class ScanState:
|
|
"""State for a scan (beat-driven state machine)."""
|
|
value: Any = 0
|
|
rng: random.Random = field(default_factory=random.Random)
|
|
|
|
|
|
class StreamingScans:
|
|
"""
|
|
Real-time scan executor.
|
|
|
|
Scans are state machines that evolve on each beat.
|
|
They drive effect parameters like invert triggers, hue shifts, etc.
|
|
"""
|
|
|
|
def __init__(self, seed: int = 42, n_sources: int = 4):
|
|
self.master_seed = seed
|
|
self.n_sources = n_sources
|
|
self.scans: Dict[str, ScanState] = {}
|
|
self.beat_count = 0
|
|
self.current_time = 0.0
|
|
self.last_beat_time = 0.0
|
|
self._init_scans()
|
|
|
|
def _init_scans(self):
|
|
"""Initialize all scans with their own RNG seeds."""
|
|
scan_names = []
|
|
|
|
# Per-pair scans (dynamic based on n_sources)
|
|
for i in range(self.n_sources):
|
|
scan_names.extend([
|
|
f"inv_a_{i}", f"inv_b_{i}", f"hue_a_{i}", f"hue_b_{i}",
|
|
f"ascii_a_{i}", f"ascii_b_{i}", f"pair_mix_{i}", f"pair_rot_{i}",
|
|
])
|
|
|
|
# Global scans
|
|
scan_names.extend(["whole_spin", "ripple_gate", "cycle"])
|
|
|
|
for i, name in enumerate(scan_names):
|
|
rng = random.Random(self.master_seed + i)
|
|
self.scans[name] = ScanState(value=self._init_value(name), rng=rng)
|
|
|
|
def _init_value(self, name: str) -> Any:
|
|
"""Get initial value for a scan."""
|
|
if name.startswith("inv_") or name.startswith("ascii_"):
|
|
return 0 # Counter for remaining beats
|
|
elif name.startswith("hue_"):
|
|
return {"rem": 0, "hue": 0}
|
|
elif name.startswith("pair_mix"):
|
|
return {"rem": 0, "opacity": 0.5}
|
|
elif name.startswith("pair_rot"):
|
|
pair_idx = int(name.split("_")[-1])
|
|
rot_dir = 1 if pair_idx % 2 == 0 else -1
|
|
return {"beat": 0, "clen": 25, "dir": rot_dir, "angle": 0}
|
|
elif name == "whole_spin":
|
|
return {
|
|
"phase": 0, # 0 = waiting, 1 = spinning
|
|
"beat": 0, # beats into current phase
|
|
"plen": 20, # beats in this phase
|
|
"dir": 1, # spin direction
|
|
"total_angle": 0.0, # cumulative angle after all spins
|
|
"spin_start_angle": 0.0, # angle when current spin started
|
|
"spin_start_time": 0.0, # time when current spin started
|
|
"spin_end_time": 0.0, # estimated time when spin ends
|
|
}
|
|
elif name == "ripple_gate":
|
|
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
|
elif name == "cycle":
|
|
return {"cycle": 0, "beat": 0, "clen": 60}
|
|
return 0
|
|
|
|
def on_beat(self):
|
|
"""Update all scans on a beat."""
|
|
self.beat_count += 1
|
|
# Estimate beat interval from last two beats
|
|
beat_interval = self.current_time - self.last_beat_time if self.last_beat_time > 0 else 0.5
|
|
self.last_beat_time = self.current_time
|
|
|
|
for name, state in self.scans.items():
|
|
state.value = self._step_scan(name, state.value, state.rng, beat_interval)
|
|
|
|
def _step_scan(self, name: str, value: Any, rng: random.Random, beat_interval: float = 0.5) -> Any:
|
|
"""Step a scan forward by one beat."""
|
|
|
|
# Invert scan: 10% chance, lasts 1-5 beats
|
|
if name.startswith("inv_"):
|
|
if value > 0:
|
|
return value - 1
|
|
elif rng.random() < 0.1:
|
|
return rng.randint(1, 5)
|
|
return 0
|
|
|
|
# Hue scan: 10% chance, random hue 30-330, lasts 1-5 beats
|
|
elif name.startswith("hue_"):
|
|
if value["rem"] > 0:
|
|
return {"rem": value["rem"] - 1, "hue": value["hue"]}
|
|
elif rng.random() < 0.1:
|
|
return {"rem": rng.randint(1, 5), "hue": rng.uniform(30, 330)}
|
|
return {"rem": 0, "hue": 0}
|
|
|
|
# ASCII scan: 5% chance, lasts 1-3 beats
|
|
elif name.startswith("ascii_"):
|
|
if value > 0:
|
|
return value - 1
|
|
elif rng.random() < 0.05:
|
|
return rng.randint(1, 3)
|
|
return 0
|
|
|
|
# Pair mix: changes every 1-11 beats
|
|
elif name.startswith("pair_mix"):
|
|
if value["rem"] > 0:
|
|
return {"rem": value["rem"] - 1, "opacity": value["opacity"]}
|
|
return {"rem": rng.randint(1, 11), "opacity": rng.choice([0, 0.5, 1.0])}
|
|
|
|
# Pair rotation: full rotation every 20-30 beats
|
|
elif name.startswith("pair_rot"):
|
|
beat = value["beat"]
|
|
clen = value["clen"]
|
|
dir_ = value["dir"]
|
|
angle = value["angle"]
|
|
|
|
if beat + 1 < clen:
|
|
new_angle = angle + dir_ * (360 / clen)
|
|
return {"beat": beat + 1, "clen": clen, "dir": dir_, "angle": new_angle}
|
|
else:
|
|
return {"beat": 0, "clen": rng.randint(20, 30), "dir": -dir_, "angle": angle}
|
|
|
|
# Whole spin: sporadic 720 degree spins (cumulative - stays rotated)
|
|
elif name == "whole_spin":
|
|
phase = value["phase"]
|
|
beat = value["beat"]
|
|
plen = value["plen"]
|
|
dir_ = value["dir"]
|
|
total_angle = value.get("total_angle", 0.0)
|
|
spin_start_angle = value.get("spin_start_angle", 0.0)
|
|
spin_start_time = value.get("spin_start_time", 0.0)
|
|
spin_end_time = value.get("spin_end_time", 0.0)
|
|
|
|
if phase == 1:
|
|
# Currently spinning
|
|
if beat + 1 < plen:
|
|
return {
|
|
"phase": 1, "beat": beat + 1, "plen": plen, "dir": dir_,
|
|
"total_angle": total_angle,
|
|
"spin_start_angle": spin_start_angle,
|
|
"spin_start_time": spin_start_time,
|
|
"spin_end_time": spin_end_time,
|
|
}
|
|
else:
|
|
# Spin complete - update total_angle with final spin
|
|
new_total = spin_start_angle + dir_ * 720.0
|
|
return {
|
|
"phase": 0, "beat": 0, "plen": rng.randint(20, 40), "dir": dir_,
|
|
"total_angle": new_total,
|
|
"spin_start_angle": new_total,
|
|
"spin_start_time": self.current_time,
|
|
"spin_end_time": self.current_time,
|
|
}
|
|
else:
|
|
# Waiting phase
|
|
if beat + 1 < plen:
|
|
return {
|
|
"phase": 0, "beat": beat + 1, "plen": plen, "dir": dir_,
|
|
"total_angle": total_angle,
|
|
"spin_start_angle": spin_start_angle,
|
|
"spin_start_time": spin_start_time,
|
|
"spin_end_time": spin_end_time,
|
|
}
|
|
else:
|
|
# Start new spin
|
|
new_dir = 1 if rng.random() < 0.5 else -1
|
|
new_plen = rng.randint(10, 25)
|
|
spin_duration = new_plen * beat_interval
|
|
return {
|
|
"phase": 1, "beat": 0, "plen": new_plen, "dir": new_dir,
|
|
"total_angle": total_angle,
|
|
"spin_start_angle": total_angle,
|
|
"spin_start_time": self.current_time,
|
|
"spin_end_time": self.current_time + spin_duration,
|
|
}
|
|
|
|
# Ripple gate: 5% chance, lasts 1-20 beats
|
|
elif name == "ripple_gate":
|
|
if value["rem"] > 0:
|
|
return {"rem": value["rem"] - 1, "cx": value["cx"], "cy": value["cy"]}
|
|
elif rng.random() < 0.05:
|
|
return {"rem": rng.randint(1, 20),
|
|
"cx": rng.uniform(0.1, 0.9),
|
|
"cy": rng.uniform(0.1, 0.9)}
|
|
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
|
|
|
# Cycle: track which video pair is active
|
|
elif name == "cycle":
|
|
beat = value["beat"]
|
|
clen = value["clen"]
|
|
cycle = value["cycle"]
|
|
|
|
if beat + 1 < clen:
|
|
return {"cycle": cycle, "beat": beat + 1, "clen": clen}
|
|
else:
|
|
# Move to next pair, vary cycle length
|
|
return {"cycle": (cycle + 1) % 4, "beat": 0,
|
|
"clen": 40 + (self.beat_count * 7) % 41}
|
|
|
|
return value
|
|
|
|
def get_emit(self, name: str) -> float:
|
|
"""Get emitted value for a scan."""
|
|
value = self.scans[name].value
|
|
|
|
if name.startswith("inv_") or name.startswith("ascii_"):
|
|
return 1.0 if value > 0 else 0.0
|
|
|
|
elif name.startswith("hue_"):
|
|
return value["hue"] if value["rem"] > 0 else 0.0
|
|
|
|
elif name.startswith("pair_mix"):
|
|
return value["opacity"]
|
|
|
|
elif name.startswith("pair_rot"):
|
|
return value["angle"]
|
|
|
|
elif name == "whole_spin":
|
|
# Smooth time-based interpolation during spin
|
|
phase = value.get("phase", 0)
|
|
if phase == 1:
|
|
# Currently spinning - interpolate based on time
|
|
spin_start_time = value.get("spin_start_time", 0.0)
|
|
spin_end_time = value.get("spin_end_time", spin_start_time + 1.0)
|
|
spin_start_angle = value.get("spin_start_angle", 0.0)
|
|
dir_ = value.get("dir", 1)
|
|
|
|
duration = spin_end_time - spin_start_time
|
|
if duration > 0:
|
|
progress = (self.current_time - spin_start_time) / duration
|
|
progress = max(0.0, min(1.0, progress)) # clamp to 0-1
|
|
else:
|
|
progress = 1.0
|
|
|
|
return spin_start_angle + progress * 720.0 * dir_
|
|
else:
|
|
# Not spinning - return cumulative angle
|
|
return value.get("total_angle", 0.0)
|
|
|
|
elif name == "ripple_gate":
|
|
return 1.0 if value["rem"] > 0 else 0.0
|
|
|
|
elif name == "cycle":
|
|
return value
|
|
|
|
return 0.0
|
|
|
|
|
|
class StreamingRecipeExecutor:
|
|
"""
|
|
Executes a recipe in streaming mode.
|
|
|
|
Implements:
|
|
- process-pair: two video clips with opposite effects, blended
|
|
- cycle-crossfade: dynamic cycling through video pairs
|
|
- Final effects: whole-spin rotation, ripple
|
|
"""
|
|
|
|
def __init__(self, n_sources: int = 4, seed: int = 42):
|
|
self.n_sources = n_sources
|
|
self.scans = StreamingScans(seed, n_sources=n_sources)
|
|
self.last_beat_detected = False
|
|
self.current_time = 0.0
|
|
|
|
def on_frame(self, energy: float, is_beat: bool, t: float = 0.0):
|
|
"""Called each frame with current audio analysis."""
|
|
self.current_time = t
|
|
self.scans.current_time = t
|
|
# Update scans on beat
|
|
if is_beat and not self.last_beat_detected:
|
|
self.scans.on_beat()
|
|
self.last_beat_detected = is_beat
|
|
|
|
def get_effect_params(self, source_idx: int, clip: str, energy: float) -> Dict:
|
|
"""
|
|
Get effect parameters for a source clip.
|
|
|
|
Args:
|
|
source_idx: Which video source (0-3)
|
|
clip: "a" or "b" (each source has two clips)
|
|
energy: Current audio energy (0-1)
|
|
"""
|
|
suffix = f"_{source_idx}"
|
|
|
|
# Rotation ranges alternate
|
|
if source_idx % 2 == 0:
|
|
rot_range = [0, 45] if clip == "a" else [0, -45]
|
|
zoom_range = [1, 1.5] if clip == "a" else [1, 0.5]
|
|
else:
|
|
rot_range = [0, -45] if clip == "a" else [0, 45]
|
|
zoom_range = [1, 0.5] if clip == "a" else [1, 1.5]
|
|
|
|
return {
|
|
"rotate_angle": rot_range[0] + energy * (rot_range[1] - rot_range[0]),
|
|
"zoom_amount": zoom_range[0] + energy * (zoom_range[1] - zoom_range[0]),
|
|
"invert_amount": self.scans.get_emit(f"inv_{clip}{suffix}"),
|
|
"hue_degrees": self.scans.get_emit(f"hue_{clip}{suffix}"),
|
|
"ascii_mix": 0, # Disabled - too slow without GPU
|
|
"ascii_char_size": 4 + energy * 28, # 4-32
|
|
}
|
|
|
|
def get_pair_params(self, source_idx: int) -> Dict:
|
|
"""Get blend and rotation params for a video pair."""
|
|
suffix = f"_{source_idx}"
|
|
return {
|
|
"blend_opacity": self.scans.get_emit(f"pair_mix{suffix}"),
|
|
"pair_rotation": self.scans.get_emit(f"pair_rot{suffix}"),
|
|
}
|
|
|
|
def get_cycle_weights(self) -> List[float]:
|
|
"""Get blend weights for cycle-crossfade composition."""
|
|
cycle_state = self.scans.get_emit("cycle")
|
|
active = cycle_state["cycle"]
|
|
beat = cycle_state["beat"]
|
|
clen = cycle_state["clen"]
|
|
n = self.n_sources
|
|
|
|
phase3 = beat * 3
|
|
weights = []
|
|
|
|
for p in range(n):
|
|
prev = (p + n - 1) % n
|
|
|
|
if active == p:
|
|
if phase3 < clen:
|
|
w = 0.9
|
|
elif phase3 < clen * 2:
|
|
w = 0.9 - ((phase3 - clen) / clen) * 0.85
|
|
else:
|
|
w = 0.05
|
|
elif active == prev:
|
|
if phase3 < clen:
|
|
w = 0.05
|
|
elif phase3 < clen * 2:
|
|
w = 0.05 + ((phase3 - clen) / clen) * 0.85
|
|
else:
|
|
w = 0.9
|
|
else:
|
|
w = 0.05
|
|
|
|
weights.append(w)
|
|
|
|
# Normalize
|
|
total = sum(weights)
|
|
if total > 0:
|
|
weights = [w / total for w in weights]
|
|
|
|
return weights
|
|
|
|
def get_cycle_zooms(self) -> List[float]:
|
|
"""Get zoom amounts for cycle-crossfade."""
|
|
cycle_state = self.scans.get_emit("cycle")
|
|
active = cycle_state["cycle"]
|
|
beat = cycle_state["beat"]
|
|
clen = cycle_state["clen"]
|
|
n = self.n_sources
|
|
|
|
phase3 = beat * 3
|
|
zooms = []
|
|
|
|
for p in range(n):
|
|
prev = (p + n - 1) % n
|
|
|
|
if active == p:
|
|
if phase3 < clen:
|
|
z = 1.0
|
|
elif phase3 < clen * 2:
|
|
z = 1.0 + ((phase3 - clen) / clen) * 1.0
|
|
else:
|
|
z = 0.1
|
|
elif active == prev:
|
|
if phase3 < clen:
|
|
z = 3.0 # Start big
|
|
elif phase3 < clen * 2:
|
|
z = 3.0 - ((phase3 - clen) / clen) * 2.0 # Shrink to 1.0
|
|
else:
|
|
z = 1.0
|
|
else:
|
|
z = 0.1
|
|
|
|
zooms.append(z)
|
|
|
|
return zooms
|
|
|
|
def get_final_effects(self, energy: float) -> Dict:
|
|
"""Get final composition effects (whole-spin, ripple)."""
|
|
ripple_gate = self.scans.get_emit("ripple_gate")
|
|
ripple_state = self.scans.scans["ripple_gate"].value
|
|
|
|
return {
|
|
"whole_spin_angle": self.scans.get_emit("whole_spin"),
|
|
"ripple_amplitude": ripple_gate * (5 + energy * 45), # 5-50
|
|
"ripple_cx": ripple_state["cx"],
|
|
"ripple_cy": ripple_state["cy"],
|
|
}
|