Replace batch DAG system with streaming architecture
- Remove legacy_tasks.py, hybrid_state.py, render.py - Remove old task modules (analyze, execute, execute_sexp, orchestrate) - Add streaming interpreter from test repo - Add sexp_effects with primitives and video effects - Add streaming Celery task with CID-based asset resolution - Support both CID and friendly name references for assets - Add .dockerignore to prevent local clones from conflicting Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
44
streaming/__init__.py
Normal file
44
streaming/__init__.py
Normal file
@@ -0,0 +1,44 @@
|
||||
"""
|
||||
Streaming video compositor for real-time effect processing.
|
||||
|
||||
This module provides a frame-by-frame streaming architecture that:
|
||||
- Reads from multiple video sources with automatic looping
|
||||
- Applies effects inline (no intermediate files)
|
||||
- Composites layers with time-varying weights
|
||||
- Outputs to display, file, or stream
|
||||
|
||||
Usage:
|
||||
from streaming import StreamingCompositor, VideoSource, AudioAnalyzer
|
||||
|
||||
compositor = StreamingCompositor(
|
||||
sources=["video1.mp4", "video2.mp4"],
|
||||
effects_per_source=[...],
|
||||
compositor_config={...},
|
||||
)
|
||||
|
||||
# With live audio
|
||||
audio = AudioAnalyzer(device=0)
|
||||
compositor.run(output="output.mp4", duration=60, audio=audio)
|
||||
|
||||
# With preview window
|
||||
compositor.run(output="preview", duration=60)
|
||||
|
||||
Backends:
|
||||
- numpy: Works everywhere, ~3-5 fps (default)
|
||||
- glsl: Requires GPU, 30+ fps real-time (future)
|
||||
"""
|
||||
|
||||
from .sources import VideoSource, ImageSource
|
||||
from .compositor import StreamingCompositor
|
||||
from .backends import NumpyBackend, get_backend
|
||||
from .output import DisplayOutput, FileOutput
|
||||
|
||||
__all__ = [
|
||||
"StreamingCompositor",
|
||||
"VideoSource",
|
||||
"ImageSource",
|
||||
"NumpyBackend",
|
||||
"get_backend",
|
||||
"DisplayOutput",
|
||||
"FileOutput",
|
||||
]
|
||||
486
streaming/audio.py
Normal file
486
streaming/audio.py
Normal file
@@ -0,0 +1,486 @@
|
||||
"""
|
||||
Live audio analysis for reactive effects.
|
||||
|
||||
Provides real-time audio features:
|
||||
- Energy (RMS amplitude)
|
||||
- Beat detection
|
||||
- Frequency bands (bass, mid, high)
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from typing import Optional
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
class AudioAnalyzer:
|
||||
"""
|
||||
Real-time audio analyzer using sounddevice.
|
||||
|
||||
Captures audio from microphone/line-in and computes
|
||||
features in real-time for effect parameter bindings.
|
||||
|
||||
Example:
|
||||
analyzer = AudioAnalyzer(device=0)
|
||||
analyzer.start()
|
||||
|
||||
# In compositor loop:
|
||||
energy = analyzer.get_energy()
|
||||
beat = analyzer.get_beat()
|
||||
|
||||
analyzer.stop()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device: int = None,
|
||||
sample_rate: int = 44100,
|
||||
block_size: int = 1024,
|
||||
buffer_seconds: float = 0.5,
|
||||
):
|
||||
"""
|
||||
Initialize audio analyzer.
|
||||
|
||||
Args:
|
||||
device: Audio input device index (None = default)
|
||||
sample_rate: Audio sample rate
|
||||
block_size: Samples per block
|
||||
buffer_seconds: Ring buffer duration
|
||||
"""
|
||||
self.sample_rate = sample_rate
|
||||
self.block_size = block_size
|
||||
self.device = device
|
||||
|
||||
# Ring buffer for recent audio
|
||||
buffer_size = int(sample_rate * buffer_seconds)
|
||||
self._buffer = np.zeros(buffer_size, dtype=np.float32)
|
||||
self._buffer_pos = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Beat detection state
|
||||
self._last_energy = 0
|
||||
self._energy_history = []
|
||||
self._last_beat_time = 0
|
||||
self._beat_threshold = 1.5 # Energy ratio for beat detection
|
||||
self._min_beat_interval = 0.1 # Min seconds between beats
|
||||
|
||||
# Stream state
|
||||
self._stream = None
|
||||
self._running = False
|
||||
|
||||
def _audio_callback(self, indata, frames, time_info, status):
|
||||
"""Called by sounddevice for each audio block."""
|
||||
with self._lock:
|
||||
# Add to ring buffer
|
||||
data = indata[:, 0] if len(indata.shape) > 1 else indata
|
||||
n = len(data)
|
||||
if self._buffer_pos + n <= len(self._buffer):
|
||||
self._buffer[self._buffer_pos:self._buffer_pos + n] = data
|
||||
else:
|
||||
# Wrap around
|
||||
first = len(self._buffer) - self._buffer_pos
|
||||
self._buffer[self._buffer_pos:] = data[:first]
|
||||
self._buffer[:n - first] = data[first:]
|
||||
self._buffer_pos = (self._buffer_pos + n) % len(self._buffer)
|
||||
|
||||
def start(self):
|
||||
"""Start audio capture."""
|
||||
try:
|
||||
import sounddevice as sd
|
||||
except ImportError:
|
||||
print("Warning: sounddevice not installed. Audio analysis disabled.")
|
||||
print("Install with: pip install sounddevice")
|
||||
return
|
||||
|
||||
self._stream = sd.InputStream(
|
||||
device=self.device,
|
||||
channels=1,
|
||||
samplerate=self.sample_rate,
|
||||
blocksize=self.block_size,
|
||||
callback=self._audio_callback,
|
||||
)
|
||||
self._stream.start()
|
||||
self._running = True
|
||||
|
||||
def stop(self):
|
||||
"""Stop audio capture."""
|
||||
if self._stream:
|
||||
self._stream.stop()
|
||||
self._stream.close()
|
||||
self._stream = None
|
||||
self._running = False
|
||||
|
||||
def get_energy(self) -> float:
|
||||
"""
|
||||
Get current audio energy (RMS amplitude).
|
||||
|
||||
Returns:
|
||||
Energy value normalized to 0-1 range (approximately)
|
||||
"""
|
||||
with self._lock:
|
||||
# Use recent samples
|
||||
recent = 2048
|
||||
if self._buffer_pos >= recent:
|
||||
data = self._buffer[self._buffer_pos - recent:self._buffer_pos]
|
||||
else:
|
||||
data = np.concatenate([
|
||||
self._buffer[-(recent - self._buffer_pos):],
|
||||
self._buffer[:self._buffer_pos]
|
||||
])
|
||||
|
||||
# RMS energy
|
||||
rms = np.sqrt(np.mean(data ** 2))
|
||||
|
||||
# Normalize (typical mic input is quite low)
|
||||
normalized = min(1.0, rms * 10)
|
||||
|
||||
return normalized
|
||||
|
||||
def get_beat(self) -> bool:
|
||||
"""
|
||||
Detect if current moment is a beat.
|
||||
|
||||
Simple onset detection based on energy spikes.
|
||||
|
||||
Returns:
|
||||
True if beat detected, False otherwise
|
||||
"""
|
||||
current_energy = self.get_energy()
|
||||
now = time.time()
|
||||
|
||||
# Update energy history
|
||||
self._energy_history.append(current_energy)
|
||||
if len(self._energy_history) > 20:
|
||||
self._energy_history.pop(0)
|
||||
|
||||
# Need enough history
|
||||
if len(self._energy_history) < 5:
|
||||
self._last_energy = current_energy
|
||||
return False
|
||||
|
||||
# Average recent energy
|
||||
avg_energy = np.mean(self._energy_history[:-1])
|
||||
|
||||
# Beat if current energy is significantly above average
|
||||
is_beat = (
|
||||
current_energy > avg_energy * self._beat_threshold and
|
||||
now - self._last_beat_time > self._min_beat_interval and
|
||||
current_energy > self._last_energy # Rising edge
|
||||
)
|
||||
|
||||
if is_beat:
|
||||
self._last_beat_time = now
|
||||
|
||||
self._last_energy = current_energy
|
||||
return is_beat
|
||||
|
||||
def get_spectrum(self, bands: int = 3) -> np.ndarray:
|
||||
"""
|
||||
Get frequency spectrum divided into bands.
|
||||
|
||||
Args:
|
||||
bands: Number of frequency bands (default 3: bass, mid, high)
|
||||
|
||||
Returns:
|
||||
Array of band energies, normalized to 0-1
|
||||
"""
|
||||
with self._lock:
|
||||
# Use recent samples for FFT
|
||||
n = 2048
|
||||
if self._buffer_pos >= n:
|
||||
data = self._buffer[self._buffer_pos - n:self._buffer_pos]
|
||||
else:
|
||||
data = np.concatenate([
|
||||
self._buffer[-(n - self._buffer_pos):],
|
||||
self._buffer[:self._buffer_pos]
|
||||
])
|
||||
|
||||
# FFT
|
||||
fft = np.abs(np.fft.rfft(data * np.hanning(len(data))))
|
||||
|
||||
# Divide into bands
|
||||
band_size = len(fft) // bands
|
||||
result = np.zeros(bands)
|
||||
for i in range(bands):
|
||||
start = i * band_size
|
||||
end = start + band_size
|
||||
result[i] = np.mean(fft[start:end])
|
||||
|
||||
# Normalize
|
||||
max_val = np.max(result)
|
||||
if max_val > 0:
|
||||
result = result / max_val
|
||||
|
||||
return result
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._running
|
||||
|
||||
def __enter__(self):
|
||||
self.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
self.stop()
|
||||
|
||||
|
||||
class FileAudioAnalyzer:
|
||||
"""
|
||||
Audio analyzer that reads from a file (for testing/development).
|
||||
|
||||
Pre-computes analysis and plays back in sync with video.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str, analysis_data: dict = None):
|
||||
"""
|
||||
Initialize from audio file.
|
||||
|
||||
Args:
|
||||
path: Path to audio file
|
||||
analysis_data: Pre-computed analysis (times, values, etc.)
|
||||
"""
|
||||
self.path = path
|
||||
self.analysis_data = analysis_data or {}
|
||||
self._current_time = 0
|
||||
|
||||
def set_time(self, t: float):
|
||||
"""Set current playback time."""
|
||||
self._current_time = t
|
||||
|
||||
def get_energy(self) -> float:
|
||||
"""Get energy at current time from pre-computed data."""
|
||||
track = self.analysis_data.get("energy", {})
|
||||
return self._interpolate(track, self._current_time)
|
||||
|
||||
def get_beat(self) -> bool:
|
||||
"""Check if current time is near a beat."""
|
||||
track = self.analysis_data.get("beats", {})
|
||||
times = track.get("times", [])
|
||||
|
||||
# Check if we're within 50ms of a beat
|
||||
for beat_time in times:
|
||||
if abs(beat_time - self._current_time) < 0.05:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _interpolate(self, track: dict, t: float) -> float:
|
||||
"""Interpolate value at time t."""
|
||||
times = track.get("times", [])
|
||||
values = track.get("values", [])
|
||||
|
||||
if not times or not values:
|
||||
return 0.0
|
||||
|
||||
if t <= times[0]:
|
||||
return values[0]
|
||||
if t >= times[-1]:
|
||||
return values[-1]
|
||||
|
||||
# Find bracket and interpolate
|
||||
for i in range(len(times) - 1):
|
||||
if times[i] <= t <= times[i + 1]:
|
||||
alpha = (t - times[i]) / (times[i + 1] - times[i])
|
||||
return values[i] * (1 - alpha) + values[i + 1] * alpha
|
||||
|
||||
return values[-1]
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class StreamingAudioAnalyzer:
|
||||
"""
|
||||
Real-time audio analyzer that streams from a file.
|
||||
|
||||
Reads audio in sync with video time and computes features on-the-fly.
|
||||
No pre-computation needed - analysis happens as frames are processed.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str, sample_rate: int = 22050, hop_length: int = 512):
|
||||
"""
|
||||
Initialize streaming audio analyzer.
|
||||
|
||||
Args:
|
||||
path: Path to audio file
|
||||
sample_rate: Sample rate for analysis
|
||||
hop_length: Hop length for feature extraction
|
||||
"""
|
||||
import subprocess
|
||||
import json
|
||||
|
||||
self.path = path
|
||||
self.sample_rate = sample_rate
|
||||
self.hop_length = hop_length
|
||||
self._current_time = 0.0
|
||||
|
||||
# Get audio duration
|
||||
cmd = ["ffprobe", "-v", "quiet", "-print_format", "json",
|
||||
"-show_format", str(path)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
info = json.loads(result.stdout)
|
||||
self.duration = float(info["format"]["duration"])
|
||||
|
||||
# Audio buffer and state
|
||||
self._audio_data = None
|
||||
self._energy_history = []
|
||||
self._last_energy = 0
|
||||
self._last_beat_time = -1
|
||||
self._beat_threshold = 1.5
|
||||
self._min_beat_interval = 0.15
|
||||
|
||||
# Load audio lazily
|
||||
self._loaded = False
|
||||
|
||||
def _load_audio(self):
|
||||
"""Load audio data on first use."""
|
||||
if self._loaded:
|
||||
return
|
||||
|
||||
import subprocess
|
||||
|
||||
# Use ffmpeg to decode audio to raw PCM
|
||||
cmd = [
|
||||
"ffmpeg", "-v", "quiet",
|
||||
"-i", str(self.path),
|
||||
"-f", "f32le", # 32-bit float, little-endian
|
||||
"-ac", "1", # mono
|
||||
"-ar", str(self.sample_rate),
|
||||
"-"
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
self._audio_data = np.frombuffer(result.stdout, dtype=np.float32)
|
||||
self._loaded = True
|
||||
|
||||
def set_time(self, t: float):
|
||||
"""Set current playback time."""
|
||||
self._current_time = t
|
||||
|
||||
def get_energy(self) -> float:
|
||||
"""Compute energy at current time."""
|
||||
self._load_audio()
|
||||
|
||||
if self._audio_data is None or len(self._audio_data) == 0:
|
||||
return 0.0
|
||||
|
||||
# Get sample index for current time
|
||||
sample_idx = int(self._current_time * self.sample_rate)
|
||||
window_size = self.hop_length * 2
|
||||
|
||||
start = max(0, sample_idx - window_size // 2)
|
||||
end = min(len(self._audio_data), sample_idx + window_size // 2)
|
||||
|
||||
if start >= end:
|
||||
return 0.0
|
||||
|
||||
# RMS energy
|
||||
chunk = self._audio_data[start:end]
|
||||
rms = np.sqrt(np.mean(chunk ** 2))
|
||||
|
||||
# Normalize to 0-1 range (approximate)
|
||||
energy = min(1.0, rms * 3.0)
|
||||
|
||||
self._last_energy = energy
|
||||
return energy
|
||||
|
||||
def get_beat(self) -> bool:
|
||||
"""Detect beat using spectral flux (change in frequency content)."""
|
||||
self._load_audio()
|
||||
|
||||
if self._audio_data is None or len(self._audio_data) == 0:
|
||||
return False
|
||||
|
||||
# Get audio chunks for current and previous frame
|
||||
sample_idx = int(self._current_time * self.sample_rate)
|
||||
chunk_size = self.hop_length * 2
|
||||
|
||||
# Current chunk
|
||||
start = max(0, sample_idx - chunk_size // 2)
|
||||
end = min(len(self._audio_data), sample_idx + chunk_size // 2)
|
||||
if end - start < chunk_size // 2:
|
||||
return False
|
||||
current_chunk = self._audio_data[start:end]
|
||||
|
||||
# Previous chunk (one hop back)
|
||||
prev_start = max(0, start - self.hop_length)
|
||||
prev_end = max(0, end - self.hop_length)
|
||||
if prev_end <= prev_start:
|
||||
return False
|
||||
prev_chunk = self._audio_data[prev_start:prev_end]
|
||||
|
||||
# Compute spectra
|
||||
current_spec = np.abs(np.fft.rfft(current_chunk * np.hanning(len(current_chunk))))
|
||||
prev_spec = np.abs(np.fft.rfft(prev_chunk * np.hanning(len(prev_chunk))))
|
||||
|
||||
# Spectral flux: sum of positive differences (onset = new frequencies appearing)
|
||||
min_len = min(len(current_spec), len(prev_spec))
|
||||
diff = current_spec[:min_len] - prev_spec[:min_len]
|
||||
flux = np.sum(np.maximum(0, diff)) # Only count increases
|
||||
|
||||
# Normalize by spectrum size
|
||||
flux = flux / (min_len + 1)
|
||||
|
||||
# Update flux history
|
||||
self._energy_history.append((self._current_time, flux))
|
||||
while self._energy_history and self._energy_history[0][0] < self._current_time - 1.5:
|
||||
self._energy_history.pop(0)
|
||||
|
||||
if len(self._energy_history) < 3:
|
||||
return False
|
||||
|
||||
# Adaptive threshold based on recent flux values
|
||||
flux_values = [f for t, f in self._energy_history]
|
||||
mean_flux = np.mean(flux_values)
|
||||
std_flux = np.std(flux_values) + 0.001 # Avoid division by zero
|
||||
|
||||
# Beat if flux is above mean (more sensitive threshold)
|
||||
threshold = mean_flux + std_flux * 0.3 # Lower = more sensitive
|
||||
min_interval = 0.1 # Allow up to 600 BPM
|
||||
time_ok = self._current_time - self._last_beat_time > min_interval
|
||||
|
||||
is_beat = flux > threshold and time_ok
|
||||
|
||||
if is_beat:
|
||||
self._last_beat_time = self._current_time
|
||||
|
||||
return is_beat
|
||||
|
||||
def get_spectrum(self, bands: int = 3) -> np.ndarray:
|
||||
"""Get frequency spectrum at current time."""
|
||||
self._load_audio()
|
||||
|
||||
if self._audio_data is None or len(self._audio_data) == 0:
|
||||
return np.zeros(bands)
|
||||
|
||||
sample_idx = int(self._current_time * self.sample_rate)
|
||||
n = 2048
|
||||
|
||||
start = max(0, sample_idx - n // 2)
|
||||
end = min(len(self._audio_data), sample_idx + n // 2)
|
||||
|
||||
if end - start < n // 2:
|
||||
return np.zeros(bands)
|
||||
|
||||
chunk = self._audio_data[start:end]
|
||||
|
||||
# FFT
|
||||
fft = np.abs(np.fft.rfft(chunk * np.hanning(len(chunk))))
|
||||
|
||||
# Divide into bands
|
||||
band_size = len(fft) // bands
|
||||
result = np.zeros(bands)
|
||||
for i in range(bands):
|
||||
s, e = i * band_size, (i + 1) * band_size
|
||||
result[i] = np.mean(fft[s:e])
|
||||
|
||||
# Normalize
|
||||
max_val = np.max(result)
|
||||
if max_val > 0:
|
||||
result = result / max_val
|
||||
|
||||
return result
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return True
|
||||
308
streaming/backends.py
Normal file
308
streaming/backends.py
Normal file
@@ -0,0 +1,308 @@
|
||||
"""
|
||||
Effect processing backends.
|
||||
|
||||
Provides abstraction over different rendering backends:
|
||||
- numpy: CPU-based, works everywhere, ~3-5 fps
|
||||
- glsl: GPU-based, requires OpenGL, 30+ fps (future)
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import List, Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Backend(ABC):
|
||||
"""Abstract base class for effect processing backends."""
|
||||
|
||||
@abstractmethod
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Process multiple input frames through effects and composite.
|
||||
|
||||
Args:
|
||||
frames: List of input frames (one per source)
|
||||
effects_per_frame: List of effect chains (one per source)
|
||||
compositor_config: How to blend the layers
|
||||
t: Current time in seconds
|
||||
analysis_data: Analysis data for binding resolution
|
||||
|
||||
Returns:
|
||||
Composited output frame
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
"""Load an effect definition."""
|
||||
pass
|
||||
|
||||
|
||||
class NumpyBackend(Backend):
|
||||
"""
|
||||
CPU-based effect processing using NumPy.
|
||||
|
||||
Uses existing sexp_effects interpreter for effect execution.
|
||||
Works on any system, but limited to ~3-5 fps for complex effects.
|
||||
"""
|
||||
|
||||
def __init__(self, recipe_dir: Path = None, minimal_primitives: bool = True):
|
||||
self.recipe_dir = recipe_dir or Path(".")
|
||||
self.minimal_primitives = minimal_primitives
|
||||
self._interpreter = None
|
||||
self._loaded_effects = {}
|
||||
|
||||
def _get_interpreter(self):
|
||||
"""Lazy-load the sexp interpreter."""
|
||||
if self._interpreter is None:
|
||||
from sexp_effects import get_interpreter
|
||||
self._interpreter = get_interpreter(minimal_primitives=self.minimal_primitives)
|
||||
return self._interpreter
|
||||
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
"""Load an effect from sexp file."""
|
||||
effect_key = str(effect_path)
|
||||
if effect_key not in self._loaded_effects:
|
||||
interp = self._get_interpreter()
|
||||
interp.load_effect(str(effect_path))
|
||||
self._loaded_effects[effect_key] = effect_path.stem
|
||||
return self._loaded_effects[effect_key]
|
||||
|
||||
def _resolve_binding(self, value: Any, t: float, analysis_data: Dict) -> Any:
|
||||
"""Resolve a parameter binding to its value at time t."""
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
|
||||
if "_binding" in value or "_bind" in value:
|
||||
source = value.get("source") or value.get("_bind")
|
||||
feature = value.get("feature", "values")
|
||||
range_map = value.get("range")
|
||||
|
||||
track = analysis_data.get(source, {})
|
||||
times = track.get("times", [])
|
||||
values = track.get("values", [])
|
||||
|
||||
if not times or not values:
|
||||
return 0.0
|
||||
|
||||
# Find value at time t (linear interpolation)
|
||||
if t <= times[0]:
|
||||
val = values[0]
|
||||
elif t >= times[-1]:
|
||||
val = values[-1]
|
||||
else:
|
||||
# Binary search for bracket
|
||||
for i in range(len(times) - 1):
|
||||
if times[i] <= t <= times[i + 1]:
|
||||
alpha = (t - times[i]) / (times[i + 1] - times[i])
|
||||
val = values[i] * (1 - alpha) + values[i + 1] * alpha
|
||||
break
|
||||
else:
|
||||
val = values[-1]
|
||||
|
||||
# Apply range mapping
|
||||
if range_map and len(range_map) == 2:
|
||||
val = range_map[0] + val * (range_map[1] - range_map[0])
|
||||
|
||||
return val
|
||||
|
||||
return value
|
||||
|
||||
def _apply_effect(
|
||||
self,
|
||||
frame: np.ndarray,
|
||||
effect_name: str,
|
||||
params: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""Apply a single effect to a frame."""
|
||||
# Resolve bindings in params
|
||||
resolved_params = {"_time": t}
|
||||
for key, value in params.items():
|
||||
if key in ("effect", "effect_path", "cid", "analysis_refs"):
|
||||
continue
|
||||
resolved_params[key] = self._resolve_binding(value, t, analysis_data)
|
||||
|
||||
# Try fast native effects first
|
||||
result = self._apply_native_effect(frame, effect_name, resolved_params)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Fall back to sexp interpreter for complex effects
|
||||
interp = self._get_interpreter()
|
||||
if effect_name in interp.effects:
|
||||
result, _ = interp.run_effect(effect_name, frame, resolved_params, {})
|
||||
return result
|
||||
|
||||
# Unknown effect - pass through
|
||||
return frame
|
||||
|
||||
def _apply_native_effect(
|
||||
self,
|
||||
frame: np.ndarray,
|
||||
effect_name: str,
|
||||
params: Dict,
|
||||
) -> Optional[np.ndarray]:
|
||||
"""Fast native numpy effects for real-time streaming."""
|
||||
import cv2
|
||||
|
||||
if effect_name == "zoom":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
if abs(amount - 1.0) < 0.01:
|
||||
return frame
|
||||
h, w = frame.shape[:2]
|
||||
# Crop center and resize
|
||||
new_w, new_h = int(w / amount), int(h / amount)
|
||||
x1, y1 = (w - new_w) // 2, (h - new_h) // 2
|
||||
cropped = frame[y1:y1+new_h, x1:x1+new_w]
|
||||
return cv2.resize(cropped, (w, h))
|
||||
|
||||
elif effect_name == "rotate":
|
||||
angle = float(params.get("angle", 0))
|
||||
if abs(angle) < 0.5:
|
||||
return frame
|
||||
h, w = frame.shape[:2]
|
||||
center = (w // 2, h // 2)
|
||||
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
||||
return cv2.warpAffine(frame, matrix, (w, h))
|
||||
|
||||
elif effect_name == "brightness":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
return np.clip(frame * amount, 0, 255).astype(np.uint8)
|
||||
|
||||
elif effect_name == "invert":
|
||||
amount = float(params.get("amount", 1.0))
|
||||
if amount < 0.5:
|
||||
return frame
|
||||
return 255 - frame
|
||||
|
||||
# Not a native effect
|
||||
return None
|
||||
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Process frames through effects and composite.
|
||||
"""
|
||||
if not frames:
|
||||
return np.zeros((720, 1280, 3), dtype=np.uint8)
|
||||
|
||||
processed = []
|
||||
|
||||
# Apply effects to each input frame
|
||||
for i, (frame, effects) in enumerate(zip(frames, effects_per_frame)):
|
||||
result = frame.copy()
|
||||
for effect_config in effects:
|
||||
effect_name = effect_config.get("effect", "")
|
||||
if effect_name:
|
||||
result = self._apply_effect(
|
||||
result, effect_name, effect_config, t, analysis_data
|
||||
)
|
||||
processed.append(result)
|
||||
|
||||
# Composite layers
|
||||
if len(processed) == 1:
|
||||
return processed[0]
|
||||
|
||||
return self._composite(processed, compositor_config, t, analysis_data)
|
||||
|
||||
def _composite(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
"""Composite multiple frames into one."""
|
||||
mode = config.get("mode", "alpha")
|
||||
weights = config.get("weights", [1.0 / len(frames)] * len(frames))
|
||||
|
||||
# Resolve weight bindings
|
||||
resolved_weights = []
|
||||
for w in weights:
|
||||
resolved_weights.append(self._resolve_binding(w, t, analysis_data))
|
||||
|
||||
# Normalize weights
|
||||
total = sum(resolved_weights)
|
||||
if total > 0:
|
||||
resolved_weights = [w / total for w in resolved_weights]
|
||||
else:
|
||||
resolved_weights = [1.0 / len(frames)] * len(frames)
|
||||
|
||||
# Resize frames to match first frame
|
||||
target_h, target_w = frames[0].shape[:2]
|
||||
resized = []
|
||||
for frame in frames:
|
||||
if frame.shape[:2] != (target_h, target_w):
|
||||
import cv2
|
||||
frame = cv2.resize(frame, (target_w, target_h))
|
||||
resized.append(frame.astype(np.float32))
|
||||
|
||||
# Weighted blend
|
||||
result = np.zeros_like(resized[0])
|
||||
for frame, weight in zip(resized, resolved_weights):
|
||||
result += frame * weight
|
||||
|
||||
return np.clip(result, 0, 255).astype(np.uint8)
|
||||
|
||||
|
||||
class GLSLBackend(Backend):
|
||||
"""
|
||||
GPU-based effect processing using OpenGL/GLSL.
|
||||
|
||||
Requires GPU with OpenGL 3.3+ support (or Mesa software renderer).
|
||||
Achieves 30+ fps real-time processing.
|
||||
|
||||
TODO: Implement when ready for GPU acceleration.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
raise NotImplementedError(
|
||||
"GLSL backend not yet implemented. Use NumpyBackend for now."
|
||||
)
|
||||
|
||||
def load_effect(self, effect_path: Path) -> Any:
|
||||
pass
|
||||
|
||||
def process_frame(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
effects_per_frame: List[List[Dict]],
|
||||
compositor_config: Dict,
|
||||
t: float,
|
||||
analysis_data: Dict,
|
||||
) -> np.ndarray:
|
||||
pass
|
||||
|
||||
|
||||
def get_backend(name: str = "numpy", **kwargs) -> Backend:
|
||||
"""
|
||||
Get a backend by name.
|
||||
|
||||
Args:
|
||||
name: "numpy" or "glsl"
|
||||
**kwargs: Backend-specific options
|
||||
|
||||
Returns:
|
||||
Backend instance
|
||||
"""
|
||||
if name == "numpy":
|
||||
return NumpyBackend(**kwargs)
|
||||
elif name == "glsl":
|
||||
return GLSLBackend(**kwargs)
|
||||
else:
|
||||
raise ValueError(f"Unknown backend: {name}")
|
||||
595
streaming/compositor.py
Normal file
595
streaming/compositor.py
Normal file
@@ -0,0 +1,595 @@
|
||||
"""
|
||||
Streaming video compositor.
|
||||
|
||||
Main entry point for the streaming pipeline. Combines:
|
||||
- Multiple video sources (with looping)
|
||||
- Per-source effect chains
|
||||
- Layer compositing
|
||||
- Optional live audio analysis
|
||||
- Output to display/file/stream
|
||||
"""
|
||||
|
||||
import time
|
||||
import sys
|
||||
import numpy as np
|
||||
from typing import List, Dict, Any, Optional, Union
|
||||
from pathlib import Path
|
||||
|
||||
from .sources import Source, VideoSource
|
||||
from .backends import Backend, NumpyBackend, get_backend
|
||||
from .output import Output, DisplayOutput, FileOutput, MultiOutput
|
||||
|
||||
|
||||
class StreamingCompositor:
|
||||
"""
|
||||
Real-time streaming video compositor.
|
||||
|
||||
Reads frames from multiple sources, applies effects, composites layers,
|
||||
and outputs the result - all frame-by-frame without intermediate files.
|
||||
|
||||
Example:
|
||||
compositor = StreamingCompositor(
|
||||
sources=["video1.mp4", "video2.mp4"],
|
||||
effects_per_source=[
|
||||
[{"effect": "rotate", "angle": 45}],
|
||||
[{"effect": "zoom", "amount": 1.5}],
|
||||
],
|
||||
compositor_config={"mode": "alpha", "weights": [0.5, 0.5]},
|
||||
)
|
||||
compositor.run(output="preview", duration=60)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sources: List[Union[str, Source]],
|
||||
effects_per_source: List[List[Dict]] = None,
|
||||
compositor_config: Dict = None,
|
||||
analysis_data: Dict = None,
|
||||
backend: str = "numpy",
|
||||
recipe_dir: Path = None,
|
||||
fps: float = 30,
|
||||
audio_source: str = None,
|
||||
):
|
||||
"""
|
||||
Initialize the streaming compositor.
|
||||
|
||||
Args:
|
||||
sources: List of video paths or Source objects
|
||||
effects_per_source: List of effect chains, one per source
|
||||
compositor_config: How to blend layers (mode, weights)
|
||||
analysis_data: Pre-computed analysis data for bindings
|
||||
backend: "numpy" or "glsl"
|
||||
recipe_dir: Directory for resolving relative effect paths
|
||||
fps: Output frame rate
|
||||
audio_source: Path to audio file for streaming analysis
|
||||
"""
|
||||
self.fps = fps
|
||||
self.recipe_dir = recipe_dir or Path(".")
|
||||
self.analysis_data = analysis_data or {}
|
||||
|
||||
# Initialize streaming audio analyzer if audio source provided
|
||||
self._audio_analyzer = None
|
||||
self._audio_source = audio_source
|
||||
if audio_source:
|
||||
from .audio import StreamingAudioAnalyzer
|
||||
self._audio_analyzer = StreamingAudioAnalyzer(audio_source)
|
||||
print(f"Streaming audio: {audio_source}", file=sys.stderr)
|
||||
|
||||
# Initialize sources
|
||||
self.sources: List[Source] = []
|
||||
for src in sources:
|
||||
if isinstance(src, Source):
|
||||
self.sources.append(src)
|
||||
elif isinstance(src, (str, Path)):
|
||||
self.sources.append(VideoSource(str(src), target_fps=fps))
|
||||
else:
|
||||
raise ValueError(f"Unknown source type: {type(src)}")
|
||||
|
||||
# Effect chains (default: no effects)
|
||||
self.effects_per_source = effects_per_source or [[] for _ in self.sources]
|
||||
if len(self.effects_per_source) != len(self.sources):
|
||||
raise ValueError(
|
||||
f"effects_per_source length ({len(self.effects_per_source)}) "
|
||||
f"must match sources length ({len(self.sources)})"
|
||||
)
|
||||
|
||||
# Compositor config (default: equal blend)
|
||||
self.compositor_config = compositor_config or {
|
||||
"mode": "alpha",
|
||||
"weights": [1.0 / len(self.sources)] * len(self.sources),
|
||||
}
|
||||
|
||||
# Initialize backend
|
||||
self.backend: Backend = get_backend(
|
||||
backend,
|
||||
recipe_dir=self.recipe_dir,
|
||||
)
|
||||
|
||||
# Load effects
|
||||
self._load_effects()
|
||||
|
||||
def _load_effects(self):
|
||||
"""Pre-load all effect definitions."""
|
||||
for effects in self.effects_per_source:
|
||||
for effect_config in effects:
|
||||
effect_path = effect_config.get("effect_path")
|
||||
if effect_path:
|
||||
full_path = self.recipe_dir / effect_path
|
||||
if full_path.exists():
|
||||
self.backend.load_effect(full_path)
|
||||
|
||||
def _create_output(
|
||||
self,
|
||||
output: Union[str, Output],
|
||||
size: tuple,
|
||||
) -> Output:
|
||||
"""Create output target from string or Output object."""
|
||||
if isinstance(output, Output):
|
||||
return output
|
||||
|
||||
if output == "preview":
|
||||
return DisplayOutput("Streaming Preview", size,
|
||||
audio_source=self._audio_source, fps=self.fps)
|
||||
elif output == "null":
|
||||
from .output import NullOutput
|
||||
return NullOutput()
|
||||
elif isinstance(output, str):
|
||||
return FileOutput(output, size, fps=self.fps, audio_source=self._audio_source)
|
||||
else:
|
||||
raise ValueError(f"Unknown output type: {output}")
|
||||
|
||||
def run(
|
||||
self,
|
||||
output: Union[str, Output] = "preview",
|
||||
duration: float = None,
|
||||
audio_analyzer=None,
|
||||
show_fps: bool = True,
|
||||
recipe_executor=None,
|
||||
):
|
||||
"""
|
||||
Run the streaming compositor.
|
||||
|
||||
Args:
|
||||
output: Output target - "preview", filename, or Output object
|
||||
duration: Duration in seconds (None = run until quit)
|
||||
audio_analyzer: Optional AudioAnalyzer for live audio reactivity
|
||||
show_fps: Show FPS counter in console
|
||||
recipe_executor: Optional StreamingRecipeExecutor for full recipe logic
|
||||
"""
|
||||
# Determine output size from first source
|
||||
output_size = self.sources[0].size
|
||||
|
||||
# Create output
|
||||
out = self._create_output(output, output_size)
|
||||
|
||||
# Determine duration
|
||||
if duration is None:
|
||||
# Run until stopped (or min source duration if not looping)
|
||||
duration = min(s.duration for s in self.sources)
|
||||
if duration == float('inf'):
|
||||
duration = 3600 # 1 hour max for live sources
|
||||
|
||||
total_frames = int(duration * self.fps)
|
||||
frame_time = 1.0 / self.fps
|
||||
|
||||
print(f"Streaming: {len(self.sources)} sources -> {output}", file=sys.stderr)
|
||||
print(f"Duration: {duration:.1f}s, {total_frames} frames @ {self.fps}fps", file=sys.stderr)
|
||||
print(f"Output size: {output_size[0]}x{output_size[1]}", file=sys.stderr)
|
||||
print(f"Press 'q' to quit (if preview)", file=sys.stderr)
|
||||
|
||||
# Frame loop
|
||||
start_time = time.time()
|
||||
frame_count = 0
|
||||
fps_update_interval = 30 # Update FPS display every N frames
|
||||
last_fps_time = start_time
|
||||
last_fps_count = 0
|
||||
|
||||
try:
|
||||
for frame_num in range(total_frames):
|
||||
if not out.is_open:
|
||||
print(f"\nOutput closed at frame {frame_num}", file=sys.stderr)
|
||||
break
|
||||
|
||||
t = frame_num * frame_time
|
||||
|
||||
try:
|
||||
# Update analysis data from streaming audio (file-based)
|
||||
energy = 0.0
|
||||
is_beat = False
|
||||
if self._audio_analyzer:
|
||||
self._update_from_audio(self._audio_analyzer, t)
|
||||
energy = self.analysis_data.get("live_energy", {}).get("values", [0])[0]
|
||||
is_beat = self.analysis_data.get("live_beat", {}).get("values", [0])[0] > 0.5
|
||||
elif audio_analyzer:
|
||||
self._update_from_audio(audio_analyzer, t)
|
||||
energy = self.analysis_data.get("live_energy", {}).get("values", [0])[0]
|
||||
is_beat = self.analysis_data.get("live_beat", {}).get("values", [0])[0] > 0.5
|
||||
|
||||
# Read frames from all sources
|
||||
frames = [src.read_frame(t) for src in self.sources]
|
||||
|
||||
# Process through recipe executor if provided
|
||||
if recipe_executor:
|
||||
result = self._process_with_executor(
|
||||
frames, recipe_executor, energy, is_beat, t
|
||||
)
|
||||
else:
|
||||
# Simple backend processing
|
||||
result = self.backend.process_frame(
|
||||
frames,
|
||||
self.effects_per_source,
|
||||
self.compositor_config,
|
||||
t,
|
||||
self.analysis_data,
|
||||
)
|
||||
|
||||
# Output
|
||||
out.write(result, t)
|
||||
frame_count += 1
|
||||
|
||||
# FPS display
|
||||
if show_fps and frame_count % fps_update_interval == 0:
|
||||
now = time.time()
|
||||
elapsed = now - last_fps_time
|
||||
if elapsed > 0:
|
||||
current_fps = (frame_count - last_fps_count) / elapsed
|
||||
progress = frame_num / total_frames * 100
|
||||
print(
|
||||
f"\r {progress:5.1f}% | {current_fps:5.1f} fps | "
|
||||
f"frame {frame_num}/{total_frames}",
|
||||
end="", file=sys.stderr
|
||||
)
|
||||
last_fps_time = now
|
||||
last_fps_count = frame_count
|
||||
|
||||
except Exception as e:
|
||||
print(f"\nError at frame {frame_num}, t={t:.1f}s: {e}", file=sys.stderr)
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
break
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted", file=sys.stderr)
|
||||
finally:
|
||||
out.close()
|
||||
for src in self.sources:
|
||||
if hasattr(src, 'close'):
|
||||
src.close()
|
||||
|
||||
# Final stats
|
||||
elapsed = time.time() - start_time
|
||||
avg_fps = frame_count / elapsed if elapsed > 0 else 0
|
||||
print(f"\nCompleted: {frame_count} frames in {elapsed:.1f}s ({avg_fps:.1f} fps avg)", file=sys.stderr)
|
||||
|
||||
def _process_with_executor(
|
||||
self,
|
||||
frames: List[np.ndarray],
|
||||
executor,
|
||||
energy: float,
|
||||
is_beat: bool,
|
||||
t: float,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Process frames using the recipe executor for full pipeline.
|
||||
|
||||
Implements:
|
||||
1. process-pair: two clips per source with effects, blended
|
||||
2. cycle-crossfade: dynamic composition with zoom and weights
|
||||
3. Final effects: whole-spin, ripple
|
||||
"""
|
||||
import cv2
|
||||
|
||||
# Target size from first source
|
||||
target_h, target_w = frames[0].shape[:2]
|
||||
|
||||
# Resize all frames to target size (letterbox to preserve aspect ratio)
|
||||
resized_frames = []
|
||||
for frame in frames:
|
||||
fh, fw = frame.shape[:2]
|
||||
if (fh, fw) != (target_h, target_w):
|
||||
# Calculate scale to fit while preserving aspect ratio
|
||||
scale = min(target_w / fw, target_h / fh)
|
||||
new_w, new_h = int(fw * scale), int(fh * scale)
|
||||
resized = cv2.resize(frame, (new_w, new_h))
|
||||
# Center on black canvas
|
||||
canvas = np.zeros((target_h, target_w, 3), dtype=np.uint8)
|
||||
x_off = (target_w - new_w) // 2
|
||||
y_off = (target_h - new_h) // 2
|
||||
canvas[y_off:y_off+new_h, x_off:x_off+new_w] = resized
|
||||
resized_frames.append(canvas)
|
||||
else:
|
||||
resized_frames.append(frame)
|
||||
frames = resized_frames
|
||||
|
||||
# Update executor state
|
||||
executor.on_frame(energy, is_beat, t)
|
||||
|
||||
# Get weights to know which sources are active
|
||||
weights = executor.get_cycle_weights()
|
||||
|
||||
# Process each source as a "pair" (clip A and B with different effects)
|
||||
processed_pairs = []
|
||||
|
||||
for i, frame in enumerate(frames):
|
||||
# Skip sources with zero weight (but still need placeholder)
|
||||
if i < len(weights) and weights[i] < 0.001:
|
||||
processed_pairs.append(None)
|
||||
continue
|
||||
# Get effect params for clip A and B
|
||||
params_a = executor.get_effect_params(i, "a", energy)
|
||||
params_b = executor.get_effect_params(i, "b", energy)
|
||||
pair_params = executor.get_pair_params(i)
|
||||
|
||||
# Process clip A
|
||||
clip_a = self._apply_clip_effects(frame.copy(), params_a, t)
|
||||
|
||||
# Process clip B
|
||||
clip_b = self._apply_clip_effects(frame.copy(), params_b, t)
|
||||
|
||||
# Blend A and B using pair_mix opacity
|
||||
opacity = pair_params["blend_opacity"]
|
||||
blended = cv2.addWeighted(
|
||||
clip_a, 1 - opacity,
|
||||
clip_b, opacity,
|
||||
0
|
||||
)
|
||||
|
||||
# Apply pair rotation
|
||||
h, w = blended.shape[:2]
|
||||
center = (w // 2, h // 2)
|
||||
angle = pair_params["pair_rotation"]
|
||||
if abs(angle) > 0.5:
|
||||
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
||||
blended = cv2.warpAffine(blended, matrix, (w, h))
|
||||
|
||||
processed_pairs.append(blended)
|
||||
|
||||
# Cycle-crossfade composition
|
||||
weights = executor.get_cycle_weights()
|
||||
zooms = executor.get_cycle_zooms()
|
||||
|
||||
# Apply zoom per pair and composite
|
||||
h, w = target_h, target_w
|
||||
result = np.zeros((h, w, 3), dtype=np.float32)
|
||||
|
||||
for idx, (pair, weight, zoom) in enumerate(zip(processed_pairs, weights, zooms)):
|
||||
# Skip zero-weight sources
|
||||
if pair is None or weight < 0.001:
|
||||
continue
|
||||
|
||||
orig_shape = pair.shape
|
||||
|
||||
# Apply zoom
|
||||
if zoom > 1.01:
|
||||
# Zoom in: crop center and resize up
|
||||
new_w, new_h = int(w / zoom), int(h / zoom)
|
||||
if new_w > 0 and new_h > 0:
|
||||
x1, y1 = (w - new_w) // 2, (h - new_h) // 2
|
||||
cropped = pair[y1:y1+new_h, x1:x1+new_w]
|
||||
pair = cv2.resize(cropped, (w, h))
|
||||
elif zoom < 0.99:
|
||||
# Zoom out: shrink video and center on black
|
||||
scaled_w, scaled_h = int(w * zoom), int(h * zoom)
|
||||
if scaled_w > 0 and scaled_h > 0:
|
||||
shrunk = cv2.resize(pair, (scaled_w, scaled_h))
|
||||
canvas = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
x_off, y_off = (w - scaled_w) // 2, (h - scaled_h) // 2
|
||||
canvas[y_off:y_off+scaled_h, x_off:x_off+scaled_w] = shrunk
|
||||
pair = canvas.copy()
|
||||
|
||||
# Draw colored border - size indicates zoom level
|
||||
border_colors = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0)]
|
||||
color = border_colors[idx % 4]
|
||||
thickness = max(3, int(10 * weight)) # Thicker border = higher weight
|
||||
pair = np.ascontiguousarray(pair)
|
||||
pair[:thickness, :] = color
|
||||
pair[-thickness:, :] = color
|
||||
pair[:, :thickness] = color
|
||||
pair[:, -thickness:] = color
|
||||
|
||||
result += pair.astype(np.float32) * weight
|
||||
|
||||
result = np.clip(result, 0, 255).astype(np.uint8)
|
||||
|
||||
# Apply final effects (whole-spin, ripple)
|
||||
final_params = executor.get_final_effects(energy)
|
||||
|
||||
# Whole spin
|
||||
spin_angle = final_params["whole_spin_angle"]
|
||||
if abs(spin_angle) > 0.5:
|
||||
center = (w // 2, h // 2)
|
||||
matrix = cv2.getRotationMatrix2D(center, spin_angle, 1.0)
|
||||
result = cv2.warpAffine(result, matrix, (w, h))
|
||||
|
||||
# Ripple effect
|
||||
amp = final_params["ripple_amplitude"]
|
||||
if amp > 1:
|
||||
result = self._apply_ripple(result, amp,
|
||||
final_params["ripple_cx"],
|
||||
final_params["ripple_cy"],
|
||||
t)
|
||||
|
||||
return result
|
||||
|
||||
def _apply_clip_effects(self, frame: np.ndarray, params: dict, t: float) -> np.ndarray:
|
||||
"""Apply per-clip effects: rotate, zoom, invert, hue_shift, ascii."""
|
||||
import cv2
|
||||
|
||||
h, w = frame.shape[:2]
|
||||
|
||||
# Rotate
|
||||
angle = params["rotate_angle"]
|
||||
if abs(angle) > 0.5:
|
||||
center = (w // 2, h // 2)
|
||||
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
||||
frame = cv2.warpAffine(frame, matrix, (w, h))
|
||||
|
||||
# Zoom
|
||||
zoom = params["zoom_amount"]
|
||||
if abs(zoom - 1.0) > 0.01:
|
||||
new_w, new_h = int(w / zoom), int(h / zoom)
|
||||
if new_w > 0 and new_h > 0:
|
||||
x1, y1 = (w - new_w) // 2, (h - new_h) // 2
|
||||
x1, y1 = max(0, x1), max(0, y1)
|
||||
x2, y2 = min(w, x1 + new_w), min(h, y1 + new_h)
|
||||
if x2 > x1 and y2 > y1:
|
||||
cropped = frame[y1:y2, x1:x2]
|
||||
frame = cv2.resize(cropped, (w, h))
|
||||
|
||||
# Invert
|
||||
if params["invert_amount"] > 0.5:
|
||||
frame = 255 - frame
|
||||
|
||||
# Hue shift
|
||||
hue_deg = params["hue_degrees"]
|
||||
if abs(hue_deg) > 1:
|
||||
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
|
||||
hsv[:, :, 0] = (hsv[:, :, 0].astype(np.int32) + int(hue_deg / 2)) % 180
|
||||
frame = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
|
||||
|
||||
# ASCII art
|
||||
if params["ascii_mix"] > 0.5:
|
||||
char_size = max(4, int(params["ascii_char_size"]))
|
||||
frame = self._apply_ascii(frame, char_size)
|
||||
|
||||
return frame
|
||||
|
||||
def _apply_ascii(self, frame: np.ndarray, char_size: int) -> np.ndarray:
|
||||
"""Apply ASCII art effect."""
|
||||
import cv2
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
h, w = frame.shape[:2]
|
||||
chars = " .:-=+*#%@"
|
||||
|
||||
# Get font
|
||||
try:
|
||||
font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", char_size)
|
||||
except:
|
||||
font = ImageFont.load_default()
|
||||
|
||||
# Sample cells using area interpolation (fast block average)
|
||||
rows = h // char_size
|
||||
cols = w // char_size
|
||||
if rows < 1 or cols < 1:
|
||||
return frame
|
||||
|
||||
# Crop to exact grid and downsample
|
||||
cropped = frame[:rows * char_size, :cols * char_size]
|
||||
cell_colors = cv2.resize(cropped, (cols, rows), interpolation=cv2.INTER_AREA)
|
||||
|
||||
# Compute luminance
|
||||
luminances = (0.299 * cell_colors[:, :, 0] +
|
||||
0.587 * cell_colors[:, :, 1] +
|
||||
0.114 * cell_colors[:, :, 2]) / 255.0
|
||||
|
||||
# Create output image
|
||||
out_h = rows * char_size
|
||||
out_w = cols * char_size
|
||||
output = Image.new('RGB', (out_w, out_h), (0, 0, 0))
|
||||
draw = ImageDraw.Draw(output)
|
||||
|
||||
# Draw characters
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
lum = luminances[r, c]
|
||||
color = tuple(cell_colors[r, c])
|
||||
|
||||
# Map luminance to character
|
||||
idx = int(lum * (len(chars) - 1))
|
||||
char = chars[idx]
|
||||
|
||||
# Draw character
|
||||
x = c * char_size
|
||||
y = r * char_size
|
||||
draw.text((x, y), char, fill=color, font=font)
|
||||
|
||||
# Convert back to numpy and resize to original
|
||||
result = np.array(output)
|
||||
if result.shape[:2] != (h, w):
|
||||
result = cv2.resize(result, (w, h), interpolation=cv2.INTER_LINEAR)
|
||||
|
||||
return result
|
||||
|
||||
def _apply_ripple(self, frame: np.ndarray, amplitude: float,
|
||||
cx: float, cy: float, t: float = 0) -> np.ndarray:
|
||||
"""Apply ripple distortion effect."""
|
||||
import cv2
|
||||
|
||||
h, w = frame.shape[:2]
|
||||
center_x, center_y = cx * w, cy * h
|
||||
max_dim = max(w, h)
|
||||
|
||||
# Create coordinate grids
|
||||
y_coords, x_coords = np.mgrid[0:h, 0:w].astype(np.float32)
|
||||
|
||||
# Distance from center
|
||||
dx = x_coords - center_x
|
||||
dy = y_coords - center_y
|
||||
dist = np.sqrt(dx*dx + dy*dy)
|
||||
|
||||
# Ripple parameters (matching recipe: frequency=8, decay=2, speed=5)
|
||||
freq = 8
|
||||
decay = 2
|
||||
speed = 5
|
||||
phase = t * speed * 2 * np.pi
|
||||
|
||||
# Ripple displacement (matching original formula)
|
||||
ripple = np.sin(2 * np.pi * freq * dist / max_dim + phase) * amplitude
|
||||
|
||||
# Apply decay
|
||||
if decay > 0:
|
||||
ripple = ripple * np.exp(-dist * decay / max_dim)
|
||||
|
||||
# Displace along radial direction
|
||||
with np.errstate(divide='ignore', invalid='ignore'):
|
||||
norm_dx = np.where(dist > 0, dx / dist, 0)
|
||||
norm_dy = np.where(dist > 0, dy / dist, 0)
|
||||
|
||||
map_x = (x_coords + ripple * norm_dx).astype(np.float32)
|
||||
map_y = (y_coords + ripple * norm_dy).astype(np.float32)
|
||||
|
||||
return cv2.remap(frame, map_x, map_y, cv2.INTER_LINEAR,
|
||||
borderMode=cv2.BORDER_REFLECT)
|
||||
|
||||
def _update_from_audio(self, analyzer, t: float):
|
||||
"""Update analysis data from audio analyzer (streaming or live)."""
|
||||
# Set time for file-based streaming analyzers
|
||||
if hasattr(analyzer, 'set_time'):
|
||||
analyzer.set_time(t)
|
||||
|
||||
# Get current audio features
|
||||
energy = analyzer.get_energy() if hasattr(analyzer, 'get_energy') else 0
|
||||
beat = analyzer.get_beat() if hasattr(analyzer, 'get_beat') else False
|
||||
|
||||
# Update analysis tracks - these can be referenced by effect bindings
|
||||
self.analysis_data["live_energy"] = {
|
||||
"times": [t],
|
||||
"values": [energy],
|
||||
"duration": float('inf'),
|
||||
}
|
||||
self.analysis_data["live_beat"] = {
|
||||
"times": [t],
|
||||
"values": [1.0 if beat else 0.0],
|
||||
"duration": float('inf'),
|
||||
}
|
||||
|
||||
|
||||
def quick_preview(
|
||||
sources: List[str],
|
||||
effects: List[List[Dict]] = None,
|
||||
duration: float = 10,
|
||||
fps: float = 30,
|
||||
):
|
||||
"""
|
||||
Quick preview helper - show sources with optional effects.
|
||||
|
||||
Example:
|
||||
quick_preview(["video1.mp4", "video2.mp4"], duration=30)
|
||||
"""
|
||||
compositor = StreamingCompositor(
|
||||
sources=sources,
|
||||
effects_per_source=effects,
|
||||
fps=fps,
|
||||
)
|
||||
compositor.run(output="preview", duration=duration)
|
||||
125
streaming/demo.py
Normal file
125
streaming/demo.py
Normal file
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Demo script for streaming compositor.
|
||||
|
||||
Usage:
|
||||
# Preview two videos blended
|
||||
python -m streaming.demo preview video1.mp4 video2.mp4
|
||||
|
||||
# Record output to file
|
||||
python -m streaming.demo record video1.mp4 video2.mp4 -o output.mp4
|
||||
|
||||
# Benchmark (no output)
|
||||
python -m streaming.demo benchmark video1.mp4 --duration 10
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from streaming import StreamingCompositor, VideoSource
|
||||
from streaming.output import NullOutput
|
||||
|
||||
|
||||
def demo_preview(sources: list, duration: float, effects: bool = False):
|
||||
"""Preview sources with optional simple effects."""
|
||||
effects_config = None
|
||||
if effects:
|
||||
effects_config = [
|
||||
[{"effect": "rotate", "angle": 15}],
|
||||
[{"effect": "zoom", "amount": 1.2}],
|
||||
][:len(sources)]
|
||||
|
||||
compositor = StreamingCompositor(
|
||||
sources=sources,
|
||||
effects_per_source=effects_config,
|
||||
recipe_dir=Path(__file__).parent.parent,
|
||||
)
|
||||
compositor.run(output="preview", duration=duration)
|
||||
|
||||
|
||||
def demo_record(sources: list, output_path: str, duration: float):
|
||||
"""Record blended output to file."""
|
||||
compositor = StreamingCompositor(
|
||||
sources=sources,
|
||||
recipe_dir=Path(__file__).parent.parent,
|
||||
)
|
||||
compositor.run(output=output_path, duration=duration)
|
||||
|
||||
|
||||
def demo_benchmark(sources: list, duration: float):
|
||||
"""Benchmark processing speed (no output)."""
|
||||
compositor = StreamingCompositor(
|
||||
sources=sources,
|
||||
recipe_dir=Path(__file__).parent.parent,
|
||||
)
|
||||
compositor.run(output="null", duration=duration)
|
||||
|
||||
|
||||
def demo_audio_reactive(sources: list, duration: float):
|
||||
"""Preview with live audio reactivity."""
|
||||
from streaming.audio import AudioAnalyzer
|
||||
|
||||
# Create compositor with energy-reactive effects
|
||||
effects_config = [
|
||||
[{
|
||||
"effect": "zoom",
|
||||
"amount": {"_binding": True, "source": "live_energy", "feature": "values", "range": [1.0, 1.5]},
|
||||
}]
|
||||
for _ in sources
|
||||
]
|
||||
|
||||
compositor = StreamingCompositor(
|
||||
sources=sources,
|
||||
effects_per_source=effects_config,
|
||||
recipe_dir=Path(__file__).parent.parent,
|
||||
)
|
||||
|
||||
# Start audio analyzer
|
||||
try:
|
||||
with AudioAnalyzer() as audio:
|
||||
print("Audio analyzer started. Make some noise!", file=sys.stderr)
|
||||
compositor.run(output="preview", duration=duration, audio_analyzer=audio)
|
||||
except Exception as e:
|
||||
print(f"Audio not available: {e}", file=sys.stderr)
|
||||
print("Running without audio...", file=sys.stderr)
|
||||
compositor.run(output="preview", duration=duration)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Streaming compositor demo")
|
||||
parser.add_argument("mode", choices=["preview", "record", "benchmark", "audio"],
|
||||
help="Demo mode")
|
||||
parser.add_argument("sources", nargs="+", help="Video source files")
|
||||
parser.add_argument("-o", "--output", help="Output file (for record mode)")
|
||||
parser.add_argument("-d", "--duration", type=float, default=30,
|
||||
help="Duration in seconds")
|
||||
parser.add_argument("--effects", action="store_true",
|
||||
help="Apply simple effects (for preview)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Verify sources exist
|
||||
for src in args.sources:
|
||||
if not Path(src).exists():
|
||||
print(f"Error: Source not found: {src}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.mode == "preview":
|
||||
demo_preview(args.sources, args.duration, args.effects)
|
||||
elif args.mode == "record":
|
||||
if not args.output:
|
||||
print("Error: --output required for record mode", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
demo_record(args.sources, args.output, args.duration)
|
||||
elif args.mode == "benchmark":
|
||||
demo_benchmark(args.sources, args.duration)
|
||||
elif args.mode == "audio":
|
||||
demo_audio_reactive(args.sources, args.duration)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
369
streaming/output.py
Normal file
369
streaming/output.py
Normal file
@@ -0,0 +1,369 @@
|
||||
"""
|
||||
Output targets for streaming compositor.
|
||||
|
||||
Supports:
|
||||
- Display window (preview)
|
||||
- File output (recording)
|
||||
- Stream output (RTMP, etc.) - future
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import subprocess
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Tuple, Optional
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class Output(ABC):
|
||||
"""Abstract base class for output targets."""
|
||||
|
||||
@abstractmethod
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
"""Write a frame to the output."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def close(self):
|
||||
"""Close the output and clean up resources."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def is_open(self) -> bool:
|
||||
"""Check if output is still open/valid."""
|
||||
pass
|
||||
|
||||
|
||||
class DisplayOutput(Output):
|
||||
"""
|
||||
Display frames using mpv (handles Wayland properly).
|
||||
|
||||
Useful for live preview. Press 'q' to quit.
|
||||
"""
|
||||
|
||||
def __init__(self, title: str = "Streaming Preview", size: Tuple[int, int] = None,
|
||||
audio_source: str = None, fps: float = 30):
|
||||
self.title = title
|
||||
self.size = size
|
||||
self.audio_source = audio_source
|
||||
self.fps = fps
|
||||
self._is_open = True
|
||||
self._process = None
|
||||
self._audio_process = None
|
||||
|
||||
def _start_mpv(self, frame_size: Tuple[int, int]):
|
||||
"""Start mpv process for display."""
|
||||
import sys
|
||||
w, h = frame_size
|
||||
cmd = [
|
||||
"mpv",
|
||||
"--no-cache",
|
||||
"--demuxer=rawvideo",
|
||||
f"--demuxer-rawvideo-w={w}",
|
||||
f"--demuxer-rawvideo-h={h}",
|
||||
"--demuxer-rawvideo-mp-format=rgb24",
|
||||
f"--demuxer-rawvideo-fps={self.fps}",
|
||||
f"--title={self.title}",
|
||||
"-",
|
||||
]
|
||||
print(f"Starting mpv: {' '.join(cmd)}", file=sys.stderr)
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
|
||||
# Start audio playback if we have an audio source
|
||||
if self.audio_source:
|
||||
audio_cmd = [
|
||||
"ffplay", "-nodisp", "-autoexit", "-loglevel", "quiet",
|
||||
str(self.audio_source)
|
||||
]
|
||||
print(f"Starting audio: {self.audio_source}", file=sys.stderr)
|
||||
self._audio_process = subprocess.Popen(
|
||||
audio_cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
"""Display frame."""
|
||||
if not self._is_open:
|
||||
return
|
||||
|
||||
# Ensure frame is correct format
|
||||
if frame.dtype != np.uint8:
|
||||
frame = np.clip(frame, 0, 255).astype(np.uint8)
|
||||
if not frame.flags['C_CONTIGUOUS']:
|
||||
frame = np.ascontiguousarray(frame)
|
||||
|
||||
# Start mpv on first frame
|
||||
if self._process is None:
|
||||
self._start_mpv((frame.shape[1], frame.shape[0]))
|
||||
|
||||
# Check if mpv is still running
|
||||
if self._process.poll() is not None:
|
||||
self._is_open = False
|
||||
return
|
||||
|
||||
try:
|
||||
self._process.stdin.write(frame.tobytes())
|
||||
self._process.stdin.flush() # Prevent buffering
|
||||
except BrokenPipeError:
|
||||
self._is_open = False
|
||||
|
||||
def close(self):
|
||||
"""Close the display and audio."""
|
||||
if self._process:
|
||||
try:
|
||||
self._process.stdin.close()
|
||||
except:
|
||||
pass
|
||||
self._process.terminate()
|
||||
self._process.wait()
|
||||
if self._audio_process:
|
||||
self._audio_process.terminate()
|
||||
self._audio_process.wait()
|
||||
self._is_open = False
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
if self._process and self._process.poll() is not None:
|
||||
self._is_open = False
|
||||
return self._is_open
|
||||
|
||||
|
||||
class FileOutput(Output):
|
||||
"""
|
||||
Write frames to a video file using ffmpeg.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: str,
|
||||
size: Tuple[int, int],
|
||||
fps: float = 30,
|
||||
codec: str = "libx264",
|
||||
crf: int = 18,
|
||||
preset: str = "fast",
|
||||
audio_source: str = None,
|
||||
):
|
||||
self.path = Path(path)
|
||||
self.size = size
|
||||
self.fps = fps
|
||||
self._is_open = True
|
||||
|
||||
# Build ffmpeg command
|
||||
cmd = [
|
||||
"ffmpeg", "-y",
|
||||
"-f", "rawvideo",
|
||||
"-vcodec", "rawvideo",
|
||||
"-pix_fmt", "rgb24",
|
||||
"-s", f"{size[0]}x{size[1]}",
|
||||
"-r", str(fps),
|
||||
"-i", "-",
|
||||
]
|
||||
|
||||
# Add audio input if provided
|
||||
if audio_source:
|
||||
cmd.extend(["-i", str(audio_source)])
|
||||
# Explicitly map: video from input 0 (rawvideo), audio from input 1
|
||||
cmd.extend(["-map", "0:v", "-map", "1:a"])
|
||||
|
||||
cmd.extend([
|
||||
"-c:v", codec,
|
||||
"-preset", preset,
|
||||
"-crf", str(crf),
|
||||
"-pix_fmt", "yuv420p",
|
||||
])
|
||||
|
||||
# Add audio codec if we have audio
|
||||
if audio_source:
|
||||
cmd.extend(["-c:a", "aac", "-b:a", "192k", "-shortest"])
|
||||
|
||||
cmd.append(str(self.path))
|
||||
|
||||
import sys
|
||||
print(f"FileOutput cmd: {' '.join(cmd)}", file=sys.stderr)
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stderr=None, # Show errors for debugging
|
||||
)
|
||||
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
"""Write frame to video file."""
|
||||
if not self._is_open or self._process.poll() is not None:
|
||||
self._is_open = False
|
||||
return
|
||||
|
||||
# Resize if needed
|
||||
if frame.shape[1] != self.size[0] or frame.shape[0] != self.size[1]:
|
||||
import cv2
|
||||
frame = cv2.resize(frame, self.size)
|
||||
|
||||
try:
|
||||
self._process.stdin.write(frame.tobytes())
|
||||
except BrokenPipeError:
|
||||
self._is_open = False
|
||||
|
||||
def close(self):
|
||||
"""Close the video file."""
|
||||
if self._process:
|
||||
self._process.stdin.close()
|
||||
self._process.wait()
|
||||
self._is_open = False
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return self._is_open and self._process.poll() is None
|
||||
|
||||
|
||||
class MultiOutput(Output):
|
||||
"""
|
||||
Write to multiple outputs simultaneously.
|
||||
|
||||
Useful for recording while showing preview.
|
||||
"""
|
||||
|
||||
def __init__(self, outputs: list):
|
||||
self.outputs = outputs
|
||||
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
for output in self.outputs:
|
||||
if output.is_open:
|
||||
output.write(frame, t)
|
||||
|
||||
def close(self):
|
||||
for output in self.outputs:
|
||||
output.close()
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return any(o.is_open for o in self.outputs)
|
||||
|
||||
|
||||
class NullOutput(Output):
|
||||
"""
|
||||
Discard frames (for benchmarking).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._is_open = True
|
||||
self.frame_count = 0
|
||||
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
self.frame_count += 1
|
||||
|
||||
def close(self):
|
||||
self._is_open = False
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
return self._is_open
|
||||
|
||||
|
||||
class PipeOutput(Output):
|
||||
"""
|
||||
Pipe frames directly to mpv.
|
||||
|
||||
Launches mpv with rawvideo demuxer and writes frames to stdin.
|
||||
"""
|
||||
|
||||
def __init__(self, size: Tuple[int, int], fps: float = 30, audio_source: str = None):
|
||||
self.size = size
|
||||
self.fps = fps
|
||||
self.audio_source = audio_source
|
||||
self._is_open = True
|
||||
self._process = None
|
||||
self._audio_process = None
|
||||
self._started = False
|
||||
|
||||
def _start(self):
|
||||
"""Start mpv and audio on first frame."""
|
||||
if self._started:
|
||||
return
|
||||
self._started = True
|
||||
|
||||
import sys
|
||||
w, h = self.size
|
||||
|
||||
# Start mpv
|
||||
cmd = [
|
||||
"mpv", "--no-cache",
|
||||
"--demuxer=rawvideo",
|
||||
f"--demuxer-rawvideo-w={w}",
|
||||
f"--demuxer-rawvideo-h={h}",
|
||||
"--demuxer-rawvideo-mp-format=rgb24",
|
||||
f"--demuxer-rawvideo-fps={self.fps}",
|
||||
"--title=Streaming",
|
||||
"-"
|
||||
]
|
||||
print(f"Starting mpv: {w}x{h} @ {self.fps}fps", file=sys.stderr)
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
# Start audio
|
||||
if self.audio_source:
|
||||
audio_cmd = [
|
||||
"ffplay", "-nodisp", "-autoexit", "-loglevel", "quiet",
|
||||
str(self.audio_source)
|
||||
]
|
||||
print(f"Starting audio: {self.audio_source}", file=sys.stderr)
|
||||
self._audio_process = subprocess.Popen(
|
||||
audio_cmd,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
|
||||
def write(self, frame: np.ndarray, t: float):
|
||||
"""Write frame to mpv."""
|
||||
if not self._is_open:
|
||||
return
|
||||
|
||||
self._start()
|
||||
|
||||
# Check mpv still running
|
||||
if self._process.poll() is not None:
|
||||
self._is_open = False
|
||||
return
|
||||
|
||||
# Resize if needed
|
||||
if frame.shape[1] != self.size[0] or frame.shape[0] != self.size[1]:
|
||||
import cv2
|
||||
frame = cv2.resize(frame, self.size)
|
||||
|
||||
# Ensure correct format
|
||||
if frame.dtype != np.uint8:
|
||||
frame = np.clip(frame, 0, 255).astype(np.uint8)
|
||||
if not frame.flags['C_CONTIGUOUS']:
|
||||
frame = np.ascontiguousarray(frame)
|
||||
|
||||
try:
|
||||
self._process.stdin.write(frame.tobytes())
|
||||
self._process.stdin.flush()
|
||||
except BrokenPipeError:
|
||||
self._is_open = False
|
||||
|
||||
def close(self):
|
||||
"""Close mpv and audio."""
|
||||
if self._process:
|
||||
try:
|
||||
self._process.stdin.close()
|
||||
except:
|
||||
pass
|
||||
self._process.terminate()
|
||||
self._process.wait()
|
||||
if self._audio_process:
|
||||
self._audio_process.terminate()
|
||||
self._audio_process.wait()
|
||||
self._is_open = False
|
||||
|
||||
@property
|
||||
def is_open(self) -> bool:
|
||||
if self._process and self._process.poll() is not None:
|
||||
self._is_open = False
|
||||
return self._is_open
|
||||
846
streaming/pipeline.py
Normal file
846
streaming/pipeline.py
Normal file
@@ -0,0 +1,846 @@
|
||||
"""
|
||||
Streaming pipeline executor.
|
||||
|
||||
Directly executes compiled sexp recipes frame-by-frame.
|
||||
No adapter layer - frames and analysis flow through the DAG.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .sources import VideoSource
|
||||
from .audio import StreamingAudioAnalyzer
|
||||
from .output import DisplayOutput, FileOutput
|
||||
from .sexp_interp import SexpInterpreter
|
||||
|
||||
|
||||
@dataclass
|
||||
class FrameContext:
|
||||
"""Context passed through the pipeline for each frame."""
|
||||
t: float # Current time
|
||||
energy: float = 0.0
|
||||
is_beat: bool = False
|
||||
beat_count: int = 0
|
||||
analysis: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class StreamingPipeline:
|
||||
"""
|
||||
Executes a compiled sexp recipe as a streaming pipeline.
|
||||
|
||||
Frames flow through the DAG directly - no adapter needed.
|
||||
Each node is evaluated lazily when its output is requested.
|
||||
"""
|
||||
|
||||
def __init__(self, compiled_recipe, recipe_dir: Path = None, fps: float = 30, seed: int = 42,
|
||||
output_size: tuple = None):
|
||||
self.recipe = compiled_recipe
|
||||
self.recipe_dir = recipe_dir or Path(".")
|
||||
self.fps = fps
|
||||
self.seed = seed
|
||||
|
||||
# Build node lookup
|
||||
self.nodes = {n['id']: n for n in compiled_recipe.nodes}
|
||||
|
||||
# Runtime state
|
||||
self.sources: Dict[str, VideoSource] = {}
|
||||
self.audio_analyzer: Optional[StreamingAudioAnalyzer] = None
|
||||
self.audio_source_path: Optional[str] = None
|
||||
|
||||
# Sexp interpreter for expressions
|
||||
self.interp = SexpInterpreter()
|
||||
|
||||
# Scan state (node_id -> current value)
|
||||
self.scan_state: Dict[str, Any] = {}
|
||||
self.scan_emit: Dict[str, Any] = {}
|
||||
|
||||
# SLICE_ON state
|
||||
self.slice_on_acc: Dict[str, Any] = {}
|
||||
self.slice_on_result: Dict[str, Any] = {}
|
||||
|
||||
# Frame cache for current timestep (cleared each frame)
|
||||
self._frame_cache: Dict[str, np.ndarray] = {}
|
||||
|
||||
# Context for current frame
|
||||
self.ctx = FrameContext(t=0.0)
|
||||
|
||||
# Output size (w, h) - set after sources are initialized
|
||||
self._output_size = output_size
|
||||
|
||||
# Initialize
|
||||
self._init_sources()
|
||||
self._init_scans()
|
||||
self._init_slice_on()
|
||||
|
||||
# Set output size from first source if not specified
|
||||
if self._output_size is None and self.sources:
|
||||
first_source = next(iter(self.sources.values()))
|
||||
self._output_size = first_source._size
|
||||
|
||||
def _init_sources(self):
|
||||
"""Initialize video and audio sources."""
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SOURCE':
|
||||
config = node.get('config', {})
|
||||
path = config.get('path')
|
||||
if path:
|
||||
full_path = (self.recipe_dir / path).resolve()
|
||||
suffix = full_path.suffix.lower()
|
||||
|
||||
if suffix in ('.mp4', '.webm', '.mov', '.avi', '.mkv'):
|
||||
if not full_path.exists():
|
||||
print(f"Warning: video not found: {full_path}", file=sys.stderr)
|
||||
continue
|
||||
self.sources[node['id']] = VideoSource(
|
||||
str(full_path),
|
||||
target_fps=self.fps
|
||||
)
|
||||
elif suffix in ('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac'):
|
||||
if not full_path.exists():
|
||||
print(f"Warning: audio not found: {full_path}", file=sys.stderr)
|
||||
continue
|
||||
self.audio_source_path = str(full_path)
|
||||
self.audio_analyzer = StreamingAudioAnalyzer(str(full_path))
|
||||
|
||||
def _init_scans(self):
|
||||
"""Initialize scan nodes with their initial state."""
|
||||
import random
|
||||
seed_offset = 0
|
||||
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SCAN':
|
||||
config = node.get('config', {})
|
||||
|
||||
# Create RNG for this scan
|
||||
scan_seed = config.get('seed', self.seed + seed_offset)
|
||||
rng = random.Random(scan_seed)
|
||||
seed_offset += 1
|
||||
|
||||
# Evaluate initial value
|
||||
init_expr = config.get('init', 0)
|
||||
init_value = self.interp.eval(init_expr, {})
|
||||
|
||||
self.scan_state[node['id']] = {
|
||||
'value': init_value,
|
||||
'rng': rng,
|
||||
'config': config,
|
||||
}
|
||||
|
||||
# Compute initial emit
|
||||
self._update_scan_emit(node['id'])
|
||||
|
||||
def _update_scan_emit(self, node_id: str):
|
||||
"""Update the emit value for a scan."""
|
||||
state = self.scan_state[node_id]
|
||||
config = state['config']
|
||||
emit_expr = config.get('emit_expr', config.get('emit', None))
|
||||
|
||||
if emit_expr is None:
|
||||
# No emit expression - emit the value directly
|
||||
self.scan_emit[node_id] = state['value']
|
||||
return
|
||||
|
||||
# Build environment from state
|
||||
env = {}
|
||||
if isinstance(state['value'], dict):
|
||||
env.update(state['value'])
|
||||
else:
|
||||
env['acc'] = state['value']
|
||||
|
||||
env['beat_count'] = self.ctx.beat_count
|
||||
env['time'] = self.ctx.t
|
||||
|
||||
# Set RNG for interpreter
|
||||
self.interp.rng = state['rng']
|
||||
|
||||
self.scan_emit[node_id] = self.interp.eval(emit_expr, env)
|
||||
|
||||
def _step_scan(self, node_id: str):
|
||||
"""Step a scan forward on beat."""
|
||||
state = self.scan_state[node_id]
|
||||
config = state['config']
|
||||
step_expr = config.get('step_expr', config.get('step', None))
|
||||
|
||||
if step_expr is None:
|
||||
return
|
||||
|
||||
# Build environment
|
||||
env = {}
|
||||
if isinstance(state['value'], dict):
|
||||
env.update(state['value'])
|
||||
else:
|
||||
env['acc'] = state['value']
|
||||
|
||||
env['beat_count'] = self.ctx.beat_count
|
||||
env['time'] = self.ctx.t
|
||||
|
||||
# Set RNG
|
||||
self.interp.rng = state['rng']
|
||||
|
||||
# Evaluate step
|
||||
new_value = self.interp.eval(step_expr, env)
|
||||
state['value'] = new_value
|
||||
|
||||
# Update emit
|
||||
self._update_scan_emit(node_id)
|
||||
|
||||
def _init_slice_on(self):
|
||||
"""Initialize SLICE_ON nodes."""
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
config = node.get('config', {})
|
||||
init = config.get('init', {})
|
||||
self.slice_on_acc[node['id']] = dict(init)
|
||||
|
||||
# Evaluate initial state
|
||||
self._eval_slice_on(node['id'])
|
||||
|
||||
def _eval_slice_on(self, node_id: str):
|
||||
"""Evaluate a SLICE_ON node's Lambda."""
|
||||
node = self.nodes[node_id]
|
||||
config = node.get('config', {})
|
||||
fn = config.get('fn')
|
||||
videos = config.get('videos', [])
|
||||
|
||||
if not fn:
|
||||
return
|
||||
|
||||
acc = self.slice_on_acc[node_id]
|
||||
n_videos = len(videos)
|
||||
|
||||
# Set up environment
|
||||
self.interp.globals['videos'] = list(range(n_videos))
|
||||
|
||||
try:
|
||||
from .sexp_interp import eval_slice_on_lambda
|
||||
result = eval_slice_on_lambda(
|
||||
fn, acc, self.ctx.beat_count, 0, 1,
|
||||
list(range(n_videos)), self.interp
|
||||
)
|
||||
self.slice_on_result[node_id] = result
|
||||
|
||||
# Update accumulator
|
||||
if 'acc' in result:
|
||||
self.slice_on_acc[node_id] = result['acc']
|
||||
except Exception as e:
|
||||
print(f"SLICE_ON eval error: {e}", file=sys.stderr)
|
||||
|
||||
def _on_beat(self):
|
||||
"""Called when a beat is detected."""
|
||||
self.ctx.beat_count += 1
|
||||
|
||||
# Step all scans
|
||||
for node_id in self.scan_state:
|
||||
self._step_scan(node_id)
|
||||
|
||||
# Step all SLICE_ON nodes
|
||||
for node_id in self.slice_on_acc:
|
||||
self._eval_slice_on(node_id)
|
||||
|
||||
def _get_frame(self, node_id: str) -> Optional[np.ndarray]:
|
||||
"""
|
||||
Get the output frame for a node at current time.
|
||||
|
||||
Recursively evaluates inputs as needed.
|
||||
Results are cached for the current timestep.
|
||||
"""
|
||||
if node_id in self._frame_cache:
|
||||
return self._frame_cache[node_id]
|
||||
|
||||
node = self.nodes.get(node_id)
|
||||
if not node:
|
||||
return None
|
||||
|
||||
node_type = node.get('type')
|
||||
|
||||
if node_type == 'SOURCE':
|
||||
frame = self._eval_source(node)
|
||||
elif node_type == 'SEGMENT':
|
||||
frame = self._eval_segment(node)
|
||||
elif node_type == 'EFFECT':
|
||||
frame = self._eval_effect(node)
|
||||
elif node_type == 'SLICE_ON':
|
||||
frame = self._eval_slice_on_frame(node)
|
||||
else:
|
||||
# Unknown node type - try to pass through input
|
||||
inputs = node.get('inputs', [])
|
||||
frame = self._get_frame(inputs[0]) if inputs else None
|
||||
|
||||
self._frame_cache[node_id] = frame
|
||||
return frame
|
||||
|
||||
def _eval_source(self, node: dict) -> Optional[np.ndarray]:
|
||||
"""Evaluate a SOURCE node."""
|
||||
source = self.sources.get(node['id'])
|
||||
if source:
|
||||
return source.read_frame(self.ctx.t)
|
||||
return None
|
||||
|
||||
def _eval_segment(self, node: dict) -> Optional[np.ndarray]:
|
||||
"""Evaluate a SEGMENT node (time segment of source)."""
|
||||
inputs = node.get('inputs', [])
|
||||
if not inputs:
|
||||
return None
|
||||
|
||||
config = node.get('config', {})
|
||||
start = config.get('start', 0)
|
||||
duration = config.get('duration')
|
||||
|
||||
# Resolve any bindings
|
||||
if isinstance(start, dict):
|
||||
start = self._resolve_binding(start) if start.get('_binding') else 0
|
||||
if isinstance(duration, dict):
|
||||
duration = self._resolve_binding(duration) if duration.get('_binding') else None
|
||||
|
||||
# Adjust time for segment
|
||||
t_local = self.ctx.t + (start if isinstance(start, (int, float)) else 0)
|
||||
if duration and isinstance(duration, (int, float)):
|
||||
t_local = t_local % duration # Loop within segment
|
||||
|
||||
# Get source frame at adjusted time
|
||||
source_id = inputs[0]
|
||||
source = self.sources.get(source_id)
|
||||
if source:
|
||||
return source.read_frame(t_local)
|
||||
|
||||
return self._get_frame(source_id)
|
||||
|
||||
def _eval_effect(self, node: dict) -> Optional[np.ndarray]:
|
||||
"""Evaluate an EFFECT node."""
|
||||
import cv2
|
||||
|
||||
inputs = node.get('inputs', [])
|
||||
config = node.get('config', {})
|
||||
effect_name = config.get('effect')
|
||||
|
||||
# Get input frame(s)
|
||||
input_frames = [self._get_frame(inp) for inp in inputs]
|
||||
input_frames = [f for f in input_frames if f is not None]
|
||||
|
||||
if not input_frames:
|
||||
return None
|
||||
|
||||
frame = input_frames[0]
|
||||
|
||||
# Resolve bindings in config
|
||||
params = self._resolve_config(config)
|
||||
|
||||
# Apply effect based on name
|
||||
if effect_name == 'rotate':
|
||||
angle = params.get('angle', 0)
|
||||
if abs(angle) > 0.5:
|
||||
h, w = frame.shape[:2]
|
||||
center = (w // 2, h // 2)
|
||||
matrix = cv2.getRotationMatrix2D(center, angle, 1.0)
|
||||
frame = cv2.warpAffine(frame, matrix, (w, h))
|
||||
|
||||
elif effect_name == 'zoom':
|
||||
amount = params.get('amount', 1.0)
|
||||
if abs(amount - 1.0) > 0.01:
|
||||
frame = self._apply_zoom(frame, amount)
|
||||
|
||||
elif effect_name == 'invert':
|
||||
amount = params.get('amount', 0)
|
||||
if amount > 0.01:
|
||||
inverted = 255 - frame
|
||||
frame = cv2.addWeighted(frame, 1 - amount, inverted, amount, 0)
|
||||
|
||||
elif effect_name == 'hue_shift':
|
||||
degrees = params.get('degrees', 0)
|
||||
if abs(degrees) > 1:
|
||||
hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
|
||||
hsv[:, :, 0] = (hsv[:, :, 0].astype(int) + int(degrees / 2)) % 180
|
||||
frame = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)
|
||||
|
||||
elif effect_name == 'blend':
|
||||
if len(input_frames) >= 2:
|
||||
opacity = params.get('opacity', 0.5)
|
||||
frame = cv2.addWeighted(input_frames[0], 1 - opacity,
|
||||
input_frames[1], opacity, 0)
|
||||
|
||||
elif effect_name == 'blend_multi':
|
||||
weights = params.get('weights', [])
|
||||
if len(input_frames) > 1 and weights:
|
||||
h, w = input_frames[0].shape[:2]
|
||||
result = np.zeros((h, w, 3), dtype=np.float32)
|
||||
for f, wt in zip(input_frames, weights):
|
||||
if f is not None and wt > 0.001:
|
||||
if f.shape[:2] != (h, w):
|
||||
f = cv2.resize(f, (w, h))
|
||||
result += f.astype(np.float32) * wt
|
||||
frame = np.clip(result, 0, 255).astype(np.uint8)
|
||||
|
||||
elif effect_name == 'ripple':
|
||||
amp = params.get('amplitude', 0)
|
||||
if amp > 1:
|
||||
frame = self._apply_ripple(frame, amp,
|
||||
params.get('center_x', 0.5),
|
||||
params.get('center_y', 0.5),
|
||||
params.get('frequency', 8),
|
||||
params.get('decay', 2),
|
||||
params.get('speed', 5))
|
||||
|
||||
return frame
|
||||
|
||||
def _eval_slice_on_frame(self, node: dict) -> Optional[np.ndarray]:
|
||||
"""Evaluate a SLICE_ON node - returns composited frame."""
|
||||
import cv2
|
||||
|
||||
config = node.get('config', {})
|
||||
video_ids = config.get('videos', [])
|
||||
result = self.slice_on_result.get(node['id'], {})
|
||||
|
||||
if not result:
|
||||
# No result yet - return first video
|
||||
if video_ids:
|
||||
return self._get_frame(video_ids[0])
|
||||
return None
|
||||
|
||||
# Get layers and compose info
|
||||
layers = result.get('layers', [])
|
||||
compose = result.get('compose', {})
|
||||
weights = compose.get('weights', [])
|
||||
|
||||
if not layers or not weights:
|
||||
if video_ids:
|
||||
return self._get_frame(video_ids[0])
|
||||
return None
|
||||
|
||||
# Get frames for each layer
|
||||
frames = []
|
||||
for i, layer in enumerate(layers):
|
||||
video_idx = layer.get('video', i)
|
||||
if video_idx < len(video_ids):
|
||||
frame = self._get_frame(video_ids[video_idx])
|
||||
|
||||
# Apply layer effects (zoom)
|
||||
effects = layer.get('effects', [])
|
||||
for eff in effects:
|
||||
eff_name = eff.get('effect')
|
||||
if hasattr(eff_name, 'name'):
|
||||
eff_name = eff_name.name
|
||||
if eff_name == 'zoom':
|
||||
zoom_amt = eff.get('amount', 1.0)
|
||||
if frame is not None:
|
||||
frame = self._apply_zoom(frame, zoom_amt)
|
||||
|
||||
frames.append(frame)
|
||||
else:
|
||||
frames.append(None)
|
||||
|
||||
# Composite with weights - use consistent output size
|
||||
if self._output_size:
|
||||
w, h = self._output_size
|
||||
else:
|
||||
# Fallback to first non-None frame size
|
||||
for f in frames:
|
||||
if f is not None:
|
||||
h, w = f.shape[:2]
|
||||
break
|
||||
else:
|
||||
return None
|
||||
|
||||
output = np.zeros((h, w, 3), dtype=np.float32)
|
||||
|
||||
for frame, weight in zip(frames, weights):
|
||||
if frame is None or weight < 0.001:
|
||||
continue
|
||||
|
||||
# Resize to output size
|
||||
if frame.shape[1] != w or frame.shape[0] != h:
|
||||
frame = cv2.resize(frame, (w, h))
|
||||
|
||||
output += frame.astype(np.float32) * weight
|
||||
|
||||
# Normalize weights
|
||||
total_weight = sum(wt for wt in weights if wt > 0.001)
|
||||
if total_weight > 0 and abs(total_weight - 1.0) > 0.01:
|
||||
output /= total_weight
|
||||
|
||||
return np.clip(output, 0, 255).astype(np.uint8)
|
||||
|
||||
def _resolve_config(self, config: dict) -> dict:
|
||||
"""Resolve bindings in effect config to actual values."""
|
||||
resolved = {}
|
||||
|
||||
for key, value in config.items():
|
||||
if key in ('effect', 'effect_path', 'effect_cid', 'effects_registry',
|
||||
'analysis_refs', 'inputs', 'cid'):
|
||||
continue
|
||||
|
||||
if isinstance(value, dict) and value.get('_binding'):
|
||||
resolved[key] = self._resolve_binding(value)
|
||||
elif isinstance(value, dict) and value.get('_expr'):
|
||||
resolved[key] = self._resolve_expr(value)
|
||||
else:
|
||||
resolved[key] = value
|
||||
|
||||
return resolved
|
||||
|
||||
def _resolve_binding(self, binding: dict) -> Any:
|
||||
"""Resolve a binding to its current value."""
|
||||
source_id = binding.get('source')
|
||||
feature = binding.get('feature', 'values')
|
||||
range_map = binding.get('range')
|
||||
|
||||
# Get raw value from scan or analysis
|
||||
if source_id in self.scan_emit:
|
||||
value = self.scan_emit[source_id]
|
||||
elif source_id in self.ctx.analysis:
|
||||
data = self.ctx.analysis[source_id]
|
||||
value = data.get(feature, data.get('values', [0]))[0] if isinstance(data, dict) else data
|
||||
else:
|
||||
# Fallback to energy
|
||||
value = self.ctx.energy
|
||||
|
||||
# Extract feature from dict
|
||||
if isinstance(value, dict) and feature in value:
|
||||
value = value[feature]
|
||||
|
||||
# Apply range mapping
|
||||
if range_map and isinstance(value, (int, float)):
|
||||
lo, hi = range_map
|
||||
value = lo + value * (hi - lo)
|
||||
|
||||
return value
|
||||
|
||||
def _resolve_expr(self, expr: dict) -> Any:
|
||||
"""Resolve a compiled expression."""
|
||||
env = {
|
||||
'energy': self.ctx.energy,
|
||||
'beat_count': self.ctx.beat_count,
|
||||
't': self.ctx.t,
|
||||
}
|
||||
|
||||
# Add scan values
|
||||
for scan_id, value in self.scan_emit.items():
|
||||
# Use short form if available
|
||||
env[scan_id] = value
|
||||
|
||||
# Extract the actual expression from _expr wrapper
|
||||
actual_expr = expr.get('_expr', expr)
|
||||
return self.interp.eval(actual_expr, env)
|
||||
|
||||
def _apply_zoom(self, frame: np.ndarray, amount: float) -> np.ndarray:
|
||||
"""Apply zoom to frame."""
|
||||
import cv2
|
||||
h, w = frame.shape[:2]
|
||||
|
||||
if amount > 1.01:
|
||||
# Zoom in: crop center
|
||||
new_w, new_h = int(w / amount), int(h / amount)
|
||||
if new_w > 0 and new_h > 0:
|
||||
x1, y1 = (w - new_w) // 2, (h - new_h) // 2
|
||||
cropped = frame[y1:y1+new_h, x1:x1+new_w]
|
||||
return cv2.resize(cropped, (w, h))
|
||||
elif amount < 0.99:
|
||||
# Zoom out: shrink and center
|
||||
scaled_w, scaled_h = int(w * amount), int(h * amount)
|
||||
if scaled_w > 0 and scaled_h > 0:
|
||||
shrunk = cv2.resize(frame, (scaled_w, scaled_h))
|
||||
canvas = np.zeros((h, w, 3), dtype=np.uint8)
|
||||
x_off, y_off = (w - scaled_w) // 2, (h - scaled_h) // 2
|
||||
canvas[y_off:y_off+scaled_h, x_off:x_off+scaled_w] = shrunk
|
||||
return canvas
|
||||
|
||||
return frame
|
||||
|
||||
def _apply_ripple(self, frame: np.ndarray, amplitude: float,
|
||||
cx: float, cy: float, frequency: float,
|
||||
decay: float, speed: float) -> np.ndarray:
|
||||
"""Apply ripple effect."""
|
||||
import cv2
|
||||
h, w = frame.shape[:2]
|
||||
|
||||
# Create coordinate grids
|
||||
y_coords, x_coords = np.mgrid[0:h, 0:w].astype(np.float32)
|
||||
|
||||
# Normalize to center
|
||||
center_x, center_y = w * cx, h * cy
|
||||
dx = x_coords - center_x
|
||||
dy = y_coords - center_y
|
||||
dist = np.sqrt(dx**2 + dy**2)
|
||||
|
||||
# Ripple displacement
|
||||
phase = self.ctx.t * speed
|
||||
ripple = amplitude * np.sin(dist / frequency - phase) * np.exp(-dist * decay / max(w, h))
|
||||
|
||||
# Displace coordinates
|
||||
angle = np.arctan2(dy, dx)
|
||||
map_x = (x_coords + ripple * np.cos(angle)).astype(np.float32)
|
||||
map_y = (y_coords + ripple * np.sin(angle)).astype(np.float32)
|
||||
|
||||
return cv2.remap(frame, map_x, map_y, cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)
|
||||
|
||||
def _find_output_node(self) -> Optional[str]:
|
||||
"""Find the final output node (MUX or last EFFECT)."""
|
||||
# Look for MUX node
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'MUX':
|
||||
return node['id']
|
||||
|
||||
# Otherwise find last EFFECT after SLICE_ON
|
||||
last_effect = None
|
||||
found_slice_on = False
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
found_slice_on = True
|
||||
elif node.get('type') == 'EFFECT' and found_slice_on:
|
||||
last_effect = node['id']
|
||||
|
||||
return last_effect
|
||||
|
||||
def render_frame(self, t: float) -> Optional[np.ndarray]:
|
||||
"""Render a single frame at time t."""
|
||||
# Clear frame cache
|
||||
self._frame_cache.clear()
|
||||
|
||||
# Update context
|
||||
self.ctx.t = t
|
||||
|
||||
# Update audio analysis
|
||||
if self.audio_analyzer:
|
||||
self.audio_analyzer.set_time(t)
|
||||
energy = self.audio_analyzer.get_energy()
|
||||
is_beat = self.audio_analyzer.get_beat()
|
||||
|
||||
# Beat edge detection
|
||||
was_beat = self.ctx.is_beat
|
||||
self.ctx.energy = energy
|
||||
self.ctx.is_beat = is_beat
|
||||
|
||||
if is_beat and not was_beat:
|
||||
self._on_beat()
|
||||
|
||||
# Store in analysis dict
|
||||
self.ctx.analysis['live_energy'] = {'values': [energy]}
|
||||
self.ctx.analysis['live_beat'] = {'values': [1.0 if is_beat else 0.0]}
|
||||
|
||||
# Find output node and render
|
||||
output_node = self._find_output_node()
|
||||
if output_node:
|
||||
frame = self._get_frame(output_node)
|
||||
# Normalize to output size
|
||||
if frame is not None and self._output_size:
|
||||
w, h = self._output_size
|
||||
if frame.shape[1] != w or frame.shape[0] != h:
|
||||
import cv2
|
||||
frame = cv2.resize(frame, (w, h))
|
||||
return frame
|
||||
|
||||
return None
|
||||
|
||||
def run(self, output: str = "preview", duration: float = None):
|
||||
"""
|
||||
Run the pipeline.
|
||||
|
||||
Args:
|
||||
output: "preview", filename, or Output object
|
||||
duration: Duration in seconds (default: audio duration or 60s)
|
||||
"""
|
||||
# Determine duration
|
||||
if duration is None:
|
||||
if self.audio_analyzer:
|
||||
duration = self.audio_analyzer.duration
|
||||
else:
|
||||
duration = 60.0
|
||||
|
||||
# Create output
|
||||
if output == "preview":
|
||||
# Get frame size from first source
|
||||
first_source = next(iter(self.sources.values()), None)
|
||||
if first_source:
|
||||
w, h = first_source._size
|
||||
else:
|
||||
w, h = 720, 720
|
||||
out = DisplayOutput(size=(w, h), fps=self.fps, audio_source=self.audio_source_path)
|
||||
elif isinstance(output, str):
|
||||
first_source = next(iter(self.sources.values()), None)
|
||||
if first_source:
|
||||
w, h = first_source._size
|
||||
else:
|
||||
w, h = 720, 720
|
||||
out = FileOutput(output, size=(w, h), fps=self.fps, audio_source=self.audio_source_path)
|
||||
else:
|
||||
out = output
|
||||
|
||||
frame_time = 1.0 / self.fps
|
||||
n_frames = int(duration * self.fps)
|
||||
|
||||
print(f"Streaming: {len(self.sources)} sources -> {output}", file=sys.stderr)
|
||||
print(f"Duration: {duration:.1f}s, {n_frames} frames @ {self.fps}fps", file=sys.stderr)
|
||||
|
||||
start_time = time.time()
|
||||
frame_count = 0
|
||||
|
||||
try:
|
||||
for frame_num in range(n_frames):
|
||||
t = frame_num * frame_time
|
||||
|
||||
frame = self.render_frame(t)
|
||||
|
||||
if frame is not None:
|
||||
out.write(frame, t)
|
||||
frame_count += 1
|
||||
|
||||
# Progress
|
||||
if frame_num % 50 == 0:
|
||||
elapsed = time.time() - start_time
|
||||
fps = frame_count / elapsed if elapsed > 0 else 0
|
||||
pct = 100 * frame_num / n_frames
|
||||
print(f"\r{pct:5.1f}% | {fps:5.1f} fps | frame {frame_num}/{n_frames}",
|
||||
end="", file=sys.stderr)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted", file=sys.stderr)
|
||||
finally:
|
||||
out.close()
|
||||
for src in self.sources.values():
|
||||
src.close()
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
avg_fps = frame_count / elapsed if elapsed > 0 else 0
|
||||
print(f"\nCompleted: {frame_count} frames in {elapsed:.1f}s ({avg_fps:.1f} fps avg)",
|
||||
file=sys.stderr)
|
||||
|
||||
|
||||
def run_pipeline(recipe_path: str, output: str = "preview",
|
||||
duration: float = None, fps: float = None):
|
||||
"""
|
||||
Run a recipe through the streaming pipeline.
|
||||
|
||||
No adapter layer - directly executes the compiled recipe.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
# Add artdag to path
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "artdag"))
|
||||
|
||||
from artdag.sexp.compiler import compile_string
|
||||
|
||||
recipe_path = Path(recipe_path)
|
||||
recipe_text = recipe_path.read_text()
|
||||
compiled = compile_string(recipe_text, {}, recipe_dir=recipe_path.parent)
|
||||
|
||||
pipeline = StreamingPipeline(
|
||||
compiled,
|
||||
recipe_dir=recipe_path.parent,
|
||||
fps=fps or compiled.encoding.get('fps', 30),
|
||||
)
|
||||
|
||||
pipeline.run(output=output, duration=duration)
|
||||
|
||||
|
||||
def run_pipeline_piped(recipe_path: str, duration: float = None, fps: float = None):
|
||||
"""
|
||||
Run pipeline and pipe directly to mpv with audio.
|
||||
"""
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
import sys
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "artdag"))
|
||||
from artdag.sexp.compiler import compile_string
|
||||
|
||||
recipe_path = Path(recipe_path)
|
||||
recipe_text = recipe_path.read_text()
|
||||
compiled = compile_string(recipe_text, {}, recipe_dir=recipe_path.parent)
|
||||
|
||||
pipeline = StreamingPipeline(
|
||||
compiled,
|
||||
recipe_dir=recipe_path.parent,
|
||||
fps=fps or compiled.encoding.get('fps', 30),
|
||||
)
|
||||
|
||||
# Get frame info
|
||||
first_source = next(iter(pipeline.sources.values()), None)
|
||||
if first_source:
|
||||
w, h = first_source._size
|
||||
else:
|
||||
w, h = 720, 720
|
||||
|
||||
# Determine duration
|
||||
if duration is None:
|
||||
if pipeline.audio_analyzer:
|
||||
duration = pipeline.audio_analyzer.duration
|
||||
else:
|
||||
duration = 60.0
|
||||
|
||||
actual_fps = fps or compiled.encoding.get('fps', 30)
|
||||
n_frames = int(duration * actual_fps)
|
||||
frame_time = 1.0 / actual_fps
|
||||
|
||||
print(f"Streaming {n_frames} frames @ {actual_fps}fps to mpv", file=sys.stderr)
|
||||
|
||||
# Start mpv
|
||||
mpv_cmd = [
|
||||
"mpv", "--no-cache",
|
||||
"--demuxer=rawvideo",
|
||||
f"--demuxer-rawvideo-w={w}",
|
||||
f"--demuxer-rawvideo-h={h}",
|
||||
"--demuxer-rawvideo-mp-format=rgb24",
|
||||
f"--demuxer-rawvideo-fps={actual_fps}",
|
||||
"--title=Streaming Pipeline",
|
||||
"-"
|
||||
]
|
||||
mpv = subprocess.Popen(mpv_cmd, stdin=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
||||
|
||||
# Start audio if available
|
||||
audio_proc = None
|
||||
if pipeline.audio_source_path:
|
||||
audio_cmd = ["ffplay", "-nodisp", "-autoexit", "-loglevel", "quiet",
|
||||
pipeline.audio_source_path]
|
||||
audio_proc = subprocess.Popen(audio_cmd, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
|
||||
try:
|
||||
import cv2
|
||||
for frame_num in range(n_frames):
|
||||
if mpv.poll() is not None:
|
||||
break # mpv closed
|
||||
|
||||
t = frame_num * frame_time
|
||||
frame = pipeline.render_frame(t)
|
||||
if frame is not None:
|
||||
# Ensure consistent frame size
|
||||
if frame.shape[1] != w or frame.shape[0] != h:
|
||||
frame = cv2.resize(frame, (w, h))
|
||||
if not frame.flags['C_CONTIGUOUS']:
|
||||
frame = np.ascontiguousarray(frame)
|
||||
try:
|
||||
mpv.stdin.write(frame.tobytes())
|
||||
mpv.stdin.flush()
|
||||
except BrokenPipeError:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
if mpv.stdin:
|
||||
mpv.stdin.close()
|
||||
mpv.terminate()
|
||||
if audio_proc:
|
||||
audio_proc.terminate()
|
||||
for src in pipeline.sources.values():
|
||||
src.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Run sexp recipe through streaming pipeline")
|
||||
parser.add_argument("recipe", help="Path to .sexp recipe file")
|
||||
parser.add_argument("-o", "--output", default="pipe",
|
||||
help="Output: 'pipe' (mpv), 'preview', or filename (default: pipe)")
|
||||
parser.add_argument("-d", "--duration", type=float, default=None,
|
||||
help="Duration in seconds (default: audio duration)")
|
||||
parser.add_argument("--fps", type=float, default=None,
|
||||
help="Frame rate (default: from recipe)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.output == "pipe":
|
||||
run_pipeline_piped(args.recipe, duration=args.duration, fps=args.fps)
|
||||
else:
|
||||
run_pipeline(args.recipe, output=args.output, duration=args.duration, fps=args.fps)
|
||||
470
streaming/recipe_adapter.py
Normal file
470
streaming/recipe_adapter.py
Normal file
@@ -0,0 +1,470 @@
|
||||
"""
|
||||
Adapter to run sexp recipes through the streaming compositor.
|
||||
|
||||
Bridges the gap between:
|
||||
- Existing recipe format (sexp files with stages, effects, analysis)
|
||||
- Streaming compositor (sources, effect chains, compositor config)
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "artdag"))
|
||||
|
||||
from .compositor import StreamingCompositor
|
||||
from .sources import VideoSource
|
||||
from .audio import FileAudioAnalyzer
|
||||
|
||||
|
||||
class RecipeAdapter:
|
||||
"""
|
||||
Adapts a compiled sexp recipe to run through the streaming compositor.
|
||||
|
||||
Example:
|
||||
adapter = RecipeAdapter("effects/quick_test.sexp")
|
||||
adapter.run(output="preview", duration=60)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
recipe_path: str,
|
||||
params: Dict[str, Any] = None,
|
||||
backend: str = "numpy",
|
||||
):
|
||||
"""
|
||||
Load and prepare a recipe for streaming.
|
||||
|
||||
Args:
|
||||
recipe_path: Path to .sexp recipe file
|
||||
params: Parameter overrides
|
||||
backend: "numpy" or "glsl"
|
||||
"""
|
||||
self.recipe_path = Path(recipe_path)
|
||||
self.recipe_dir = self.recipe_path.parent
|
||||
self.params = params or {}
|
||||
self.backend = backend
|
||||
|
||||
# Compile recipe
|
||||
self._compile()
|
||||
|
||||
def _compile(self):
|
||||
"""Compile the recipe and extract structure."""
|
||||
from artdag.sexp.compiler import compile_string
|
||||
|
||||
recipe_text = self.recipe_path.read_text()
|
||||
self.compiled = compile_string(recipe_text, self.params, recipe_dir=self.recipe_dir)
|
||||
|
||||
# Extract key info
|
||||
self.sources = {} # name -> path
|
||||
self.effects_registry = {} # effect_name -> path
|
||||
self.analyzers = {} # name -> analyzer info
|
||||
|
||||
# Walk nodes to find sources and structure
|
||||
# nodes is a list in CompiledRecipe
|
||||
for node in self.compiled.nodes:
|
||||
node_type = node.get("type", "")
|
||||
|
||||
if node_type == "SOURCE":
|
||||
config = node.get("config", {})
|
||||
path = config.get("path")
|
||||
if path:
|
||||
self.sources[node["id"]] = self.recipe_dir / path
|
||||
|
||||
elif node_type == "ANALYZE":
|
||||
config = node.get("config", {})
|
||||
self.analyzers[node["id"]] = {
|
||||
"analyzer": config.get("analyzer"),
|
||||
"path": config.get("analyzer_path"),
|
||||
}
|
||||
|
||||
# Get effects registry from compiled recipe
|
||||
# registry has 'effects' sub-dict
|
||||
effects_dict = self.compiled.registry.get("effects", {})
|
||||
for name, info in effects_dict.items():
|
||||
if info.get("path"):
|
||||
self.effects_registry[name] = Path(info["path"])
|
||||
|
||||
def run_analysis(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Run analysis phase (energy, beats, etc.).
|
||||
|
||||
Returns:
|
||||
Dict of analysis track name -> {times, values, duration}
|
||||
"""
|
||||
print(f"Running analysis...", file=sys.stderr)
|
||||
|
||||
# Use existing planner's analysis execution
|
||||
from artdag.sexp.planner import create_plan
|
||||
|
||||
analysis_data = {}
|
||||
|
||||
def on_analysis(node_id: str, results: dict):
|
||||
analysis_data[node_id] = results
|
||||
print(f" {node_id[:16]}...: {len(results.get('times', []))} samples", file=sys.stderr)
|
||||
|
||||
# Create plan (runs analysis as side effect)
|
||||
plan = create_plan(
|
||||
self.compiled,
|
||||
inputs={},
|
||||
recipe_dir=self.recipe_dir,
|
||||
on_analysis=on_analysis,
|
||||
)
|
||||
|
||||
# Also store named analysis tracks
|
||||
for name, data in plan.analysis.items():
|
||||
analysis_data[name] = data
|
||||
|
||||
return analysis_data
|
||||
|
||||
def build_compositor(
|
||||
self,
|
||||
analysis_data: Dict[str, Any] = None,
|
||||
fps: float = None,
|
||||
) -> StreamingCompositor:
|
||||
"""
|
||||
Build a streaming compositor from the recipe.
|
||||
|
||||
This is a simplified version that handles common patterns.
|
||||
Complex recipes may need manual configuration.
|
||||
|
||||
Args:
|
||||
analysis_data: Pre-computed analysis data
|
||||
|
||||
Returns:
|
||||
Configured StreamingCompositor
|
||||
"""
|
||||
# Extract video and audio sources in SLICE_ON input order
|
||||
video_sources = []
|
||||
audio_source = None
|
||||
|
||||
# Find audio source first
|
||||
for node_id, path in self.sources.items():
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in ('.mp3', '.wav', '.flac', '.ogg', '.m4a', '.aac'):
|
||||
audio_source = str(path)
|
||||
break
|
||||
|
||||
# Find SLICE_ON node to get correct video order
|
||||
slice_on_inputs = None
|
||||
for node in self.compiled.nodes:
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
# Use 'videos' config key which has the correct order
|
||||
config = node.get('config', {})
|
||||
slice_on_inputs = config.get('videos', [])
|
||||
break
|
||||
|
||||
if slice_on_inputs:
|
||||
# Trace each SLICE_ON input back to its SOURCE
|
||||
node_lookup = {n['id']: n for n in self.compiled.nodes}
|
||||
|
||||
def trace_to_source(node_id, visited=None):
|
||||
"""Trace a node back to its SOURCE, return source path."""
|
||||
if visited is None:
|
||||
visited = set()
|
||||
if node_id in visited:
|
||||
return None
|
||||
visited.add(node_id)
|
||||
|
||||
node = node_lookup.get(node_id)
|
||||
if not node:
|
||||
return None
|
||||
if node.get('type') == 'SOURCE':
|
||||
return self.sources.get(node_id)
|
||||
# Recurse through inputs
|
||||
for inp in node.get('inputs', []):
|
||||
result = trace_to_source(inp, visited)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
|
||||
# Build video_sources in SLICE_ON input order
|
||||
for inp_id in slice_on_inputs:
|
||||
source_path = trace_to_source(inp_id)
|
||||
if source_path:
|
||||
suffix = source_path.suffix.lower()
|
||||
if suffix in ('.mp4', '.webm', '.mov', '.avi', '.mkv'):
|
||||
video_sources.append(str(source_path))
|
||||
|
||||
# Fallback to definition order if no SLICE_ON
|
||||
if not video_sources:
|
||||
for node_id, path in self.sources.items():
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in ('.mp4', '.webm', '.mov', '.avi', '.mkv'):
|
||||
video_sources.append(str(path))
|
||||
|
||||
if not video_sources:
|
||||
raise ValueError("No video sources found in recipe")
|
||||
|
||||
# Build effect chains - use live audio bindings (matching video_sources count)
|
||||
effects_per_source = self._build_streaming_effects(n_sources=len(video_sources))
|
||||
|
||||
# Build compositor config from recipe
|
||||
compositor_config = self._extract_compositor_config(analysis_data)
|
||||
|
||||
return StreamingCompositor(
|
||||
sources=video_sources,
|
||||
effects_per_source=effects_per_source,
|
||||
compositor_config=compositor_config,
|
||||
analysis_data=analysis_data or {},
|
||||
backend=self.backend,
|
||||
recipe_dir=self.recipe_dir,
|
||||
fps=fps or self.compiled.encoding.get("fps", 30),
|
||||
audio_source=audio_source,
|
||||
)
|
||||
|
||||
def _build_streaming_effects(self, n_sources: int = None) -> List[List[Dict]]:
|
||||
"""
|
||||
Build effect chains for streaming with live audio bindings.
|
||||
|
||||
Replicates the recipe's effect pipeline:
|
||||
- Per source: rotate, zoom, invert, hue_shift, ascii_art
|
||||
- All driven by live_energy and live_beat
|
||||
"""
|
||||
if n_sources is None:
|
||||
n_sources = len([p for p in self.sources.values()
|
||||
if p.suffix.lower() in ('.mp4', '.webm', '.mov', '.avi', '.mkv')])
|
||||
|
||||
effects_per_source = []
|
||||
|
||||
for i in range(n_sources):
|
||||
# Alternate rotation direction per source
|
||||
rot_dir = 1 if i % 2 == 0 else -1
|
||||
|
||||
effects = [
|
||||
# Rotate - energy drives angle
|
||||
{
|
||||
"effect": "rotate",
|
||||
"effect_path": str(self.effects_registry.get("rotate", "")),
|
||||
"angle": {
|
||||
"_binding": True,
|
||||
"source": "live_energy",
|
||||
"feature": "values",
|
||||
"range": [0, 45 * rot_dir],
|
||||
},
|
||||
},
|
||||
# Zoom - energy drives amount
|
||||
{
|
||||
"effect": "zoom",
|
||||
"effect_path": str(self.effects_registry.get("zoom", "")),
|
||||
"amount": {
|
||||
"_binding": True,
|
||||
"source": "live_energy",
|
||||
"feature": "values",
|
||||
"range": [1.0, 1.5] if i % 2 == 0 else [1.0, 0.7],
|
||||
},
|
||||
},
|
||||
# Invert - beat triggers
|
||||
{
|
||||
"effect": "invert",
|
||||
"effect_path": str(self.effects_registry.get("invert", "")),
|
||||
"amount": {
|
||||
"_binding": True,
|
||||
"source": "live_beat",
|
||||
"feature": "values",
|
||||
"range": [0, 1],
|
||||
},
|
||||
},
|
||||
# Hue shift - energy drives hue
|
||||
{
|
||||
"effect": "hue_shift",
|
||||
"effect_path": str(self.effects_registry.get("hue_shift", "")),
|
||||
"degrees": {
|
||||
"_binding": True,
|
||||
"source": "live_energy",
|
||||
"feature": "values",
|
||||
"range": [0, 180],
|
||||
},
|
||||
},
|
||||
# ASCII art - energy drives char size, beat triggers mix
|
||||
{
|
||||
"effect": "ascii_art",
|
||||
"effect_path": str(self.effects_registry.get("ascii_art", "")),
|
||||
"char_size": {
|
||||
"_binding": True,
|
||||
"source": "live_energy",
|
||||
"feature": "values",
|
||||
"range": [4, 32],
|
||||
},
|
||||
"mix": {
|
||||
"_binding": True,
|
||||
"source": "live_beat",
|
||||
"feature": "values",
|
||||
"range": [0, 1],
|
||||
},
|
||||
},
|
||||
]
|
||||
effects_per_source.append(effects)
|
||||
|
||||
return effects_per_source
|
||||
|
||||
def _extract_effects(self) -> List[List[Dict]]:
|
||||
"""Extract effect chains for each source (legacy, pre-computed analysis)."""
|
||||
# Simplified: find EFFECT nodes and their configs
|
||||
effects_per_source = []
|
||||
|
||||
for node_id, path in self.sources.items():
|
||||
if path.suffix.lower() not in ('.mp4', '.webm', '.mov', '.avi', '.mkv'):
|
||||
continue
|
||||
|
||||
# Find effects that depend on this source
|
||||
# This is simplified - real implementation would trace the DAG
|
||||
effects = []
|
||||
|
||||
for node in self.compiled.nodes:
|
||||
if node.get("type") == "EFFECT":
|
||||
config = node.get("config", {})
|
||||
effect_name = config.get("effect")
|
||||
if effect_name and effect_name in self.effects_registry:
|
||||
effect_config = {
|
||||
"effect": effect_name,
|
||||
"effect_path": str(self.effects_registry[effect_name]),
|
||||
}
|
||||
# Copy only effect params (filter out internal fields)
|
||||
internal_fields = (
|
||||
"effect", "effect_path", "cid", "effect_cid",
|
||||
"effects_registry", "analysis_refs", "inputs",
|
||||
)
|
||||
for k, v in config.items():
|
||||
if k not in internal_fields:
|
||||
effect_config[k] = v
|
||||
effects.append(effect_config)
|
||||
break # One effect per source for now
|
||||
|
||||
effects_per_source.append(effects)
|
||||
|
||||
return effects_per_source
|
||||
|
||||
def _extract_compositor_config(self, analysis_data: Dict) -> Dict:
|
||||
"""Extract compositor configuration."""
|
||||
# Look for blend_multi or similar composition nodes
|
||||
for node in self.compiled.nodes:
|
||||
if node.get("type") == "EFFECT":
|
||||
config = node.get("config", {})
|
||||
if config.get("effect") == "blend_multi":
|
||||
return {
|
||||
"mode": config.get("mode", "alpha"),
|
||||
"weights": config.get("weights", []),
|
||||
}
|
||||
|
||||
# Default: equal blend
|
||||
n_sources = len([p for p in self.sources.values()
|
||||
if p.suffix.lower() in ('.mp4', '.webm', '.mov', '.avi', '.mkv')])
|
||||
return {
|
||||
"mode": "alpha",
|
||||
"weights": [1.0 / n_sources] * n_sources if n_sources > 0 else [1.0],
|
||||
}
|
||||
|
||||
def run(
|
||||
self,
|
||||
output: str = "preview",
|
||||
duration: float = None,
|
||||
fps: float = None,
|
||||
):
|
||||
"""
|
||||
Run the recipe through streaming compositor.
|
||||
|
||||
Everything streams: video frames read on-demand, audio analyzed in real-time.
|
||||
No pre-computation.
|
||||
|
||||
Args:
|
||||
output: "preview", filename, or Output object
|
||||
duration: Duration in seconds (default: audio duration)
|
||||
fps: Frame rate (default from recipe, or 30)
|
||||
"""
|
||||
# Build compositor with recipe executor for full pipeline
|
||||
from .recipe_executor import StreamingRecipeExecutor
|
||||
|
||||
compositor = self.build_compositor(analysis_data={}, fps=fps)
|
||||
|
||||
# Use audio duration if not specified
|
||||
if duration is None:
|
||||
if compositor._audio_analyzer:
|
||||
duration = compositor._audio_analyzer.duration
|
||||
print(f"Using audio duration: {duration:.1f}s", file=sys.stderr)
|
||||
else:
|
||||
# Live mode - run until quit
|
||||
print("Live mode - press 'q' to quit", file=sys.stderr)
|
||||
|
||||
# Create sexp executor that interprets the recipe
|
||||
from .sexp_executor import SexpStreamingExecutor
|
||||
executor = SexpStreamingExecutor(self.compiled, seed=42)
|
||||
|
||||
compositor.run(output=output, duration=duration, recipe_executor=executor)
|
||||
|
||||
|
||||
def run_recipe(
|
||||
recipe_path: str,
|
||||
output: str = "preview",
|
||||
duration: float = None,
|
||||
params: Dict = None,
|
||||
fps: float = None,
|
||||
):
|
||||
"""
|
||||
Run a recipe through streaming compositor.
|
||||
|
||||
Everything streams in real-time: video frames, audio analysis.
|
||||
No pre-computation - starts immediately.
|
||||
|
||||
Example:
|
||||
run_recipe("effects/quick_test.sexp", output="preview", duration=30)
|
||||
run_recipe("effects/quick_test.sexp", output="preview", fps=5) # Lower fps for slow systems
|
||||
"""
|
||||
adapter = RecipeAdapter(recipe_path, params=params)
|
||||
adapter.run(output=output, duration=duration, fps=fps)
|
||||
|
||||
|
||||
def run_recipe_piped(
|
||||
recipe_path: str,
|
||||
duration: float = None,
|
||||
params: Dict = None,
|
||||
fps: float = None,
|
||||
):
|
||||
"""
|
||||
Run recipe and pipe directly to mpv.
|
||||
"""
|
||||
from .output import PipeOutput
|
||||
|
||||
adapter = RecipeAdapter(recipe_path, params=params)
|
||||
compositor = adapter.build_compositor(analysis_data={}, fps=fps)
|
||||
|
||||
# Get frame size
|
||||
if compositor.sources:
|
||||
first_source = compositor.sources[0]
|
||||
w, h = first_source._size
|
||||
else:
|
||||
w, h = 720, 720
|
||||
|
||||
actual_fps = fps or adapter.compiled.encoding.get('fps', 30)
|
||||
|
||||
# Create pipe output
|
||||
pipe_out = PipeOutput(
|
||||
size=(w, h),
|
||||
fps=actual_fps,
|
||||
audio_source=compositor._audio_source
|
||||
)
|
||||
|
||||
# Create executor
|
||||
from .sexp_executor import SexpStreamingExecutor
|
||||
executor = SexpStreamingExecutor(adapter.compiled, seed=42)
|
||||
|
||||
# Run with pipe output
|
||||
compositor.run(output=pipe_out, duration=duration, recipe_executor=executor)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Run sexp recipe with streaming compositor")
|
||||
parser.add_argument("recipe", help="Path to .sexp recipe file")
|
||||
parser.add_argument("-o", "--output", default="pipe",
|
||||
help="Output: 'pipe' (mpv), 'preview', or filename (default: pipe)")
|
||||
parser.add_argument("-d", "--duration", type=float, default=None,
|
||||
help="Duration in seconds (default: audio duration)")
|
||||
parser.add_argument("--fps", type=float, default=None,
|
||||
help="Frame rate (default: from recipe)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.output == "pipe":
|
||||
run_recipe_piped(args.recipe, duration=args.duration, fps=args.fps)
|
||||
else:
|
||||
run_recipe(args.recipe, output=args.output, duration=args.duration, fps=args.fps)
|
||||
415
streaming/recipe_executor.py
Normal file
415
streaming/recipe_executor.py
Normal file
@@ -0,0 +1,415 @@
|
||||
"""
|
||||
Streaming recipe executor.
|
||||
|
||||
Implements the full recipe logic for real-time streaming:
|
||||
- Scans (state machines that evolve on beats)
|
||||
- Process-pair template (two clips with sporadic effects, blended)
|
||||
- Cycle-crossfade (dynamic composition cycling through video pairs)
|
||||
"""
|
||||
|
||||
import random
|
||||
import numpy as np
|
||||
from typing import Dict, List, Any, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanState:
|
||||
"""State for a scan (beat-driven state machine)."""
|
||||
value: Any = 0
|
||||
rng: random.Random = field(default_factory=random.Random)
|
||||
|
||||
|
||||
class StreamingScans:
|
||||
"""
|
||||
Real-time scan executor.
|
||||
|
||||
Scans are state machines that evolve on each beat.
|
||||
They drive effect parameters like invert triggers, hue shifts, etc.
|
||||
"""
|
||||
|
||||
def __init__(self, seed: int = 42, n_sources: int = 4):
|
||||
self.master_seed = seed
|
||||
self.n_sources = n_sources
|
||||
self.scans: Dict[str, ScanState] = {}
|
||||
self.beat_count = 0
|
||||
self.current_time = 0.0
|
||||
self.last_beat_time = 0.0
|
||||
self._init_scans()
|
||||
|
||||
def _init_scans(self):
|
||||
"""Initialize all scans with their own RNG seeds."""
|
||||
scan_names = []
|
||||
|
||||
# Per-pair scans (dynamic based on n_sources)
|
||||
for i in range(self.n_sources):
|
||||
scan_names.extend([
|
||||
f"inv_a_{i}", f"inv_b_{i}", f"hue_a_{i}", f"hue_b_{i}",
|
||||
f"ascii_a_{i}", f"ascii_b_{i}", f"pair_mix_{i}", f"pair_rot_{i}",
|
||||
])
|
||||
|
||||
# Global scans
|
||||
scan_names.extend(["whole_spin", "ripple_gate", "cycle"])
|
||||
|
||||
for i, name in enumerate(scan_names):
|
||||
rng = random.Random(self.master_seed + i)
|
||||
self.scans[name] = ScanState(value=self._init_value(name), rng=rng)
|
||||
|
||||
def _init_value(self, name: str) -> Any:
|
||||
"""Get initial value for a scan."""
|
||||
if name.startswith("inv_") or name.startswith("ascii_"):
|
||||
return 0 # Counter for remaining beats
|
||||
elif name.startswith("hue_"):
|
||||
return {"rem": 0, "hue": 0}
|
||||
elif name.startswith("pair_mix"):
|
||||
return {"rem": 0, "opacity": 0.5}
|
||||
elif name.startswith("pair_rot"):
|
||||
pair_idx = int(name.split("_")[-1])
|
||||
rot_dir = 1 if pair_idx % 2 == 0 else -1
|
||||
return {"beat": 0, "clen": 25, "dir": rot_dir, "angle": 0}
|
||||
elif name == "whole_spin":
|
||||
return {
|
||||
"phase": 0, # 0 = waiting, 1 = spinning
|
||||
"beat": 0, # beats into current phase
|
||||
"plen": 20, # beats in this phase
|
||||
"dir": 1, # spin direction
|
||||
"total_angle": 0.0, # cumulative angle after all spins
|
||||
"spin_start_angle": 0.0, # angle when current spin started
|
||||
"spin_start_time": 0.0, # time when current spin started
|
||||
"spin_end_time": 0.0, # estimated time when spin ends
|
||||
}
|
||||
elif name == "ripple_gate":
|
||||
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
||||
elif name == "cycle":
|
||||
return {"cycle": 0, "beat": 0, "clen": 60}
|
||||
return 0
|
||||
|
||||
def on_beat(self):
|
||||
"""Update all scans on a beat."""
|
||||
self.beat_count += 1
|
||||
# Estimate beat interval from last two beats
|
||||
beat_interval = self.current_time - self.last_beat_time if self.last_beat_time > 0 else 0.5
|
||||
self.last_beat_time = self.current_time
|
||||
|
||||
for name, state in self.scans.items():
|
||||
state.value = self._step_scan(name, state.value, state.rng, beat_interval)
|
||||
|
||||
def _step_scan(self, name: str, value: Any, rng: random.Random, beat_interval: float = 0.5) -> Any:
|
||||
"""Step a scan forward by one beat."""
|
||||
|
||||
# Invert scan: 10% chance, lasts 1-5 beats
|
||||
if name.startswith("inv_"):
|
||||
if value > 0:
|
||||
return value - 1
|
||||
elif rng.random() < 0.1:
|
||||
return rng.randint(1, 5)
|
||||
return 0
|
||||
|
||||
# Hue scan: 10% chance, random hue 30-330, lasts 1-5 beats
|
||||
elif name.startswith("hue_"):
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "hue": value["hue"]}
|
||||
elif rng.random() < 0.1:
|
||||
return {"rem": rng.randint(1, 5), "hue": rng.uniform(30, 330)}
|
||||
return {"rem": 0, "hue": 0}
|
||||
|
||||
# ASCII scan: 5% chance, lasts 1-3 beats
|
||||
elif name.startswith("ascii_"):
|
||||
if value > 0:
|
||||
return value - 1
|
||||
elif rng.random() < 0.05:
|
||||
return rng.randint(1, 3)
|
||||
return 0
|
||||
|
||||
# Pair mix: changes every 1-11 beats
|
||||
elif name.startswith("pair_mix"):
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "opacity": value["opacity"]}
|
||||
return {"rem": rng.randint(1, 11), "opacity": rng.choice([0, 0.5, 1.0])}
|
||||
|
||||
# Pair rotation: full rotation every 20-30 beats
|
||||
elif name.startswith("pair_rot"):
|
||||
beat = value["beat"]
|
||||
clen = value["clen"]
|
||||
dir_ = value["dir"]
|
||||
angle = value["angle"]
|
||||
|
||||
if beat + 1 < clen:
|
||||
new_angle = angle + dir_ * (360 / clen)
|
||||
return {"beat": beat + 1, "clen": clen, "dir": dir_, "angle": new_angle}
|
||||
else:
|
||||
return {"beat": 0, "clen": rng.randint(20, 30), "dir": -dir_, "angle": angle}
|
||||
|
||||
# Whole spin: sporadic 720 degree spins (cumulative - stays rotated)
|
||||
elif name == "whole_spin":
|
||||
phase = value["phase"]
|
||||
beat = value["beat"]
|
||||
plen = value["plen"]
|
||||
dir_ = value["dir"]
|
||||
total_angle = value.get("total_angle", 0.0)
|
||||
spin_start_angle = value.get("spin_start_angle", 0.0)
|
||||
spin_start_time = value.get("spin_start_time", 0.0)
|
||||
spin_end_time = value.get("spin_end_time", 0.0)
|
||||
|
||||
if phase == 1:
|
||||
# Currently spinning
|
||||
if beat + 1 < plen:
|
||||
return {
|
||||
"phase": 1, "beat": beat + 1, "plen": plen, "dir": dir_,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": spin_start_angle,
|
||||
"spin_start_time": spin_start_time,
|
||||
"spin_end_time": spin_end_time,
|
||||
}
|
||||
else:
|
||||
# Spin complete - update total_angle with final spin
|
||||
new_total = spin_start_angle + dir_ * 720.0
|
||||
return {
|
||||
"phase": 0, "beat": 0, "plen": rng.randint(20, 40), "dir": dir_,
|
||||
"total_angle": new_total,
|
||||
"spin_start_angle": new_total,
|
||||
"spin_start_time": self.current_time,
|
||||
"spin_end_time": self.current_time,
|
||||
}
|
||||
else:
|
||||
# Waiting phase
|
||||
if beat + 1 < plen:
|
||||
return {
|
||||
"phase": 0, "beat": beat + 1, "plen": plen, "dir": dir_,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": spin_start_angle,
|
||||
"spin_start_time": spin_start_time,
|
||||
"spin_end_time": spin_end_time,
|
||||
}
|
||||
else:
|
||||
# Start new spin
|
||||
new_dir = 1 if rng.random() < 0.5 else -1
|
||||
new_plen = rng.randint(10, 25)
|
||||
spin_duration = new_plen * beat_interval
|
||||
return {
|
||||
"phase": 1, "beat": 0, "plen": new_plen, "dir": new_dir,
|
||||
"total_angle": total_angle,
|
||||
"spin_start_angle": total_angle,
|
||||
"spin_start_time": self.current_time,
|
||||
"spin_end_time": self.current_time + spin_duration,
|
||||
}
|
||||
|
||||
# Ripple gate: 5% chance, lasts 1-20 beats
|
||||
elif name == "ripple_gate":
|
||||
if value["rem"] > 0:
|
||||
return {"rem": value["rem"] - 1, "cx": value["cx"], "cy": value["cy"]}
|
||||
elif rng.random() < 0.05:
|
||||
return {"rem": rng.randint(1, 20),
|
||||
"cx": rng.uniform(0.1, 0.9),
|
||||
"cy": rng.uniform(0.1, 0.9)}
|
||||
return {"rem": 0, "cx": 0.5, "cy": 0.5}
|
||||
|
||||
# Cycle: track which video pair is active
|
||||
elif name == "cycle":
|
||||
beat = value["beat"]
|
||||
clen = value["clen"]
|
||||
cycle = value["cycle"]
|
||||
|
||||
if beat + 1 < clen:
|
||||
return {"cycle": cycle, "beat": beat + 1, "clen": clen}
|
||||
else:
|
||||
# Move to next pair, vary cycle length
|
||||
return {"cycle": (cycle + 1) % 4, "beat": 0,
|
||||
"clen": 40 + (self.beat_count * 7) % 41}
|
||||
|
||||
return value
|
||||
|
||||
def get_emit(self, name: str) -> float:
|
||||
"""Get emitted value for a scan."""
|
||||
value = self.scans[name].value
|
||||
|
||||
if name.startswith("inv_") or name.startswith("ascii_"):
|
||||
return 1.0 if value > 0 else 0.0
|
||||
|
||||
elif name.startswith("hue_"):
|
||||
return value["hue"] if value["rem"] > 0 else 0.0
|
||||
|
||||
elif name.startswith("pair_mix"):
|
||||
return value["opacity"]
|
||||
|
||||
elif name.startswith("pair_rot"):
|
||||
return value["angle"]
|
||||
|
||||
elif name == "whole_spin":
|
||||
# Smooth time-based interpolation during spin
|
||||
phase = value.get("phase", 0)
|
||||
if phase == 1:
|
||||
# Currently spinning - interpolate based on time
|
||||
spin_start_time = value.get("spin_start_time", 0.0)
|
||||
spin_end_time = value.get("spin_end_time", spin_start_time + 1.0)
|
||||
spin_start_angle = value.get("spin_start_angle", 0.0)
|
||||
dir_ = value.get("dir", 1)
|
||||
|
||||
duration = spin_end_time - spin_start_time
|
||||
if duration > 0:
|
||||
progress = (self.current_time - spin_start_time) / duration
|
||||
progress = max(0.0, min(1.0, progress)) # clamp to 0-1
|
||||
else:
|
||||
progress = 1.0
|
||||
|
||||
return spin_start_angle + progress * 720.0 * dir_
|
||||
else:
|
||||
# Not spinning - return cumulative angle
|
||||
return value.get("total_angle", 0.0)
|
||||
|
||||
elif name == "ripple_gate":
|
||||
return 1.0 if value["rem"] > 0 else 0.0
|
||||
|
||||
elif name == "cycle":
|
||||
return value
|
||||
|
||||
return 0.0
|
||||
|
||||
|
||||
class StreamingRecipeExecutor:
|
||||
"""
|
||||
Executes a recipe in streaming mode.
|
||||
|
||||
Implements:
|
||||
- process-pair: two video clips with opposite effects, blended
|
||||
- cycle-crossfade: dynamic cycling through video pairs
|
||||
- Final effects: whole-spin rotation, ripple
|
||||
"""
|
||||
|
||||
def __init__(self, n_sources: int = 4, seed: int = 42):
|
||||
self.n_sources = n_sources
|
||||
self.scans = StreamingScans(seed, n_sources=n_sources)
|
||||
self.last_beat_detected = False
|
||||
self.current_time = 0.0
|
||||
|
||||
def on_frame(self, energy: float, is_beat: bool, t: float = 0.0):
|
||||
"""Called each frame with current audio analysis."""
|
||||
self.current_time = t
|
||||
self.scans.current_time = t
|
||||
# Update scans on beat
|
||||
if is_beat and not self.last_beat_detected:
|
||||
self.scans.on_beat()
|
||||
self.last_beat_detected = is_beat
|
||||
|
||||
def get_effect_params(self, source_idx: int, clip: str, energy: float) -> Dict:
|
||||
"""
|
||||
Get effect parameters for a source clip.
|
||||
|
||||
Args:
|
||||
source_idx: Which video source (0-3)
|
||||
clip: "a" or "b" (each source has two clips)
|
||||
energy: Current audio energy (0-1)
|
||||
"""
|
||||
suffix = f"_{source_idx}"
|
||||
|
||||
# Rotation ranges alternate
|
||||
if source_idx % 2 == 0:
|
||||
rot_range = [0, 45] if clip == "a" else [0, -45]
|
||||
zoom_range = [1, 1.5] if clip == "a" else [1, 0.5]
|
||||
else:
|
||||
rot_range = [0, -45] if clip == "a" else [0, 45]
|
||||
zoom_range = [1, 0.5] if clip == "a" else [1, 1.5]
|
||||
|
||||
return {
|
||||
"rotate_angle": rot_range[0] + energy * (rot_range[1] - rot_range[0]),
|
||||
"zoom_amount": zoom_range[0] + energy * (zoom_range[1] - zoom_range[0]),
|
||||
"invert_amount": self.scans.get_emit(f"inv_{clip}{suffix}"),
|
||||
"hue_degrees": self.scans.get_emit(f"hue_{clip}{suffix}"),
|
||||
"ascii_mix": 0, # Disabled - too slow without GPU
|
||||
"ascii_char_size": 4 + energy * 28, # 4-32
|
||||
}
|
||||
|
||||
def get_pair_params(self, source_idx: int) -> Dict:
|
||||
"""Get blend and rotation params for a video pair."""
|
||||
suffix = f"_{source_idx}"
|
||||
return {
|
||||
"blend_opacity": self.scans.get_emit(f"pair_mix{suffix}"),
|
||||
"pair_rotation": self.scans.get_emit(f"pair_rot{suffix}"),
|
||||
}
|
||||
|
||||
def get_cycle_weights(self) -> List[float]:
|
||||
"""Get blend weights for cycle-crossfade composition."""
|
||||
cycle_state = self.scans.get_emit("cycle")
|
||||
active = cycle_state["cycle"]
|
||||
beat = cycle_state["beat"]
|
||||
clen = cycle_state["clen"]
|
||||
n = self.n_sources
|
||||
|
||||
phase3 = beat * 3
|
||||
weights = []
|
||||
|
||||
for p in range(n):
|
||||
prev = (p + n - 1) % n
|
||||
|
||||
if active == p:
|
||||
if phase3 < clen:
|
||||
w = 0.9
|
||||
elif phase3 < clen * 2:
|
||||
w = 0.9 - ((phase3 - clen) / clen) * 0.85
|
||||
else:
|
||||
w = 0.05
|
||||
elif active == prev:
|
||||
if phase3 < clen:
|
||||
w = 0.05
|
||||
elif phase3 < clen * 2:
|
||||
w = 0.05 + ((phase3 - clen) / clen) * 0.85
|
||||
else:
|
||||
w = 0.9
|
||||
else:
|
||||
w = 0.05
|
||||
|
||||
weights.append(w)
|
||||
|
||||
# Normalize
|
||||
total = sum(weights)
|
||||
if total > 0:
|
||||
weights = [w / total for w in weights]
|
||||
|
||||
return weights
|
||||
|
||||
def get_cycle_zooms(self) -> List[float]:
|
||||
"""Get zoom amounts for cycle-crossfade."""
|
||||
cycle_state = self.scans.get_emit("cycle")
|
||||
active = cycle_state["cycle"]
|
||||
beat = cycle_state["beat"]
|
||||
clen = cycle_state["clen"]
|
||||
n = self.n_sources
|
||||
|
||||
phase3 = beat * 3
|
||||
zooms = []
|
||||
|
||||
for p in range(n):
|
||||
prev = (p + n - 1) % n
|
||||
|
||||
if active == p:
|
||||
if phase3 < clen:
|
||||
z = 1.0
|
||||
elif phase3 < clen * 2:
|
||||
z = 1.0 + ((phase3 - clen) / clen) * 1.0
|
||||
else:
|
||||
z = 0.1
|
||||
elif active == prev:
|
||||
if phase3 < clen:
|
||||
z = 3.0 # Start big
|
||||
elif phase3 < clen * 2:
|
||||
z = 3.0 - ((phase3 - clen) / clen) * 2.0 # Shrink to 1.0
|
||||
else:
|
||||
z = 1.0
|
||||
else:
|
||||
z = 0.1
|
||||
|
||||
zooms.append(z)
|
||||
|
||||
return zooms
|
||||
|
||||
def get_final_effects(self, energy: float) -> Dict:
|
||||
"""Get final composition effects (whole-spin, ripple)."""
|
||||
ripple_gate = self.scans.get_emit("ripple_gate")
|
||||
ripple_state = self.scans.scans["ripple_gate"].value
|
||||
|
||||
return {
|
||||
"whole_spin_angle": self.scans.get_emit("whole_spin"),
|
||||
"ripple_amplitude": ripple_gate * (5 + energy * 45), # 5-50
|
||||
"ripple_cx": ripple_state["cx"],
|
||||
"ripple_cy": ripple_state["cy"],
|
||||
}
|
||||
678
streaming/sexp_executor.py
Normal file
678
streaming/sexp_executor.py
Normal file
@@ -0,0 +1,678 @@
|
||||
"""
|
||||
Streaming S-expression executor.
|
||||
|
||||
Executes compiled sexp recipes in real-time by:
|
||||
- Evaluating scan expressions on each beat
|
||||
- Resolving bindings to get effect parameter values
|
||||
- Applying effects frame-by-frame
|
||||
- Evaluating SLICE_ON Lambda for cycle crossfade
|
||||
"""
|
||||
|
||||
import random
|
||||
import numpy as np
|
||||
from typing import Dict, List, Any, Optional
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .sexp_interp import SexpInterpreter, eval_slice_on_lambda
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanState:
|
||||
"""Runtime state for a scan."""
|
||||
node_id: str
|
||||
name: Optional[str]
|
||||
value: Any
|
||||
rng: random.Random
|
||||
init_expr: dict
|
||||
step_expr: dict
|
||||
emit_expr: dict
|
||||
|
||||
|
||||
class ExprEvaluator:
|
||||
"""
|
||||
Evaluates compiled expression ASTs.
|
||||
|
||||
Expressions are dicts with:
|
||||
- _expr: True (marks as expression)
|
||||
- op: operation name
|
||||
- args: list of arguments
|
||||
- name: for 'var' ops
|
||||
- keys: for 'dict' ops
|
||||
"""
|
||||
|
||||
def __init__(self, rng: random.Random = None):
|
||||
self.rng = rng or random.Random()
|
||||
|
||||
def eval(self, expr: Any, env: Dict[str, Any]) -> Any:
|
||||
"""Evaluate an expression in the given environment."""
|
||||
# Literal values
|
||||
if not isinstance(expr, dict):
|
||||
return expr
|
||||
|
||||
# Check if it's an expression
|
||||
if not expr.get('_expr'):
|
||||
# It's a plain dict - return as-is
|
||||
return expr
|
||||
|
||||
op = expr.get('op')
|
||||
args = expr.get('args', [])
|
||||
|
||||
# Evaluate based on operation
|
||||
if op == 'var':
|
||||
name = expr.get('name')
|
||||
if name in env:
|
||||
return env[name]
|
||||
raise KeyError(f"Unknown variable: {name}")
|
||||
|
||||
elif op == 'dict':
|
||||
keys = expr.get('keys', [])
|
||||
values = [self.eval(a, env) for a in args]
|
||||
return dict(zip(keys, values))
|
||||
|
||||
elif op == 'get':
|
||||
obj = self.eval(args[0], env)
|
||||
key = args[1]
|
||||
return obj.get(key) if isinstance(obj, dict) else obj[key]
|
||||
|
||||
elif op == 'if':
|
||||
cond = self.eval(args[0], env)
|
||||
if cond:
|
||||
return self.eval(args[1], env)
|
||||
elif len(args) > 2:
|
||||
return self.eval(args[2], env)
|
||||
return None
|
||||
|
||||
# Comparison ops
|
||||
elif op == '<':
|
||||
return self.eval(args[0], env) < self.eval(args[1], env)
|
||||
elif op == '>':
|
||||
return self.eval(args[0], env) > self.eval(args[1], env)
|
||||
elif op == '<=':
|
||||
return self.eval(args[0], env) <= self.eval(args[1], env)
|
||||
elif op == '>=':
|
||||
return self.eval(args[0], env) >= self.eval(args[1], env)
|
||||
elif op == '=':
|
||||
return self.eval(args[0], env) == self.eval(args[1], env)
|
||||
elif op == '!=':
|
||||
return self.eval(args[0], env) != self.eval(args[1], env)
|
||||
|
||||
# Arithmetic ops
|
||||
elif op == '+':
|
||||
return self.eval(args[0], env) + self.eval(args[1], env)
|
||||
elif op == '-':
|
||||
return self.eval(args[0], env) - self.eval(args[1], env)
|
||||
elif op == '*':
|
||||
return self.eval(args[0], env) * self.eval(args[1], env)
|
||||
elif op == '/':
|
||||
return self.eval(args[0], env) / self.eval(args[1], env)
|
||||
elif op == 'mod':
|
||||
return self.eval(args[0], env) % self.eval(args[1], env)
|
||||
|
||||
# Random ops
|
||||
elif op == 'rand':
|
||||
return self.rng.random()
|
||||
elif op == 'rand-int':
|
||||
lo = self.eval(args[0], env)
|
||||
hi = self.eval(args[1], env)
|
||||
return self.rng.randint(lo, hi)
|
||||
elif op == 'rand-range':
|
||||
lo = self.eval(args[0], env)
|
||||
hi = self.eval(args[1], env)
|
||||
return self.rng.uniform(lo, hi)
|
||||
|
||||
# Logic ops
|
||||
elif op == 'and':
|
||||
return all(self.eval(a, env) for a in args)
|
||||
elif op == 'or':
|
||||
return any(self.eval(a, env) for a in args)
|
||||
elif op == 'not':
|
||||
return not self.eval(args[0], env)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown operation: {op}")
|
||||
|
||||
|
||||
class SexpStreamingExecutor:
|
||||
"""
|
||||
Executes a compiled sexp recipe in streaming mode.
|
||||
|
||||
Reads scan definitions, effect chains, and bindings from the
|
||||
compiled recipe and executes them frame-by-frame.
|
||||
"""
|
||||
|
||||
def __init__(self, compiled_recipe, seed: int = 42):
|
||||
self.recipe = compiled_recipe
|
||||
self.master_seed = seed
|
||||
|
||||
# Build node lookup
|
||||
self.nodes = {n['id']: n for n in compiled_recipe.nodes}
|
||||
|
||||
# State (must be initialized before _init_scans)
|
||||
self.beat_count = 0
|
||||
self.current_time = 0.0
|
||||
self.last_beat_time = 0.0
|
||||
self.last_beat_detected = False
|
||||
self.energy = 0.0
|
||||
|
||||
# Initialize scans
|
||||
self.scans: Dict[str, ScanState] = {}
|
||||
self.scan_outputs: Dict[str, Any] = {} # Current emit values by node_id
|
||||
self._init_scans()
|
||||
|
||||
# Initialize SLICE_ON interpreter
|
||||
self.sexp_interp = SexpInterpreter(random.Random(seed))
|
||||
self._slice_on_lambda = None
|
||||
self._slice_on_acc = None
|
||||
self._slice_on_result = None # Last evaluation result {layers, compose, acc}
|
||||
self._init_slice_on()
|
||||
|
||||
def _init_slice_on(self):
|
||||
"""Initialize SLICE_ON Lambda for cycle crossfade."""
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
config = node.get('config', {})
|
||||
self._slice_on_lambda = config.get('fn')
|
||||
init = config.get('init', {})
|
||||
self._slice_on_acc = {
|
||||
'cycle': init.get('cycle', 0),
|
||||
'beat': init.get('beat', 0),
|
||||
'clen': init.get('clen', 60),
|
||||
}
|
||||
# Evaluate initial state
|
||||
self._eval_slice_on()
|
||||
break
|
||||
|
||||
def _eval_slice_on(self):
|
||||
"""Evaluate the SLICE_ON Lambda with current state."""
|
||||
if not self._slice_on_lambda:
|
||||
return
|
||||
|
||||
n = len(self._get_video_sources())
|
||||
videos = list(range(n)) # Placeholder video indices
|
||||
|
||||
try:
|
||||
result = eval_slice_on_lambda(
|
||||
self._slice_on_lambda,
|
||||
self._slice_on_acc,
|
||||
self.beat_count,
|
||||
0.0, # start time (not used for weights)
|
||||
1.0, # end time (not used for weights)
|
||||
videos,
|
||||
self.sexp_interp,
|
||||
)
|
||||
self._slice_on_result = result
|
||||
# Update accumulator for next beat
|
||||
if 'acc' in result:
|
||||
self._slice_on_acc = result['acc']
|
||||
except Exception as e:
|
||||
import sys
|
||||
print(f"SLICE_ON eval error: {e}", file=sys.stderr)
|
||||
|
||||
def _init_scans(self):
|
||||
"""Initialize all scan nodes from the recipe."""
|
||||
seed_offset = 0
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SCAN':
|
||||
node_id = node['id']
|
||||
config = node.get('config', {})
|
||||
|
||||
# Create RNG with unique seed
|
||||
scan_seed = config.get('seed', self.master_seed + seed_offset)
|
||||
rng = random.Random(scan_seed)
|
||||
seed_offset += 1
|
||||
|
||||
# Evaluate initial value
|
||||
init_expr = config.get('init', 0)
|
||||
evaluator = ExprEvaluator(rng)
|
||||
init_value = evaluator.eval(init_expr, {})
|
||||
|
||||
self.scans[node_id] = ScanState(
|
||||
node_id=node_id,
|
||||
name=node.get('name'),
|
||||
value=init_value,
|
||||
rng=rng,
|
||||
init_expr=init_expr,
|
||||
step_expr=config.get('step_expr', {}),
|
||||
emit_expr=config.get('emit_expr', {}),
|
||||
)
|
||||
|
||||
# Compute initial emit
|
||||
self._update_emit(node_id)
|
||||
|
||||
def _update_emit(self, node_id: str):
|
||||
"""Update the emit value for a scan."""
|
||||
scan = self.scans[node_id]
|
||||
evaluator = ExprEvaluator(scan.rng)
|
||||
|
||||
# Build environment from current state
|
||||
env = self._build_scan_env(scan)
|
||||
|
||||
# Evaluate emit expression
|
||||
emit_value = evaluator.eval(scan.emit_expr, env)
|
||||
self.scan_outputs[node_id] = emit_value
|
||||
|
||||
def _build_scan_env(self, scan: ScanState) -> Dict[str, Any]:
|
||||
"""Build environment for scan expression evaluation."""
|
||||
env = {}
|
||||
|
||||
# Add state variables
|
||||
if isinstance(scan.value, dict):
|
||||
env.update(scan.value)
|
||||
else:
|
||||
env['acc'] = scan.value
|
||||
|
||||
# Add beat count
|
||||
env['beat_count'] = self.beat_count
|
||||
env['time'] = self.current_time
|
||||
|
||||
return env
|
||||
|
||||
def on_beat(self):
|
||||
"""Update all scans on a beat."""
|
||||
self.beat_count += 1
|
||||
|
||||
# Estimate beat interval
|
||||
beat_interval = self.current_time - self.last_beat_time if self.last_beat_time > 0 else 0.5
|
||||
self.last_beat_time = self.current_time
|
||||
|
||||
# Step each scan
|
||||
for node_id, scan in self.scans.items():
|
||||
evaluator = ExprEvaluator(scan.rng)
|
||||
env = self._build_scan_env(scan)
|
||||
|
||||
# Evaluate step expression
|
||||
new_value = evaluator.eval(scan.step_expr, env)
|
||||
scan.value = new_value
|
||||
|
||||
# Update emit
|
||||
self._update_emit(node_id)
|
||||
|
||||
# Step the cycle state
|
||||
self._step_cycle()
|
||||
|
||||
def on_frame(self, energy: float, is_beat: bool, t: float = 0.0):
|
||||
"""Called each frame with audio analysis."""
|
||||
self.current_time = t
|
||||
self.energy = energy
|
||||
|
||||
# Update scans on beat (edge detection)
|
||||
if is_beat and not self.last_beat_detected:
|
||||
self.on_beat()
|
||||
self.last_beat_detected = is_beat
|
||||
|
||||
def resolve_binding(self, binding: dict) -> Any:
|
||||
"""Resolve a binding to get the current value."""
|
||||
if not isinstance(binding, dict) or not binding.get('_binding'):
|
||||
return binding
|
||||
|
||||
source_id = binding.get('source')
|
||||
feature = binding.get('feature', 'values')
|
||||
range_map = binding.get('range')
|
||||
|
||||
# Get the raw value
|
||||
if source_id in self.scan_outputs:
|
||||
value = self.scan_outputs[source_id]
|
||||
else:
|
||||
# Might be an analyzer reference - use energy as fallback
|
||||
value = self.energy
|
||||
|
||||
# Extract feature if value is a dict
|
||||
if isinstance(value, dict) and feature in value:
|
||||
value = value[feature]
|
||||
|
||||
# Apply range mapping
|
||||
if range_map and isinstance(value, (int, float)):
|
||||
lo, hi = range_map
|
||||
value = lo + value * (hi - lo)
|
||||
|
||||
return value
|
||||
|
||||
def get_effect_params(self, effect_node: dict) -> Dict[str, Any]:
|
||||
"""Get resolved parameters for an effect node."""
|
||||
config = effect_node.get('config', {})
|
||||
params = {}
|
||||
|
||||
for key, value in config.items():
|
||||
# Skip internal fields
|
||||
if key in ('effect', 'effect_path', 'effect_cid', 'effects_registry', 'analysis_refs'):
|
||||
continue
|
||||
|
||||
# Resolve bindings
|
||||
params[key] = self.resolve_binding(value)
|
||||
|
||||
return params
|
||||
|
||||
def get_scan_value(self, name: str) -> Any:
|
||||
"""Get scan output by name."""
|
||||
for node_id, scan in self.scans.items():
|
||||
if scan.name == name:
|
||||
return self.scan_outputs.get(node_id)
|
||||
return None
|
||||
|
||||
def get_all_scan_values(self) -> Dict[str, Any]:
|
||||
"""Get all named scan outputs."""
|
||||
result = {}
|
||||
for node_id, scan in self.scans.items():
|
||||
if scan.name:
|
||||
result[scan.name] = self.scan_outputs.get(node_id)
|
||||
return result
|
||||
|
||||
# === Compositor interface methods ===
|
||||
|
||||
def _get_video_sources(self) -> List[str]:
|
||||
"""Get list of video source node IDs."""
|
||||
sources = []
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SOURCE':
|
||||
sources.append(node['id'])
|
||||
# Filter to video only (exclude audio - last one is usually audio)
|
||||
# Look at file extensions in the paths
|
||||
return sources[:-1] if len(sources) > 1 else sources
|
||||
|
||||
def _trace_effect_chain(self, start_id: str, stop_at_blend: bool = True) -> List[dict]:
|
||||
"""Trace effect chain from a node, returning effects in order."""
|
||||
chain = []
|
||||
current_id = start_id
|
||||
|
||||
for _ in range(20): # Max depth
|
||||
# Find node that uses current as input
|
||||
next_node = None
|
||||
for node in self.recipe.nodes:
|
||||
if current_id in node.get('inputs', []):
|
||||
if node.get('type') == 'EFFECT':
|
||||
effect_type = node.get('config', {}).get('effect')
|
||||
chain.append(node)
|
||||
if stop_at_blend and effect_type == 'blend':
|
||||
return chain
|
||||
next_node = node
|
||||
break
|
||||
elif node.get('type') == 'SEGMENT':
|
||||
next_node = node
|
||||
break
|
||||
|
||||
if next_node is None:
|
||||
break
|
||||
current_id = next_node['id']
|
||||
|
||||
return chain
|
||||
|
||||
def _find_clip_chains(self, source_idx: int) -> tuple:
|
||||
"""Find effect chains for clip A and B from a source."""
|
||||
sources = self._get_video_sources()
|
||||
if source_idx >= len(sources):
|
||||
return [], []
|
||||
|
||||
source_id = sources[source_idx]
|
||||
|
||||
# Find SEGMENT node
|
||||
segment_id = None
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SEGMENT' and source_id in node.get('inputs', []):
|
||||
segment_id = node['id']
|
||||
break
|
||||
|
||||
if not segment_id:
|
||||
return [], []
|
||||
|
||||
# Find the two effect chains from segment (clip A and clip B)
|
||||
chains = []
|
||||
for node in self.recipe.nodes:
|
||||
if segment_id in node.get('inputs', []) and node.get('type') == 'EFFECT':
|
||||
chain = self._trace_effect_chain(segment_id)
|
||||
# Get chain starting from this specific branch
|
||||
branch_chain = [node]
|
||||
current = node['id']
|
||||
for _ in range(10):
|
||||
found = False
|
||||
for n in self.recipe.nodes:
|
||||
if current in n.get('inputs', []) and n.get('type') == 'EFFECT':
|
||||
branch_chain.append(n)
|
||||
if n.get('config', {}).get('effect') == 'blend':
|
||||
break
|
||||
current = n['id']
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
break
|
||||
chains.append(branch_chain)
|
||||
|
||||
# Return first two chains as A and B
|
||||
chain_a = chains[0] if len(chains) > 0 else []
|
||||
chain_b = chains[1] if len(chains) > 1 else []
|
||||
return chain_a, chain_b
|
||||
|
||||
def get_effect_params(self, source_idx: int, clip: str, energy: float) -> Dict:
|
||||
"""Get effect parameters for a source clip (compositor interface)."""
|
||||
# Get the correct chain for this clip
|
||||
chain_a, chain_b = self._find_clip_chains(source_idx)
|
||||
chain = chain_a if clip == 'a' else chain_b
|
||||
|
||||
# Default params
|
||||
params = {
|
||||
"rotate_angle": 0,
|
||||
"zoom_amount": 1.0,
|
||||
"invert_amount": 0,
|
||||
"hue_degrees": 0,
|
||||
"ascii_mix": 0,
|
||||
"ascii_char_size": 8,
|
||||
}
|
||||
|
||||
# Resolve from effects in chain
|
||||
for eff in chain:
|
||||
config = eff.get('config', {})
|
||||
effect_type = config.get('effect')
|
||||
|
||||
if effect_type == 'rotate':
|
||||
angle_binding = config.get('angle')
|
||||
if angle_binding:
|
||||
if isinstance(angle_binding, dict) and angle_binding.get('_binding'):
|
||||
# Bound to analyzer - use energy with range
|
||||
range_map = angle_binding.get('range')
|
||||
if range_map:
|
||||
lo, hi = range_map
|
||||
params["rotate_angle"] = lo + energy * (hi - lo)
|
||||
else:
|
||||
params["rotate_angle"] = self.resolve_binding(angle_binding)
|
||||
else:
|
||||
params["rotate_angle"] = angle_binding if isinstance(angle_binding, (int, float)) else 0
|
||||
|
||||
elif effect_type == 'zoom':
|
||||
amount_binding = config.get('amount')
|
||||
if amount_binding:
|
||||
if isinstance(amount_binding, dict) and amount_binding.get('_binding'):
|
||||
range_map = amount_binding.get('range')
|
||||
if range_map:
|
||||
lo, hi = range_map
|
||||
params["zoom_amount"] = lo + energy * (hi - lo)
|
||||
else:
|
||||
params["zoom_amount"] = self.resolve_binding(amount_binding)
|
||||
else:
|
||||
params["zoom_amount"] = amount_binding if isinstance(amount_binding, (int, float)) else 1.0
|
||||
|
||||
elif effect_type == 'invert':
|
||||
amount_binding = config.get('amount')
|
||||
if amount_binding:
|
||||
val = self.resolve_binding(amount_binding)
|
||||
params["invert_amount"] = val if isinstance(val, (int, float)) else 0
|
||||
|
||||
elif effect_type == 'hue_shift':
|
||||
deg_binding = config.get('degrees')
|
||||
if deg_binding:
|
||||
val = self.resolve_binding(deg_binding)
|
||||
params["hue_degrees"] = val if isinstance(val, (int, float)) else 0
|
||||
|
||||
elif effect_type == 'ascii_art':
|
||||
mix_binding = config.get('mix')
|
||||
if mix_binding:
|
||||
val = self.resolve_binding(mix_binding)
|
||||
params["ascii_mix"] = val if isinstance(val, (int, float)) else 0
|
||||
size_binding = config.get('char_size')
|
||||
if size_binding:
|
||||
if isinstance(size_binding, dict) and size_binding.get('_binding'):
|
||||
range_map = size_binding.get('range')
|
||||
if range_map:
|
||||
lo, hi = range_map
|
||||
params["ascii_char_size"] = lo + energy * (hi - lo)
|
||||
|
||||
return params
|
||||
|
||||
def get_pair_params(self, source_idx: int) -> Dict:
|
||||
"""Get blend and rotation params for a video pair (compositor interface)."""
|
||||
params = {
|
||||
"blend_opacity": 0.5,
|
||||
"pair_rotation": 0,
|
||||
}
|
||||
|
||||
# Find the blend node for this source
|
||||
chain_a, _ = self._find_clip_chains(source_idx)
|
||||
|
||||
# The last effect in chain_a should be the blend
|
||||
blend_node = None
|
||||
for eff in reversed(chain_a):
|
||||
if eff.get('config', {}).get('effect') == 'blend':
|
||||
blend_node = eff
|
||||
break
|
||||
|
||||
if blend_node:
|
||||
config = blend_node.get('config', {})
|
||||
opacity_binding = config.get('opacity')
|
||||
if opacity_binding:
|
||||
val = self.resolve_binding(opacity_binding)
|
||||
if isinstance(val, (int, float)):
|
||||
params["blend_opacity"] = val
|
||||
|
||||
# Find rotate after blend (pair rotation)
|
||||
blend_id = blend_node['id']
|
||||
for node in self.recipe.nodes:
|
||||
if blend_id in node.get('inputs', []) and node.get('type') == 'EFFECT':
|
||||
if node.get('config', {}).get('effect') == 'rotate':
|
||||
angle_binding = node.get('config', {}).get('angle')
|
||||
if angle_binding:
|
||||
val = self.resolve_binding(angle_binding)
|
||||
if isinstance(val, (int, float)):
|
||||
params["pair_rotation"] = val
|
||||
break
|
||||
|
||||
return params
|
||||
|
||||
def _get_cycle_state(self) -> dict:
|
||||
"""Get current cycle state from SLICE_ON or internal tracking."""
|
||||
if not hasattr(self, '_cycle_state'):
|
||||
# Initialize from SLICE_ON node
|
||||
for node in self.recipe.nodes:
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
init = node.get('config', {}).get('init', {})
|
||||
self._cycle_state = {
|
||||
'cycle': init.get('cycle', 0),
|
||||
'beat': init.get('beat', 0),
|
||||
'clen': init.get('clen', 60),
|
||||
}
|
||||
break
|
||||
else:
|
||||
self._cycle_state = {'cycle': 0, 'beat': 0, 'clen': 60}
|
||||
|
||||
return self._cycle_state
|
||||
|
||||
def _step_cycle(self):
|
||||
"""Step the cycle state forward on beat by evaluating SLICE_ON Lambda."""
|
||||
# Use interpreter to evaluate the Lambda
|
||||
self._eval_slice_on()
|
||||
|
||||
def get_cycle_weights(self) -> List[float]:
|
||||
"""Get blend weights for cycle-crossfade from SLICE_ON result."""
|
||||
n = len(self._get_video_sources())
|
||||
if n == 0:
|
||||
return [1.0]
|
||||
|
||||
# Get weights from interpreted result
|
||||
if self._slice_on_result:
|
||||
compose = self._slice_on_result.get('compose', {})
|
||||
weights = compose.get('weights', [])
|
||||
if weights and len(weights) == n:
|
||||
# Normalize
|
||||
total = sum(weights)
|
||||
if total > 0:
|
||||
return [w / total for w in weights]
|
||||
|
||||
# Fallback: equal weights
|
||||
return [1.0 / n] * n
|
||||
|
||||
def get_cycle_zooms(self) -> List[float]:
|
||||
"""Get zoom amounts for cycle-crossfade from SLICE_ON result."""
|
||||
n = len(self._get_video_sources())
|
||||
if n == 0:
|
||||
return [1.0]
|
||||
|
||||
# Get zooms from interpreted result (layers -> effects -> zoom amount)
|
||||
if self._slice_on_result:
|
||||
layers = self._slice_on_result.get('layers', [])
|
||||
if layers and len(layers) == n:
|
||||
zooms = []
|
||||
for layer in layers:
|
||||
effects = layer.get('effects', [])
|
||||
zoom_amt = 1.0
|
||||
for eff in effects:
|
||||
if eff.get('effect') == 'zoom' or (hasattr(eff.get('effect'), 'name') and eff.get('effect').name == 'zoom'):
|
||||
zoom_amt = eff.get('amount', 1.0)
|
||||
break
|
||||
zooms.append(zoom_amt)
|
||||
return zooms
|
||||
|
||||
# Fallback
|
||||
return [1.0] * n
|
||||
|
||||
def _get_final_rotate_scan_id(self) -> str:
|
||||
"""Find the scan ID that drives the final rotation (after SLICE_ON)."""
|
||||
if hasattr(self, '_final_rotate_scan_id'):
|
||||
return self._final_rotate_scan_id
|
||||
|
||||
# Find SLICE_ON node index
|
||||
slice_on_idx = None
|
||||
for i, node in enumerate(self.recipe.nodes):
|
||||
if node.get('type') == 'SLICE_ON':
|
||||
slice_on_idx = i
|
||||
break
|
||||
|
||||
# Find rotate effect after SLICE_ON
|
||||
if slice_on_idx is not None:
|
||||
for node in self.recipe.nodes[slice_on_idx + 1:]:
|
||||
if node.get('type') == 'EFFECT':
|
||||
config = node.get('config', {})
|
||||
if config.get('effect') == 'rotate':
|
||||
angle_binding = config.get('angle', {})
|
||||
if isinstance(angle_binding, dict) and angle_binding.get('_binding'):
|
||||
self._final_rotate_scan_id = angle_binding.get('source')
|
||||
return self._final_rotate_scan_id
|
||||
|
||||
self._final_rotate_scan_id = None
|
||||
return None
|
||||
|
||||
def get_final_effects(self, energy: float) -> Dict:
|
||||
"""Get final composition effects (compositor interface)."""
|
||||
# Get named scans
|
||||
scan_values = self.get_all_scan_values()
|
||||
|
||||
# Whole spin - get from the specific scan bound to final rotate effect
|
||||
whole_spin = 0
|
||||
final_rotate_scan_id = self._get_final_rotate_scan_id()
|
||||
if final_rotate_scan_id and final_rotate_scan_id in self.scan_outputs:
|
||||
val = self.scan_outputs[final_rotate_scan_id]
|
||||
if isinstance(val, dict) and 'angle' in val:
|
||||
whole_spin = val['angle']
|
||||
elif isinstance(val, (int, float)):
|
||||
whole_spin = val
|
||||
|
||||
# Ripple
|
||||
ripple_gate = scan_values.get('ripple-gate', 0)
|
||||
ripple_cx = scan_values.get('ripple-cx', 0.5)
|
||||
ripple_cy = scan_values.get('ripple-cy', 0.5)
|
||||
|
||||
if isinstance(ripple_gate, dict):
|
||||
ripple_gate = ripple_gate.get('gate', 0) if 'gate' in ripple_gate else 1
|
||||
|
||||
return {
|
||||
"whole_spin_angle": whole_spin,
|
||||
"ripple_amplitude": ripple_gate * (5 + energy * 45),
|
||||
"ripple_cx": ripple_cx if isinstance(ripple_cx, (int, float)) else 0.5,
|
||||
"ripple_cy": ripple_cy if isinstance(ripple_cy, (int, float)) else 0.5,
|
||||
}
|
||||
376
streaming/sexp_interp.py
Normal file
376
streaming/sexp_interp.py
Normal file
@@ -0,0 +1,376 @@
|
||||
"""
|
||||
S-expression interpreter for streaming execution.
|
||||
|
||||
Evaluates sexp expressions including:
|
||||
- let bindings
|
||||
- lambda definitions and calls
|
||||
- Arithmetic, comparison, logic operators
|
||||
- dict/list operations
|
||||
- Random number generation
|
||||
"""
|
||||
|
||||
import random
|
||||
from typing import Any, Dict, List, Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class Lambda:
|
||||
"""Runtime lambda value."""
|
||||
params: List[str]
|
||||
body: Any
|
||||
closure: Dict[str, Any]
|
||||
|
||||
|
||||
class Symbol:
|
||||
"""Symbol reference."""
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
|
||||
def __repr__(self):
|
||||
return f"Symbol({self.name})"
|
||||
|
||||
|
||||
class SexpInterpreter:
|
||||
"""
|
||||
Interprets S-expressions in real-time.
|
||||
|
||||
Handles the full sexp language used in recipes.
|
||||
"""
|
||||
|
||||
def __init__(self, rng: random.Random = None):
|
||||
self.rng = rng or random.Random()
|
||||
self.globals: Dict[str, Any] = {}
|
||||
|
||||
def eval(self, expr: Any, env: Dict[str, Any] = None) -> Any:
|
||||
"""Evaluate an expression in the given environment."""
|
||||
if env is None:
|
||||
env = {}
|
||||
|
||||
# Literals
|
||||
if isinstance(expr, (int, float, str, bool)) or expr is None:
|
||||
return expr
|
||||
|
||||
# Symbol lookup
|
||||
if isinstance(expr, Symbol) or (hasattr(expr, 'name') and hasattr(expr, '__class__') and expr.__class__.__name__ == 'Symbol'):
|
||||
name = expr.name if hasattr(expr, 'name') else str(expr)
|
||||
if name in env:
|
||||
return env[name]
|
||||
if name in self.globals:
|
||||
return self.globals[name]
|
||||
raise NameError(f"Undefined symbol: {name}")
|
||||
|
||||
# Compiled expression dict (from compiler)
|
||||
if isinstance(expr, dict):
|
||||
if expr.get('_expr'):
|
||||
return self._eval_compiled_expr(expr, env)
|
||||
# Plain dict - evaluate values that might be expressions
|
||||
result = {}
|
||||
for k, v in expr.items():
|
||||
# Some keys should keep Symbol values as strings (effect names, modes)
|
||||
if k in ('effect', 'mode') and hasattr(v, 'name'):
|
||||
result[k] = v.name
|
||||
else:
|
||||
result[k] = self.eval(v, env)
|
||||
return result
|
||||
|
||||
# List expression (sexp)
|
||||
if isinstance(expr, (list, tuple)) and len(expr) > 0:
|
||||
return self._eval_list(expr, env)
|
||||
|
||||
# Empty list
|
||||
if isinstance(expr, (list, tuple)):
|
||||
return []
|
||||
|
||||
return expr
|
||||
|
||||
def _eval_compiled_expr(self, expr: dict, env: Dict[str, Any]) -> Any:
|
||||
"""Evaluate a compiled expression dict."""
|
||||
op = expr.get('op')
|
||||
args = expr.get('args', [])
|
||||
|
||||
if op == 'var':
|
||||
name = expr.get('name')
|
||||
if name in env:
|
||||
return env[name]
|
||||
if name in self.globals:
|
||||
return self.globals[name]
|
||||
raise NameError(f"Undefined: {name}")
|
||||
|
||||
elif op == 'dict':
|
||||
keys = expr.get('keys', [])
|
||||
values = [self.eval(a, env) for a in args]
|
||||
return dict(zip(keys, values))
|
||||
|
||||
elif op == 'get':
|
||||
obj = self.eval(args[0], env)
|
||||
key = args[1]
|
||||
return obj.get(key) if isinstance(obj, dict) else obj[key]
|
||||
|
||||
elif op == 'if':
|
||||
cond = self.eval(args[0], env)
|
||||
if cond:
|
||||
return self.eval(args[1], env)
|
||||
elif len(args) > 2:
|
||||
return self.eval(args[2], env)
|
||||
return None
|
||||
|
||||
# Comparison
|
||||
elif op == '<':
|
||||
return self.eval(args[0], env) < self.eval(args[1], env)
|
||||
elif op == '>':
|
||||
return self.eval(args[0], env) > self.eval(args[1], env)
|
||||
elif op == '<=':
|
||||
return self.eval(args[0], env) <= self.eval(args[1], env)
|
||||
elif op == '>=':
|
||||
return self.eval(args[0], env) >= self.eval(args[1], env)
|
||||
elif op == '=':
|
||||
return self.eval(args[0], env) == self.eval(args[1], env)
|
||||
elif op == '!=':
|
||||
return self.eval(args[0], env) != self.eval(args[1], env)
|
||||
|
||||
# Arithmetic
|
||||
elif op == '+':
|
||||
return self.eval(args[0], env) + self.eval(args[1], env)
|
||||
elif op == '-':
|
||||
return self.eval(args[0], env) - self.eval(args[1], env)
|
||||
elif op == '*':
|
||||
return self.eval(args[0], env) * self.eval(args[1], env)
|
||||
elif op == '/':
|
||||
return self.eval(args[0], env) / self.eval(args[1], env)
|
||||
elif op == 'mod':
|
||||
return self.eval(args[0], env) % self.eval(args[1], env)
|
||||
|
||||
# Random
|
||||
elif op == 'rand':
|
||||
return self.rng.random()
|
||||
elif op == 'rand-int':
|
||||
return self.rng.randint(self.eval(args[0], env), self.eval(args[1], env))
|
||||
elif op == 'rand-range':
|
||||
return self.rng.uniform(self.eval(args[0], env), self.eval(args[1], env))
|
||||
|
||||
# Logic
|
||||
elif op == 'and':
|
||||
return all(self.eval(a, env) for a in args)
|
||||
elif op == 'or':
|
||||
return any(self.eval(a, env) for a in args)
|
||||
elif op == 'not':
|
||||
return not self.eval(args[0], env)
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown op: {op}")
|
||||
|
||||
def _eval_list(self, expr: list, env: Dict[str, Any]) -> Any:
|
||||
"""Evaluate a list expression (sexp form)."""
|
||||
if len(expr) == 0:
|
||||
return []
|
||||
|
||||
head = expr[0]
|
||||
|
||||
# Get head name
|
||||
if isinstance(head, Symbol) or (hasattr(head, 'name') and hasattr(head, '__class__')):
|
||||
head_name = head.name if hasattr(head, 'name') else str(head)
|
||||
elif isinstance(head, str):
|
||||
head_name = head
|
||||
else:
|
||||
# Not a symbol - check if it's a data list or function call
|
||||
if isinstance(head, dict):
|
||||
# List of dicts - evaluate each element as data
|
||||
return [self.eval(item, env) for item in expr]
|
||||
# Otherwise evaluate as function call
|
||||
fn = self.eval(head, env)
|
||||
args = [self.eval(a, env) for a in expr[1:]]
|
||||
return self._call(fn, args, env)
|
||||
|
||||
# Special forms
|
||||
if head_name == 'let':
|
||||
return self._eval_let(expr, env)
|
||||
elif head_name in ('lambda', 'fn'):
|
||||
return self._eval_lambda(expr, env)
|
||||
elif head_name == 'if':
|
||||
return self._eval_if(expr, env)
|
||||
elif head_name == 'dict':
|
||||
return self._eval_dict(expr, env)
|
||||
elif head_name == 'get':
|
||||
obj = self.eval(expr[1], env)
|
||||
key = self.eval(expr[2], env) if len(expr) > 2 else expr[2]
|
||||
if isinstance(key, str):
|
||||
return obj.get(key) if isinstance(obj, dict) else getattr(obj, key, None)
|
||||
return obj[key]
|
||||
elif head_name == 'len':
|
||||
return len(self.eval(expr[1], env))
|
||||
elif head_name == 'range':
|
||||
start = self.eval(expr[1], env)
|
||||
end = self.eval(expr[2], env) if len(expr) > 2 else start
|
||||
if len(expr) == 2:
|
||||
return list(range(end))
|
||||
return list(range(start, end))
|
||||
elif head_name == 'map':
|
||||
fn = self.eval(expr[1], env)
|
||||
lst = self.eval(expr[2], env)
|
||||
return [self._call(fn, [x], env) for x in lst]
|
||||
elif head_name == 'mod':
|
||||
return self.eval(expr[1], env) % self.eval(expr[2], env)
|
||||
|
||||
# Arithmetic
|
||||
elif head_name == '+':
|
||||
return self.eval(expr[1], env) + self.eval(expr[2], env)
|
||||
elif head_name == '-':
|
||||
if len(expr) == 2:
|
||||
return -self.eval(expr[1], env)
|
||||
return self.eval(expr[1], env) - self.eval(expr[2], env)
|
||||
elif head_name == '*':
|
||||
return self.eval(expr[1], env) * self.eval(expr[2], env)
|
||||
elif head_name == '/':
|
||||
return self.eval(expr[1], env) / self.eval(expr[2], env)
|
||||
|
||||
# Comparison
|
||||
elif head_name == '<':
|
||||
return self.eval(expr[1], env) < self.eval(expr[2], env)
|
||||
elif head_name == '>':
|
||||
return self.eval(expr[1], env) > self.eval(expr[2], env)
|
||||
elif head_name == '<=':
|
||||
return self.eval(expr[1], env) <= self.eval(expr[2], env)
|
||||
elif head_name == '>=':
|
||||
return self.eval(expr[1], env) >= self.eval(expr[2], env)
|
||||
elif head_name == '=':
|
||||
return self.eval(expr[1], env) == self.eval(expr[2], env)
|
||||
|
||||
# Logic
|
||||
elif head_name == 'and':
|
||||
return all(self.eval(a, env) for a in expr[1:])
|
||||
elif head_name == 'or':
|
||||
return any(self.eval(a, env) for a in expr[1:])
|
||||
elif head_name == 'not':
|
||||
return not self.eval(expr[1], env)
|
||||
|
||||
# Function call
|
||||
else:
|
||||
fn = env.get(head_name) or self.globals.get(head_name)
|
||||
if fn is None:
|
||||
raise NameError(f"Undefined function: {head_name}")
|
||||
args = [self.eval(a, env) for a in expr[1:]]
|
||||
return self._call(fn, args, env)
|
||||
|
||||
def _eval_let(self, expr: list, env: Dict[str, Any]) -> Any:
|
||||
"""Evaluate (let [bindings...] body)."""
|
||||
bindings = expr[1]
|
||||
body = expr[2]
|
||||
|
||||
# Create new environment with bindings
|
||||
new_env = dict(env)
|
||||
|
||||
# Process bindings in pairs
|
||||
i = 0
|
||||
while i < len(bindings):
|
||||
name = bindings[i]
|
||||
if isinstance(name, Symbol) or hasattr(name, 'name'):
|
||||
name = name.name if hasattr(name, 'name') else str(name)
|
||||
value = self.eval(bindings[i + 1], new_env)
|
||||
new_env[name] = value
|
||||
i += 2
|
||||
|
||||
return self.eval(body, new_env)
|
||||
|
||||
def _eval_lambda(self, expr: list, env: Dict[str, Any]) -> Lambda:
|
||||
"""Evaluate (lambda [params] body)."""
|
||||
params_expr = expr[1]
|
||||
body = expr[2]
|
||||
|
||||
# Extract parameter names
|
||||
params = []
|
||||
for p in params_expr:
|
||||
if isinstance(p, Symbol) or hasattr(p, 'name'):
|
||||
params.append(p.name if hasattr(p, 'name') else str(p))
|
||||
else:
|
||||
params.append(str(p))
|
||||
|
||||
return Lambda(params=params, body=body, closure=dict(env))
|
||||
|
||||
def _eval_if(self, expr: list, env: Dict[str, Any]) -> Any:
|
||||
"""Evaluate (if cond then else)."""
|
||||
cond = self.eval(expr[1], env)
|
||||
if cond:
|
||||
return self.eval(expr[2], env)
|
||||
elif len(expr) > 3:
|
||||
return self.eval(expr[3], env)
|
||||
return None
|
||||
|
||||
def _eval_dict(self, expr: list, env: Dict[str, Any]) -> dict:
|
||||
"""Evaluate (dict :key val ...)."""
|
||||
result = {}
|
||||
i = 1
|
||||
while i < len(expr):
|
||||
key = expr[i]
|
||||
# Handle keyword syntax (:key) and Keyword objects
|
||||
if hasattr(key, 'name'):
|
||||
key = key.name
|
||||
elif hasattr(key, '__class__') and key.__class__.__name__ == 'Keyword':
|
||||
key = str(key).lstrip(':')
|
||||
elif isinstance(key, str) and key.startswith(':'):
|
||||
key = key[1:]
|
||||
value = self.eval(expr[i + 1], env)
|
||||
result[key] = value
|
||||
i += 2
|
||||
return result
|
||||
|
||||
def _call(self, fn: Any, args: List[Any], env: Dict[str, Any]) -> Any:
|
||||
"""Call a function with arguments."""
|
||||
if isinstance(fn, Lambda):
|
||||
# Our own Lambda type
|
||||
call_env = dict(fn.closure)
|
||||
for param, arg in zip(fn.params, args):
|
||||
call_env[param] = arg
|
||||
return self.eval(fn.body, call_env)
|
||||
elif hasattr(fn, 'params') and hasattr(fn, 'body'):
|
||||
# Lambda from parser (artdag.sexp.parser.Lambda)
|
||||
call_env = dict(env)
|
||||
if hasattr(fn, 'closure') and fn.closure:
|
||||
call_env.update(fn.closure)
|
||||
# Get param names
|
||||
params = []
|
||||
for p in fn.params:
|
||||
if hasattr(p, 'name'):
|
||||
params.append(p.name)
|
||||
else:
|
||||
params.append(str(p))
|
||||
for param, arg in zip(params, args):
|
||||
call_env[param] = arg
|
||||
return self.eval(fn.body, call_env)
|
||||
elif callable(fn):
|
||||
return fn(*args)
|
||||
else:
|
||||
raise TypeError(f"Not callable: {type(fn).__name__}")
|
||||
|
||||
|
||||
def eval_slice_on_lambda(lambda_obj, acc: dict, i: int, start: float, end: float,
|
||||
videos: list, interp: SexpInterpreter = None) -> dict:
|
||||
"""
|
||||
Evaluate a SLICE_ON lambda function.
|
||||
|
||||
Args:
|
||||
lambda_obj: The Lambda object from the compiled recipe
|
||||
acc: Current accumulator state
|
||||
i: Beat index
|
||||
start: Slice start time
|
||||
end: Slice end time
|
||||
videos: List of video inputs
|
||||
interp: Interpreter to use
|
||||
|
||||
Returns:
|
||||
Dict with 'layers', 'compose', 'acc' keys
|
||||
"""
|
||||
if interp is None:
|
||||
interp = SexpInterpreter()
|
||||
|
||||
# Set up global 'videos' for (len videos) to work
|
||||
interp.globals['videos'] = videos
|
||||
|
||||
# Build initial environment with lambda parameters
|
||||
env = dict(lambda_obj.closure) if hasattr(lambda_obj, 'closure') and lambda_obj.closure else {}
|
||||
env['videos'] = videos
|
||||
|
||||
# Call the lambda
|
||||
result = interp._call(lambda_obj, [acc, i, start, end], env)
|
||||
|
||||
return result
|
||||
281
streaming/sources.py
Normal file
281
streaming/sources.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
Video and image sources with looping support.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class Source(ABC):
|
||||
"""Abstract base class for frame sources."""
|
||||
|
||||
@abstractmethod
|
||||
def read_frame(self, t: float) -> np.ndarray:
|
||||
"""Read frame at time t (with looping if needed)."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def duration(self) -> float:
|
||||
"""Source duration in seconds."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def size(self) -> Tuple[int, int]:
|
||||
"""Frame size as (width, height)."""
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def fps(self) -> float:
|
||||
"""Frames per second."""
|
||||
pass
|
||||
|
||||
|
||||
class VideoSource(Source):
|
||||
"""
|
||||
Video file source with automatic looping.
|
||||
|
||||
Reads frames on-demand, seeking as needed. When time exceeds
|
||||
duration, wraps around (loops).
|
||||
"""
|
||||
|
||||
def __init__(self, path: str, target_fps: float = 30):
|
||||
self.path = Path(path)
|
||||
self.target_fps = target_fps
|
||||
|
||||
# Initialize decode state first (before _probe which could fail)
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._current_start: Optional[float] = None
|
||||
self._frame_buffer: Optional[np.ndarray] = None
|
||||
self._buffer_time: Optional[float] = None
|
||||
|
||||
self._duration = None
|
||||
self._size = None
|
||||
self._fps = None
|
||||
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"Video not found: {path}")
|
||||
|
||||
self._probe()
|
||||
|
||||
def _probe(self):
|
||||
"""Get video metadata."""
|
||||
cmd = [
|
||||
"ffprobe", "-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format", "-show_streams",
|
||||
str(self.path)
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
data = json.loads(result.stdout)
|
||||
|
||||
# Get duration
|
||||
self._duration = float(data["format"]["duration"])
|
||||
|
||||
# Get video stream info
|
||||
for stream in data["streams"]:
|
||||
if stream["codec_type"] == "video":
|
||||
self._size = (int(stream["width"]), int(stream["height"]))
|
||||
# Parse fps from r_frame_rate (e.g., "30/1" or "30000/1001")
|
||||
fps_parts = stream.get("r_frame_rate", "30/1").split("/")
|
||||
self._fps = float(fps_parts[0]) / float(fps_parts[1])
|
||||
break
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
return self._duration
|
||||
|
||||
@property
|
||||
def size(self) -> Tuple[int, int]:
|
||||
return self._size
|
||||
|
||||
@property
|
||||
def fps(self) -> float:
|
||||
return self._fps
|
||||
|
||||
def _start_decode(self, start_time: float):
|
||||
"""Start ffmpeg decode process from given time."""
|
||||
if self._process:
|
||||
try:
|
||||
self._process.stdout.close()
|
||||
except:
|
||||
pass
|
||||
self._process.terminate()
|
||||
try:
|
||||
self._process.wait(timeout=1)
|
||||
except:
|
||||
self._process.kill()
|
||||
self._process.wait()
|
||||
|
||||
w, h = self._size
|
||||
cmd = [
|
||||
"ffmpeg", "-v", "quiet",
|
||||
"-ss", str(start_time),
|
||||
"-i", str(self.path),
|
||||
"-f", "rawvideo",
|
||||
"-pix_fmt", "rgb24",
|
||||
"-r", str(self.target_fps),
|
||||
"-"
|
||||
]
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
bufsize=w * h * 3 * 4, # Buffer a few frames
|
||||
)
|
||||
self._current_start = start_time
|
||||
self._buffer_time = start_time
|
||||
|
||||
def read_frame(self, t: float) -> np.ndarray:
|
||||
"""
|
||||
Read frame at time t.
|
||||
|
||||
If t exceeds duration, wraps around (loops).
|
||||
Seeks if needed, otherwise reads sequentially.
|
||||
"""
|
||||
# Wrap time for looping
|
||||
t_wrapped = t % self._duration
|
||||
|
||||
# Check if we need to seek (loop point or large time jump)
|
||||
need_seek = (
|
||||
self._process is None or
|
||||
self._buffer_time is None or
|
||||
abs(t_wrapped - self._buffer_time) > 1.0 / self.target_fps * 2
|
||||
)
|
||||
|
||||
if need_seek:
|
||||
self._start_decode(t_wrapped)
|
||||
|
||||
# Read frame
|
||||
w, h = self._size
|
||||
frame_size = w * h * 3
|
||||
|
||||
# Try to read with retries for seek settling
|
||||
for attempt in range(3):
|
||||
raw = self._process.stdout.read(frame_size)
|
||||
if len(raw) == frame_size:
|
||||
break
|
||||
# End of stream or seek not ready - restart from beginning
|
||||
self._start_decode(0)
|
||||
|
||||
if len(raw) < frame_size:
|
||||
# Still no data - return last frame or black
|
||||
if self._frame_buffer is not None:
|
||||
return self._frame_buffer.copy()
|
||||
return np.zeros((h, w, 3), dtype=np.uint8)
|
||||
|
||||
frame = np.frombuffer(raw, dtype=np.uint8).reshape((h, w, 3))
|
||||
self._frame_buffer = frame # Cache for fallback
|
||||
self._buffer_time = t_wrapped + 1.0 / self.target_fps
|
||||
|
||||
return frame
|
||||
|
||||
def close(self):
|
||||
"""Clean up resources."""
|
||||
if self._process:
|
||||
self._process.terminate()
|
||||
self._process.wait()
|
||||
self._process = None
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def __repr__(self):
|
||||
return f"VideoSource({self.path.name}, {self._size[0]}x{self._size[1]}, {self._duration:.1f}s)"
|
||||
|
||||
|
||||
class ImageSource(Source):
|
||||
"""
|
||||
Static image source (returns same frame for any time).
|
||||
|
||||
Useful for backgrounds, overlays, etc.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str):
|
||||
self.path = Path(path)
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"Image not found: {path}")
|
||||
|
||||
# Load image
|
||||
import cv2
|
||||
self._frame = cv2.imread(str(self.path))
|
||||
self._frame = cv2.cvtColor(self._frame, cv2.COLOR_BGR2RGB)
|
||||
self._size = (self._frame.shape[1], self._frame.shape[0])
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
return float('inf') # Images last forever
|
||||
|
||||
@property
|
||||
def size(self) -> Tuple[int, int]:
|
||||
return self._size
|
||||
|
||||
@property
|
||||
def fps(self) -> float:
|
||||
return 30.0 # Arbitrary
|
||||
|
||||
def read_frame(self, t: float) -> np.ndarray:
|
||||
return self._frame.copy()
|
||||
|
||||
def __repr__(self):
|
||||
return f"ImageSource({self.path.name}, {self._size[0]}x{self._size[1]})"
|
||||
|
||||
|
||||
class LiveSource(Source):
|
||||
"""
|
||||
Live video capture source (webcam, capture card, etc.).
|
||||
|
||||
Time parameter is ignored - always returns latest frame.
|
||||
"""
|
||||
|
||||
def __init__(self, device: int = 0, size: Tuple[int, int] = (1280, 720), fps: float = 30):
|
||||
import cv2
|
||||
self._cap = cv2.VideoCapture(device)
|
||||
self._cap.set(cv2.CAP_PROP_FRAME_WIDTH, size[0])
|
||||
self._cap.set(cv2.CAP_PROP_FRAME_HEIGHT, size[1])
|
||||
self._cap.set(cv2.CAP_PROP_FPS, fps)
|
||||
|
||||
# Get actual settings
|
||||
self._size = (
|
||||
int(self._cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
|
||||
int(self._cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||
)
|
||||
self._fps = self._cap.get(cv2.CAP_PROP_FPS)
|
||||
|
||||
if not self._cap.isOpened():
|
||||
raise RuntimeError(f"Could not open video device {device}")
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
return float('inf') # Live - no duration
|
||||
|
||||
@property
|
||||
def size(self) -> Tuple[int, int]:
|
||||
return self._size
|
||||
|
||||
@property
|
||||
def fps(self) -> float:
|
||||
return self._fps
|
||||
|
||||
def read_frame(self, t: float) -> np.ndarray:
|
||||
"""Read latest frame (t is ignored for live sources)."""
|
||||
import cv2
|
||||
ret, frame = self._cap.read()
|
||||
if not ret:
|
||||
return np.zeros((self._size[1], self._size[0], 3), dtype=np.uint8)
|
||||
return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||||
|
||||
def close(self):
|
||||
self._cap.release()
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def __repr__(self):
|
||||
return f"LiveSource({self._size[0]}x{self._size[1]}, {self._fps}fps)"
|
||||
1081
streaming/stream_sexp.py
Normal file
1081
streaming/stream_sexp.py
Normal file
File diff suppressed because it is too large
Load Diff
863
streaming/stream_sexp_generic.py
Normal file
863
streaming/stream_sexp_generic.py
Normal file
@@ -0,0 +1,863 @@
|
||||
"""
|
||||
Fully Generic Streaming S-expression Interpreter.
|
||||
|
||||
The interpreter knows NOTHING about video, audio, or any domain.
|
||||
All domain logic comes from primitives loaded via (require-primitives ...).
|
||||
|
||||
Built-in forms:
|
||||
- Control: if, cond, let, let*, lambda, ->
|
||||
- Arithmetic: +, -, *, /, mod, map-range
|
||||
- Comparison: <, >, =, <=, >=, and, or, not
|
||||
- Data: dict, get, list, nth, len, quote
|
||||
- Random: rand, rand-int, rand-range
|
||||
- Scan: bind (access scan state)
|
||||
|
||||
Everything else comes from primitives or effects.
|
||||
|
||||
Context (ctx) is passed explicitly to frame evaluation:
|
||||
- ctx.t: current time
|
||||
- ctx.frame-num: current frame number
|
||||
- ctx.fps: frames per second
|
||||
"""
|
||||
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
import math
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
# Try pip-installed artdag first, fall back to local path
|
||||
try:
|
||||
from artdag.sexp.parser import parse, parse_all, Symbol, Keyword
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent.parent / "artdag"))
|
||||
from artdag.sexp.parser import parse, parse_all, Symbol, Keyword
|
||||
|
||||
|
||||
@dataclass
|
||||
class Context:
|
||||
"""Runtime context passed to frame evaluation."""
|
||||
t: float = 0.0
|
||||
frame_num: int = 0
|
||||
fps: float = 30.0
|
||||
|
||||
|
||||
class StreamInterpreter:
|
||||
"""
|
||||
Fully generic streaming sexp interpreter.
|
||||
|
||||
No domain-specific knowledge - just evaluates expressions
|
||||
and calls primitives.
|
||||
"""
|
||||
|
||||
def __init__(self, sexp_path: str):
|
||||
self.sexp_path = Path(sexp_path)
|
||||
self.sexp_dir = self.sexp_path.parent
|
||||
|
||||
text = self.sexp_path.read_text()
|
||||
self.ast = parse(text)
|
||||
|
||||
self.config = self._parse_config()
|
||||
|
||||
# Global environment for def bindings
|
||||
self.globals: Dict[str, Any] = {}
|
||||
|
||||
# Scans
|
||||
self.scans: Dict[str, dict] = {}
|
||||
|
||||
# Audio playback path (for syncing output)
|
||||
self.audio_playback: Optional[str] = None
|
||||
|
||||
# Registries for external definitions
|
||||
self.primitives: Dict[str, Any] = {}
|
||||
self.effects: Dict[str, dict] = {}
|
||||
self.macros: Dict[str, dict] = {}
|
||||
self.primitive_lib_dir = self.sexp_dir.parent / "sexp_effects" / "primitive_libs"
|
||||
|
||||
self.frame_pipeline = None
|
||||
|
||||
# External config files (set before run())
|
||||
self.sources_config: Optional[Path] = None
|
||||
self.audio_config: Optional[Path] = None
|
||||
|
||||
import random
|
||||
self.rng = random.Random(self.config.get('seed', 42))
|
||||
|
||||
def _load_config_file(self, config_path):
|
||||
"""Load a config file and process its definitions."""
|
||||
config_path = Path(config_path) # Accept str or Path
|
||||
if not config_path.exists():
|
||||
print(f"Warning: config file not found: {config_path}", file=sys.stderr)
|
||||
return
|
||||
|
||||
text = config_path.read_text()
|
||||
ast = parse_all(text)
|
||||
|
||||
for form in ast:
|
||||
if not isinstance(form, list) or not form:
|
||||
continue
|
||||
if not isinstance(form[0], Symbol):
|
||||
continue
|
||||
|
||||
cmd = form[0].name
|
||||
|
||||
if cmd == 'require-primitives':
|
||||
lib_name = form[1] if isinstance(form[1], str) else str(form[1]).strip('"')
|
||||
self._load_primitives(lib_name)
|
||||
|
||||
elif cmd == 'def':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
value = self._eval(form[2], self.globals)
|
||||
self.globals[name] = value
|
||||
print(f"Config: {name}", file=sys.stderr)
|
||||
|
||||
elif cmd == 'audio-playback':
|
||||
# Path relative to working directory (consistent with other paths)
|
||||
path = str(form[1]).strip('"')
|
||||
self.audio_playback = str(Path(path).resolve())
|
||||
print(f"Audio playback: {self.audio_playback}", file=sys.stderr)
|
||||
|
||||
def _parse_config(self) -> dict:
|
||||
"""Parse config from (stream name :key val ...)."""
|
||||
config = {'fps': 30, 'seed': 42, 'width': 720, 'height': 720}
|
||||
if not self.ast or not isinstance(self.ast[0], Symbol):
|
||||
return config
|
||||
if self.ast[0].name != 'stream':
|
||||
return config
|
||||
|
||||
i = 2
|
||||
while i < len(self.ast):
|
||||
if isinstance(self.ast[i], Keyword):
|
||||
config[self.ast[i].name] = self.ast[i + 1] if i + 1 < len(self.ast) else None
|
||||
i += 2
|
||||
elif isinstance(self.ast[i], list):
|
||||
break
|
||||
else:
|
||||
i += 1
|
||||
return config
|
||||
|
||||
def _load_primitives(self, lib_name: str):
|
||||
"""Load primitives from a Python library file."""
|
||||
import importlib.util
|
||||
|
||||
lib_paths = [
|
||||
self.primitive_lib_dir / f"{lib_name}.py",
|
||||
self.sexp_dir / "primitive_libs" / f"{lib_name}.py",
|
||||
self.sexp_dir.parent / "sexp_effects" / "primitive_libs" / f"{lib_name}.py",
|
||||
]
|
||||
|
||||
lib_path = None
|
||||
for p in lib_paths:
|
||||
if p.exists():
|
||||
lib_path = p
|
||||
break
|
||||
|
||||
if not lib_path:
|
||||
print(f"Warning: primitive library '{lib_name}' not found", file=sys.stderr)
|
||||
return
|
||||
|
||||
spec = importlib.util.spec_from_file_location(lib_name, lib_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
count = 0
|
||||
for name in dir(module):
|
||||
if name.startswith('prim_'):
|
||||
func = getattr(module, name)
|
||||
prim_name = name[5:]
|
||||
dash_name = prim_name.replace('_', '-')
|
||||
# Register ONLY with namespace (geometry:ripple-displace)
|
||||
self.primitives[f"{lib_name}:{dash_name}"] = func
|
||||
count += 1
|
||||
|
||||
if hasattr(module, 'PRIMITIVES'):
|
||||
prims = getattr(module, 'PRIMITIVES')
|
||||
if isinstance(prims, dict):
|
||||
for name, func in prims.items():
|
||||
# Register ONLY with namespace
|
||||
dash_name = name.replace('_', '-')
|
||||
self.primitives[f"{lib_name}:{dash_name}"] = func
|
||||
count += 1
|
||||
|
||||
print(f"Loaded primitives: {lib_name} ({count} functions)", file=sys.stderr)
|
||||
|
||||
def _load_effect(self, effect_path: Path):
|
||||
"""Load and register an effect from a .sexp file."""
|
||||
if not effect_path.exists():
|
||||
print(f"Warning: effect file not found: {effect_path}", file=sys.stderr)
|
||||
return
|
||||
|
||||
text = effect_path.read_text()
|
||||
ast = parse_all(text)
|
||||
|
||||
for form in ast:
|
||||
if not isinstance(form, list) or not form:
|
||||
continue
|
||||
if not isinstance(form[0], Symbol):
|
||||
continue
|
||||
|
||||
cmd = form[0].name
|
||||
|
||||
if cmd == 'require-primitives':
|
||||
lib_name = form[1] if isinstance(form[1], str) else str(form[1]).strip('"')
|
||||
self._load_primitives(lib_name)
|
||||
|
||||
elif cmd == 'define-effect':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
params = {}
|
||||
body = None
|
||||
i = 2
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword):
|
||||
if form[i].name == 'params' and i + 1 < len(form):
|
||||
for pdef in form[i + 1]:
|
||||
if isinstance(pdef, list) and pdef:
|
||||
pname = pdef[0].name if isinstance(pdef[0], Symbol) else str(pdef[0])
|
||||
pinfo = {'default': 0}
|
||||
j = 1
|
||||
while j < len(pdef):
|
||||
if isinstance(pdef[j], Keyword) and j + 1 < len(pdef):
|
||||
pinfo[pdef[j].name] = pdef[j + 1]
|
||||
j += 2
|
||||
else:
|
||||
j += 1
|
||||
params[pname] = pinfo
|
||||
i += 2
|
||||
else:
|
||||
body = form[i]
|
||||
i += 1
|
||||
|
||||
self.effects[name] = {'params': params, 'body': body}
|
||||
print(f"Effect: {name}", file=sys.stderr)
|
||||
|
||||
elif cmd == 'defmacro':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
params = [p.name if isinstance(p, Symbol) else str(p) for p in form[2]]
|
||||
body = form[3]
|
||||
self.macros[name] = {'params': params, 'body': body}
|
||||
|
||||
elif cmd == 'effect':
|
||||
# Handle (effect name :path "...") in included files - recursive
|
||||
i = 2
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword) and form[i].name == 'path':
|
||||
path = str(form[i + 1]).strip('"')
|
||||
# Resolve relative to the file being loaded
|
||||
full = (effect_path.parent / path).resolve()
|
||||
self._load_effect(full)
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
elif cmd == 'include':
|
||||
# Handle (include :path "...") in included files - recursive
|
||||
i = 1
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword) and form[i].name == 'path':
|
||||
path = str(form[i + 1]).strip('"')
|
||||
full = (effect_path.parent / path).resolve()
|
||||
self._load_effect(full)
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
elif cmd == 'scan':
|
||||
# Handle scans from included files
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
trigger_expr = form[2]
|
||||
init_val, step_expr = {}, None
|
||||
i = 3
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword):
|
||||
if form[i].name == 'init' and i + 1 < len(form):
|
||||
init_val = self._eval(form[i + 1], self.globals)
|
||||
elif form[i].name == 'step' and i + 1 < len(form):
|
||||
step_expr = form[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
self.scans[name] = {
|
||||
'state': dict(init_val) if isinstance(init_val, dict) else {'acc': init_val},
|
||||
'init': init_val,
|
||||
'step': step_expr,
|
||||
'trigger': trigger_expr,
|
||||
}
|
||||
print(f"Scan: {name}", file=sys.stderr)
|
||||
|
||||
def _init(self):
|
||||
"""Initialize from sexp - load primitives, effects, defs, scans."""
|
||||
# Load external config files first (they can override recipe definitions)
|
||||
if self.sources_config:
|
||||
self._load_config_file(self.sources_config)
|
||||
if self.audio_config:
|
||||
self._load_config_file(self.audio_config)
|
||||
|
||||
for form in self.ast:
|
||||
if not isinstance(form, list) or not form:
|
||||
continue
|
||||
if not isinstance(form[0], Symbol):
|
||||
continue
|
||||
|
||||
cmd = form[0].name
|
||||
|
||||
if cmd == 'require-primitives':
|
||||
lib_name = form[1] if isinstance(form[1], str) else str(form[1]).strip('"')
|
||||
self._load_primitives(lib_name)
|
||||
|
||||
elif cmd == 'effect':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
i = 2
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword) and form[i].name == 'path':
|
||||
path = str(form[i + 1]).strip('"')
|
||||
full = (self.sexp_dir / path).resolve()
|
||||
self._load_effect(full)
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
elif cmd == 'include':
|
||||
i = 1
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword) and form[i].name == 'path':
|
||||
path = str(form[i + 1]).strip('"')
|
||||
full = (self.sexp_dir / path).resolve()
|
||||
self._load_effect(full)
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
elif cmd == 'audio-playback':
|
||||
# (audio-playback "path") - set audio file for playback sync
|
||||
# Skip if already set by config file
|
||||
if self.audio_playback is None:
|
||||
path = str(form[1]).strip('"')
|
||||
self.audio_playback = str((self.sexp_dir / path).resolve())
|
||||
print(f"Audio playback: {self.audio_playback}", file=sys.stderr)
|
||||
|
||||
elif cmd == 'def':
|
||||
# (def name expr) - evaluate and store in globals
|
||||
# Skip if already defined by config file
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
if name in self.globals:
|
||||
print(f"Def: {name} (from config, skipped)", file=sys.stderr)
|
||||
continue
|
||||
value = self._eval(form[2], self.globals)
|
||||
self.globals[name] = value
|
||||
print(f"Def: {name}", file=sys.stderr)
|
||||
|
||||
elif cmd == 'defmacro':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
params = [p.name if isinstance(p, Symbol) else str(p) for p in form[2]]
|
||||
body = form[3]
|
||||
self.macros[name] = {'params': params, 'body': body}
|
||||
|
||||
elif cmd == 'scan':
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
trigger_expr = form[2]
|
||||
init_val, step_expr = {}, None
|
||||
i = 3
|
||||
while i < len(form):
|
||||
if isinstance(form[i], Keyword):
|
||||
if form[i].name == 'init' and i + 1 < len(form):
|
||||
init_val = self._eval(form[i + 1], self.globals)
|
||||
elif form[i].name == 'step' and i + 1 < len(form):
|
||||
step_expr = form[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
self.scans[name] = {
|
||||
'state': dict(init_val) if isinstance(init_val, dict) else {'acc': init_val},
|
||||
'init': init_val,
|
||||
'step': step_expr,
|
||||
'trigger': trigger_expr,
|
||||
}
|
||||
print(f"Scan: {name}", file=sys.stderr)
|
||||
|
||||
elif cmd == 'frame':
|
||||
self.frame_pipeline = form[1] if len(form) > 1 else None
|
||||
|
||||
def _eval(self, expr, env: dict) -> Any:
|
||||
"""Evaluate an expression."""
|
||||
|
||||
# Primitives
|
||||
if isinstance(expr, (int, float)):
|
||||
return expr
|
||||
if isinstance(expr, str):
|
||||
return expr
|
||||
if isinstance(expr, bool):
|
||||
return expr
|
||||
|
||||
if isinstance(expr, Symbol):
|
||||
name = expr.name
|
||||
# Built-in constants
|
||||
if name == 'pi':
|
||||
return math.pi
|
||||
if name == 'true':
|
||||
return True
|
||||
if name == 'false':
|
||||
return False
|
||||
if name == 'nil':
|
||||
return None
|
||||
# Environment lookup
|
||||
if name in env:
|
||||
return env[name]
|
||||
# Global lookup
|
||||
if name in self.globals:
|
||||
return self.globals[name]
|
||||
# Scan state lookup
|
||||
if name in self.scans:
|
||||
return self.scans[name]['state']
|
||||
raise NameError(f"Undefined variable: {name}")
|
||||
|
||||
if isinstance(expr, Keyword):
|
||||
return expr.name
|
||||
|
||||
if not isinstance(expr, list) or not expr:
|
||||
return expr
|
||||
|
||||
# Dict literal {:key val ...}
|
||||
if isinstance(expr[0], Keyword):
|
||||
result = {}
|
||||
i = 0
|
||||
while i < len(expr):
|
||||
if isinstance(expr[i], Keyword):
|
||||
result[expr[i].name] = self._eval(expr[i + 1], env) if i + 1 < len(expr) else None
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
return result
|
||||
|
||||
head = expr[0]
|
||||
if not isinstance(head, Symbol):
|
||||
return [self._eval(e, env) for e in expr]
|
||||
|
||||
op = head.name
|
||||
args = expr[1:]
|
||||
|
||||
# Check for closure call
|
||||
if op in env:
|
||||
val = env[op]
|
||||
if isinstance(val, dict) and val.get('_type') == 'closure':
|
||||
closure = val
|
||||
closure_env = dict(closure['env'])
|
||||
for i, pname in enumerate(closure['params']):
|
||||
closure_env[pname] = self._eval(args[i], env) if i < len(args) else None
|
||||
return self._eval(closure['body'], closure_env)
|
||||
|
||||
if op in self.globals:
|
||||
val = self.globals[op]
|
||||
if isinstance(val, dict) and val.get('_type') == 'closure':
|
||||
closure = val
|
||||
closure_env = dict(closure['env'])
|
||||
for i, pname in enumerate(closure['params']):
|
||||
closure_env[pname] = self._eval(args[i], env) if i < len(args) else None
|
||||
return self._eval(closure['body'], closure_env)
|
||||
|
||||
# Threading macro
|
||||
if op == '->':
|
||||
result = self._eval(args[0], env)
|
||||
for form in args[1:]:
|
||||
if isinstance(form, list) and form:
|
||||
new_form = [form[0], result] + form[1:]
|
||||
result = self._eval(new_form, env)
|
||||
else:
|
||||
result = self._eval([form, result], env)
|
||||
return result
|
||||
|
||||
# === Binding ===
|
||||
|
||||
if op == 'bind':
|
||||
scan_name = args[0].name if isinstance(args[0], Symbol) else str(args[0])
|
||||
if scan_name in self.scans:
|
||||
state = self.scans[scan_name]['state']
|
||||
if len(args) > 1:
|
||||
key = args[1].name if isinstance(args[1], Keyword) else str(args[1])
|
||||
return state.get(key, 0)
|
||||
return state
|
||||
return 0
|
||||
|
||||
# === Arithmetic ===
|
||||
|
||||
if op == '+':
|
||||
return sum(self._eval(a, env) for a in args)
|
||||
if op == '-':
|
||||
vals = [self._eval(a, env) for a in args]
|
||||
return vals[0] - sum(vals[1:]) if len(vals) > 1 else -vals[0]
|
||||
if op == '*':
|
||||
result = 1
|
||||
for a in args:
|
||||
result *= self._eval(a, env)
|
||||
return result
|
||||
if op == '/':
|
||||
vals = [self._eval(a, env) for a in args]
|
||||
return vals[0] / vals[1] if len(vals) > 1 and vals[1] != 0 else 0
|
||||
if op == 'mod':
|
||||
vals = [self._eval(a, env) for a in args]
|
||||
return vals[0] % vals[1] if len(vals) > 1 and vals[1] != 0 else 0
|
||||
|
||||
# === Comparison ===
|
||||
|
||||
if op == '<':
|
||||
return self._eval(args[0], env) < self._eval(args[1], env)
|
||||
if op == '>':
|
||||
return self._eval(args[0], env) > self._eval(args[1], env)
|
||||
if op == '=':
|
||||
return self._eval(args[0], env) == self._eval(args[1], env)
|
||||
if op == '<=':
|
||||
return self._eval(args[0], env) <= self._eval(args[1], env)
|
||||
if op == '>=':
|
||||
return self._eval(args[0], env) >= self._eval(args[1], env)
|
||||
|
||||
if op == 'and':
|
||||
for arg in args:
|
||||
if not self._eval(arg, env):
|
||||
return False
|
||||
return True
|
||||
|
||||
if op == 'or':
|
||||
result = False
|
||||
for arg in args:
|
||||
result = self._eval(arg, env)
|
||||
if result:
|
||||
return result
|
||||
return result
|
||||
|
||||
if op == 'not':
|
||||
return not self._eval(args[0], env)
|
||||
|
||||
# === Logic ===
|
||||
|
||||
if op == 'if':
|
||||
cond = self._eval(args[0], env)
|
||||
if cond:
|
||||
return self._eval(args[1], env)
|
||||
return self._eval(args[2], env) if len(args) > 2 else None
|
||||
|
||||
if op == 'cond':
|
||||
i = 0
|
||||
while i < len(args) - 1:
|
||||
pred = self._eval(args[i], env)
|
||||
if pred:
|
||||
return self._eval(args[i + 1], env)
|
||||
i += 2
|
||||
return None
|
||||
|
||||
if op == 'lambda':
|
||||
params = args[0]
|
||||
body = args[1]
|
||||
param_names = [p.name if isinstance(p, Symbol) else str(p) for p in params]
|
||||
return {'_type': 'closure', 'params': param_names, 'body': body, 'env': dict(env)}
|
||||
|
||||
if op == 'let' or op == 'let*':
|
||||
bindings = args[0]
|
||||
body = args[1]
|
||||
new_env = dict(env)
|
||||
|
||||
if bindings and isinstance(bindings[0], list):
|
||||
for binding in bindings:
|
||||
if isinstance(binding, list) and len(binding) >= 2:
|
||||
name = binding[0].name if isinstance(binding[0], Symbol) else str(binding[0])
|
||||
val = self._eval(binding[1], new_env)
|
||||
new_env[name] = val
|
||||
else:
|
||||
i = 0
|
||||
while i < len(bindings):
|
||||
name = bindings[i].name if isinstance(bindings[i], Symbol) else str(bindings[i])
|
||||
val = self._eval(bindings[i + 1], new_env)
|
||||
new_env[name] = val
|
||||
i += 2
|
||||
return self._eval(body, new_env)
|
||||
|
||||
# === Dict ===
|
||||
|
||||
if op == 'dict':
|
||||
result = {}
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if isinstance(args[i], Keyword):
|
||||
key = args[i].name
|
||||
val = self._eval(args[i + 1], env) if i + 1 < len(args) else None
|
||||
result[key] = val
|
||||
i += 2
|
||||
else:
|
||||
i += 1
|
||||
return result
|
||||
|
||||
if op == 'get':
|
||||
obj = self._eval(args[0], env)
|
||||
key = args[1].name if isinstance(args[1], Keyword) else self._eval(args[1], env)
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, 0)
|
||||
return 0
|
||||
|
||||
# === List ===
|
||||
|
||||
if op == 'list':
|
||||
return [self._eval(a, env) for a in args]
|
||||
|
||||
if op == 'quote':
|
||||
return args[0] if args else None
|
||||
|
||||
if op == 'nth':
|
||||
lst = self._eval(args[0], env)
|
||||
idx = int(self._eval(args[1], env))
|
||||
if isinstance(lst, (list, tuple)) and 0 <= idx < len(lst):
|
||||
return lst[idx]
|
||||
return None
|
||||
|
||||
if op == 'len':
|
||||
val = self._eval(args[0], env)
|
||||
return len(val) if hasattr(val, '__len__') else 0
|
||||
|
||||
if op == 'map':
|
||||
seq = self._eval(args[0], env)
|
||||
fn = self._eval(args[1], env)
|
||||
if not isinstance(seq, (list, tuple)):
|
||||
return []
|
||||
# Handle closure (lambda from sexp)
|
||||
if isinstance(fn, dict) and fn.get('_type') == 'closure':
|
||||
results = []
|
||||
for item in seq:
|
||||
closure_env = dict(fn['env'])
|
||||
if fn['params']:
|
||||
closure_env[fn['params'][0]] = item
|
||||
results.append(self._eval(fn['body'], closure_env))
|
||||
return results
|
||||
# Handle Python callable
|
||||
if callable(fn):
|
||||
return [fn(item) for item in seq]
|
||||
return []
|
||||
|
||||
# === Effects ===
|
||||
|
||||
if op in self.effects:
|
||||
effect = self.effects[op]
|
||||
effect_env = dict(env)
|
||||
|
||||
param_names = list(effect['params'].keys())
|
||||
for pname, pdef in effect['params'].items():
|
||||
effect_env[pname] = pdef.get('default', 0)
|
||||
|
||||
positional_idx = 0
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if isinstance(args[i], Keyword):
|
||||
pname = args[i].name
|
||||
if pname in effect['params'] and i + 1 < len(args):
|
||||
effect_env[pname] = self._eval(args[i + 1], env)
|
||||
i += 2
|
||||
else:
|
||||
val = self._eval(args[i], env)
|
||||
if positional_idx == 0:
|
||||
effect_env['frame'] = val
|
||||
elif positional_idx - 1 < len(param_names):
|
||||
effect_env[param_names[positional_idx - 1]] = val
|
||||
positional_idx += 1
|
||||
i += 1
|
||||
|
||||
return self._eval(effect['body'], effect_env)
|
||||
|
||||
# === Primitives ===
|
||||
|
||||
if op in self.primitives:
|
||||
prim_func = self.primitives[op]
|
||||
evaluated_args = []
|
||||
kwargs = {}
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if isinstance(args[i], Keyword):
|
||||
k = args[i].name
|
||||
v = self._eval(args[i + 1], env) if i + 1 < len(args) else None
|
||||
kwargs[k] = v
|
||||
i += 2
|
||||
else:
|
||||
evaluated_args.append(self._eval(args[i], env))
|
||||
i += 1
|
||||
try:
|
||||
if kwargs:
|
||||
return prim_func(*evaluated_args, **kwargs)
|
||||
return prim_func(*evaluated_args)
|
||||
except Exception as e:
|
||||
print(f"Primitive {op} error: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# === Macros (function-like: args evaluated before binding) ===
|
||||
|
||||
if op in self.macros:
|
||||
macro = self.macros[op]
|
||||
macro_env = dict(env)
|
||||
for i, pname in enumerate(macro['params']):
|
||||
# Evaluate args in calling environment before binding
|
||||
macro_env[pname] = self._eval(args[i], env) if i < len(args) else None
|
||||
return self._eval(macro['body'], macro_env)
|
||||
|
||||
# Underscore variant lookup
|
||||
prim_name = op.replace('-', '_')
|
||||
if prim_name in self.primitives:
|
||||
prim_func = self.primitives[prim_name]
|
||||
evaluated_args = []
|
||||
kwargs = {}
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if isinstance(args[i], Keyword):
|
||||
k = args[i].name.replace('-', '_')
|
||||
v = self._eval(args[i + 1], env) if i + 1 < len(args) else None
|
||||
kwargs[k] = v
|
||||
i += 2
|
||||
else:
|
||||
evaluated_args.append(self._eval(args[i], env))
|
||||
i += 1
|
||||
|
||||
try:
|
||||
if kwargs:
|
||||
return prim_func(*evaluated_args, **kwargs)
|
||||
return prim_func(*evaluated_args)
|
||||
except Exception as e:
|
||||
print(f"Primitive {op} error: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Unknown - return as-is
|
||||
return expr
|
||||
|
||||
def _step_scans(self, ctx: Context, env: dict):
|
||||
"""Step scans based on trigger evaluation."""
|
||||
for name, scan in self.scans.items():
|
||||
trigger_expr = scan['trigger']
|
||||
|
||||
# Evaluate trigger in context
|
||||
should_step = self._eval(trigger_expr, env)
|
||||
|
||||
if should_step:
|
||||
state = scan['state']
|
||||
step_env = dict(state)
|
||||
step_env.update(env)
|
||||
|
||||
new_state = self._eval(scan['step'], step_env)
|
||||
if isinstance(new_state, dict):
|
||||
scan['state'] = new_state
|
||||
else:
|
||||
scan['state'] = {'acc': new_state}
|
||||
|
||||
def run(self, duration: float = None, output: str = "pipe"):
|
||||
"""Run the streaming pipeline."""
|
||||
# Import output classes - handle both package and direct execution
|
||||
try:
|
||||
from .output import PipeOutput, DisplayOutput, FileOutput
|
||||
except ImportError:
|
||||
from output import PipeOutput, DisplayOutput, FileOutput
|
||||
|
||||
self._init()
|
||||
|
||||
if not self.frame_pipeline:
|
||||
print("Error: no (frame ...) pipeline defined", file=sys.stderr)
|
||||
return
|
||||
|
||||
w = self.config.get('width', 720)
|
||||
h = self.config.get('height', 720)
|
||||
fps = self.config.get('fps', 30)
|
||||
|
||||
if duration is None:
|
||||
# Try to get duration from audio if available
|
||||
for name, val in self.globals.items():
|
||||
if hasattr(val, 'duration'):
|
||||
duration = val.duration
|
||||
print(f"Using audio duration: {duration:.1f}s", file=sys.stderr)
|
||||
break
|
||||
else:
|
||||
duration = 60.0
|
||||
|
||||
n_frames = int(duration * fps)
|
||||
frame_time = 1.0 / fps
|
||||
|
||||
print(f"Streaming {n_frames} frames @ {fps}fps", file=sys.stderr)
|
||||
|
||||
# Create context
|
||||
ctx = Context(fps=fps)
|
||||
|
||||
# Output (with optional audio sync)
|
||||
audio = self.audio_playback
|
||||
if output == "pipe":
|
||||
out = PipeOutput(size=(w, h), fps=fps, audio_source=audio)
|
||||
elif output == "preview":
|
||||
out = DisplayOutput(size=(w, h), fps=fps, audio_source=audio)
|
||||
else:
|
||||
out = FileOutput(output, size=(w, h), fps=fps, audio_source=audio)
|
||||
|
||||
try:
|
||||
frame_times = []
|
||||
for frame_num in range(n_frames):
|
||||
if not out.is_open:
|
||||
break
|
||||
|
||||
frame_start = time.time()
|
||||
ctx.t = frame_num * frame_time
|
||||
ctx.frame_num = frame_num
|
||||
|
||||
# Build frame environment with context
|
||||
frame_env = {
|
||||
'ctx': {
|
||||
't': ctx.t,
|
||||
'frame-num': ctx.frame_num,
|
||||
'fps': ctx.fps,
|
||||
},
|
||||
't': ctx.t, # Also expose t directly for convenience
|
||||
'frame-num': ctx.frame_num,
|
||||
}
|
||||
|
||||
# Step scans
|
||||
self._step_scans(ctx, frame_env)
|
||||
|
||||
# Evaluate pipeline
|
||||
result = self._eval(self.frame_pipeline, frame_env)
|
||||
|
||||
if result is not None and hasattr(result, 'shape'):
|
||||
out.write(result, ctx.t)
|
||||
|
||||
frame_elapsed = time.time() - frame_start
|
||||
frame_times.append(frame_elapsed)
|
||||
|
||||
# Progress with timing
|
||||
if frame_num % 30 == 0:
|
||||
pct = 100 * frame_num / n_frames
|
||||
avg_ms = 1000 * sum(frame_times[-30:]) / max(1, len(frame_times[-30:]))
|
||||
target_ms = 1000 * frame_time
|
||||
print(f"\r{pct:5.1f}% [{avg_ms:.0f}ms/frame, target {target_ms:.0f}ms]", end="", file=sys.stderr, flush=True)
|
||||
|
||||
finally:
|
||||
out.close()
|
||||
print("\nDone", file=sys.stderr)
|
||||
|
||||
|
||||
def run_stream(sexp_path: str, duration: float = None, output: str = "pipe", fps: float = None,
|
||||
sources_config: str = None, audio_config: str = None):
|
||||
"""Run a streaming sexp."""
|
||||
interp = StreamInterpreter(sexp_path)
|
||||
if fps:
|
||||
interp.config['fps'] = fps
|
||||
if sources_config:
|
||||
interp.sources_config = Path(sources_config)
|
||||
if audio_config:
|
||||
interp.audio_config = Path(audio_config)
|
||||
interp.run(duration=duration, output=output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="Run streaming sexp (generic interpreter)")
|
||||
parser.add_argument("sexp", help="Path to .sexp file")
|
||||
parser.add_argument("-d", "--duration", type=float, default=None)
|
||||
parser.add_argument("-o", "--output", default="pipe")
|
||||
parser.add_argument("--fps", type=float, default=None)
|
||||
parser.add_argument("--sources", dest="sources_config", help="Path to sources config .sexp file")
|
||||
parser.add_argument("--audio", dest="audio_config", help="Path to audio config .sexp file")
|
||||
args = parser.parse_args()
|
||||
|
||||
run_stream(args.sexp, duration=args.duration, output=args.output, fps=args.fps,
|
||||
sources_config=args.sources_config, audio_config=args.audio_config)
|
||||
Reference in New Issue
Block a user