Add streaming video compositor with sexp interpreter
- New streaming/ module for real-time video processing: - compositor.py: Main streaming compositor with cycle-crossfade - sexp_executor.py: Executes compiled sexp recipes in real-time - sexp_interp.py: Full S-expression interpreter for SLICE_ON Lambda - recipe_adapter.py: Bridges recipes to streaming compositor - sources.py: Video source with ffmpeg streaming - audio.py: Real-time audio analysis (energy, beats) - output.py: Preview (mpv) and file output with audio muxing - New templates/: - cycle-crossfade.sexp: Smooth zoom-based video cycling - process-pair.sexp: Dual-clip processing with effects - Key features: - Videos cycle in input-videos order (not definition order) - Cumulative whole-spin rotation - Zero-weight sources skip processing - Live audio-reactive effects - New effects: blend_multi for weighted layer compositing - Updated primitives and interpreter for streaming compatibility Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
65
templates/cycle-crossfade.sexp
Normal file
65
templates/cycle-crossfade.sexp
Normal file
@@ -0,0 +1,65 @@
|
||||
;; cycle-crossfade template
|
||||
;;
|
||||
;; Generalized cycling zoom-crossfade for any number of video layers.
|
||||
;; Cycles through videos with smooth zoom-based crossfade transitions.
|
||||
;;
|
||||
;; Parameters:
|
||||
;; beat-data - beat analysis node (drives timing)
|
||||
;; input-videos - list of video nodes to cycle through
|
||||
;; init-clen - initial cycle length in beats
|
||||
;;
|
||||
;; NOTE: The parameter is named "input-videos" (not "videos") because
|
||||
;; template substitution replaces param names everywhere in the AST.
|
||||
;; The planner's _expand_slice_on injects env["videos"] at plan time,
|
||||
;; so (len videos) inside the lambda references that injected value.
|
||||
|
||||
(deftemplate cycle-crossfade
|
||||
(beat-data input-videos init-clen)
|
||||
|
||||
(slice-on beat-data
|
||||
:videos input-videos
|
||||
:init {:cycle 0 :beat 0 :clen init-clen}
|
||||
:fn (lambda [acc i start end]
|
||||
(let [beat (get acc "beat")
|
||||
clen (get acc "clen")
|
||||
active (get acc "cycle")
|
||||
n (len videos)
|
||||
phase3 (* beat 3)
|
||||
wt (lambda [p]
|
||||
(let [prev (mod (+ p (- n 1)) n)]
|
||||
(if (= active p)
|
||||
(if (< phase3 clen) 1.0
|
||||
(if (< phase3 (* clen 2))
|
||||
(- 1.0 (* (/ (- phase3 clen) clen) 1.0))
|
||||
0.0))
|
||||
(if (= active prev)
|
||||
(if (< phase3 clen) 0.0
|
||||
(if (< phase3 (* clen 2))
|
||||
(* (/ (- phase3 clen) clen) 1.0)
|
||||
1.0))
|
||||
0.0))))
|
||||
zm (lambda [p]
|
||||
(let [prev (mod (+ p (- n 1)) n)]
|
||||
(if (= active p)
|
||||
;; Active video: normal -> zoom out during transition -> tiny
|
||||
(if (< phase3 clen) 1.0
|
||||
(if (< phase3 (* clen 2))
|
||||
(+ 1.0 (* (/ (- phase3 clen) clen) 1.0))
|
||||
0.1))
|
||||
(if (= active prev)
|
||||
;; Incoming video: tiny -> zoom in during transition -> normal
|
||||
(if (< phase3 clen) 0.1
|
||||
(if (< phase3 (* clen 2))
|
||||
(+ 0.1 (* (/ (- phase3 clen) clen) 0.9))
|
||||
1.0))
|
||||
0.1))))
|
||||
new-acc (if (< (+ beat 1) clen)
|
||||
(dict :cycle active :beat (+ beat 1) :clen clen)
|
||||
(dict :cycle (mod (+ active 1) n) :beat 0
|
||||
:clen (+ 40 (mod (* i 7) 41))))]
|
||||
{:layers (map (lambda [p]
|
||||
{:video p :effects [{:effect zoom :amount (zm p)}]})
|
||||
(range 0 n))
|
||||
:compose {:effect blend_multi :mode "alpha"
|
||||
:weights (map (lambda [p] (wt p)) (range 0 n))}
|
||||
:acc new-acc}))))
|
||||
112
templates/process-pair.sexp
Normal file
112
templates/process-pair.sexp
Normal file
@@ -0,0 +1,112 @@
|
||||
;; process-pair template
|
||||
;;
|
||||
;; Reusable video-pair processor: takes a single video source, creates two
|
||||
;; clips (A and B) with opposite rotations and sporadic effects, blends them,
|
||||
;; and applies a per-pair slow rotation driven by a beat scan.
|
||||
;;
|
||||
;; All sporadic triggers (invert, hue-shift, ascii) and pair-level controls
|
||||
;; (blend opacity, rotation) are defined internally using seed offsets.
|
||||
;;
|
||||
;; Parameters:
|
||||
;; video - source video node
|
||||
;; energy - energy analysis node (drives rotation/zoom amounts)
|
||||
;; beat-data - beat analysis node (drives sporadic triggers)
|
||||
;; rng - RNG object from (make-rng seed) for auto-derived seeds
|
||||
;; rot-dir - initial rotation direction: 1 (clockwise) or -1 (anti-clockwise)
|
||||
;; rot-a/b - rotation ranges for clip A/B (e.g. [0 45])
|
||||
;; zoom-a/b - zoom ranges for clip A/B (e.g. [1 1.5])
|
||||
|
||||
(deftemplate process-pair
|
||||
(video energy beat-data rng rot-dir rot-a rot-b zoom-a zoom-b)
|
||||
|
||||
;; --- Sporadic triggers for clip A ---
|
||||
|
||||
;; Invert: 10% chance per beat, lasts 1-5 beats
|
||||
(def inv-a (scan beat-data :rng rng :init 0
|
||||
:step (if (> acc 0) (- acc 1) (if (< (rand) 0.1) (rand-int 1 5) 0))
|
||||
:emit (if (> acc 0) 1 0)))
|
||||
|
||||
;; Hue shift: 10% chance, random hue 30-330 deg, lasts 1-5 beats
|
||||
(def hue-a (scan beat-data :rng rng :init (dict :rem 0 :hue 0)
|
||||
:step (if (> rem 0)
|
||||
(dict :rem (- rem 1) :hue hue)
|
||||
(if (< (rand) 0.1)
|
||||
(dict :rem (rand-int 1 5) :hue (rand-range 30 330))
|
||||
(dict :rem 0 :hue 0)))
|
||||
:emit (if (> rem 0) hue 0)))
|
||||
|
||||
;; ASCII art: 5% chance, lasts 1-3 beats
|
||||
(def ascii-a (scan beat-data :rng rng :init 0
|
||||
:step (if (> acc 0) (- acc 1) (if (< (rand) 0.05) (rand-int 1 3) 0))
|
||||
:emit (if (> acc 0) 1 0)))
|
||||
|
||||
;; --- Sporadic triggers for clip B (offset seeds) ---
|
||||
|
||||
(def inv-b (scan beat-data :rng rng :init 0
|
||||
:step (if (> acc 0) (- acc 1) (if (< (rand) 0.1) (rand-int 1 5) 0))
|
||||
:emit (if (> acc 0) 1 0)))
|
||||
|
||||
(def hue-b (scan beat-data :rng rng :init (dict :rem 0 :hue 0)
|
||||
:step (if (> rem 0)
|
||||
(dict :rem (- rem 1) :hue hue)
|
||||
(if (< (rand) 0.1)
|
||||
(dict :rem (rand-int 1 5) :hue (rand-range 30 330))
|
||||
(dict :rem 0 :hue 0)))
|
||||
:emit (if (> rem 0) hue 0)))
|
||||
|
||||
(def ascii-b (scan beat-data :rng rng :init 0
|
||||
:step (if (> acc 0) (- acc 1) (if (< (rand) 0.05) (rand-int 1 3) 0))
|
||||
:emit (if (> acc 0) 1 0)))
|
||||
|
||||
;; --- Pair-level controls ---
|
||||
|
||||
;; Internal A/B blend: randomly show A (0), both (0.5), or B (1), every 1-11 beats
|
||||
(def pair-mix (scan beat-data :rng rng
|
||||
:init (dict :rem 0 :opacity 0.5)
|
||||
:step (if (> rem 0)
|
||||
(dict :rem (- rem 1) :opacity opacity)
|
||||
(dict :rem (rand-int 1 11) :opacity (* (rand-int 0 2) 0.5)))
|
||||
:emit opacity))
|
||||
|
||||
;; Per-pair rotation: one full rotation every 20-30 beats, alternating direction
|
||||
(def pair-rot (scan beat-data :rng rng
|
||||
:init (dict :beat 0 :clen 25 :dir rot-dir :angle 0)
|
||||
:step (if (< (+ beat 1) clen)
|
||||
(dict :beat (+ beat 1) :clen clen :dir dir
|
||||
:angle (+ angle (* dir (/ 360 clen))))
|
||||
(dict :beat 0 :clen (rand-int 20 30) :dir (* dir -1)
|
||||
:angle angle))
|
||||
:emit angle))
|
||||
|
||||
;; --- Clip A processing ---
|
||||
(def clip-a (-> video (segment :start 0 :duration (bind energy duration))))
|
||||
(def rotated-a (-> clip-a
|
||||
(effect rotate :angle (bind energy values :range rot-a))
|
||||
(effect zoom :amount (bind energy values :range zoom-a))
|
||||
(effect invert :amount (bind inv-a values))
|
||||
(effect hue_shift :degrees (bind hue-a values))
|
||||
;; ASCII disabled - too slow without GPU
|
||||
;; (effect ascii_art
|
||||
;; :char_size (bind energy values :range [4 32])
|
||||
;; :mix (bind ascii-a values))
|
||||
))
|
||||
|
||||
;; --- Clip B processing ---
|
||||
(def clip-b (-> video (segment :start 0 :duration (bind energy duration))))
|
||||
(def rotated-b (-> clip-b
|
||||
(effect rotate :angle (bind energy values :range rot-b))
|
||||
(effect zoom :amount (bind energy values :range zoom-b))
|
||||
(effect invert :amount (bind inv-b values))
|
||||
(effect hue_shift :degrees (bind hue-b values))
|
||||
;; ASCII disabled - too slow without GPU
|
||||
;; (effect ascii_art
|
||||
;; :char_size (bind energy values :range [4 32])
|
||||
;; :mix (bind ascii-b values))
|
||||
))
|
||||
|
||||
;; --- Blend A+B and apply pair rotation ---
|
||||
(-> rotated-a
|
||||
(effect blend rotated-b
|
||||
:mode "alpha" :opacity (bind pair-mix values) :resize_mode "fit")
|
||||
(effect rotate
|
||||
:angle (bind pair-rot values))))
|
||||
Reference in New Issue
Block a user