art-dag/test
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add streaming video compositor with sexp interpreter

Browse Source 
- New streaming/ module for real-time video processing:
  - compositor.py: Main streaming compositor with cycle-crossfade
  - sexp_executor.py: Executes compiled sexp recipes in real-time
  - sexp_interp.py: Full S-expression interpreter for SLICE_ON Lambda
  - recipe_adapter.py: Bridges recipes to streaming compositor
  - sources.py: Video source with ffmpeg streaming
  - audio.py: Real-time audio analysis (energy, beats)
  - output.py: Preview (mpv) and file output with audio muxing

- New templates/:
  - cycle-crossfade.sexp: Smooth zoom-based video cycling
  - process-pair.sexp: Dual-clip processing with effects

- Key features:
  - Videos cycle in input-videos order (not definition order)
  - Cumulative whole-spin rotation
  - Zero-weight sources skip processing
  - Live audio-reactive effects

- New effects: blend_multi for weighted layer compositing
- Updated primitives and interpreter for streaming compatibility

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
gilesb
2026-01-29 01:27:39 +00:00
parent 17e3e23f06
commit d241e2a663
31 changed files with 5143 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
effects/ascii_alternating_rotate.sexp
View File

@@ -14,6 +14,8 @@
:desc "Number of character columns")
(rotation_scale :type float :default 60 :range [0 180]
:desc "Max rotation in degrees")
(duration :type float :default 10 :range [1 300]
:desc "Clip duration in seconds")
)
;; Registry
@@ -29,7 +31,7 @@
;; Stage 1: Analysis
(stage :analyze
:outputs [energy-data]
(def audio-clip (-> audio (segment :start 60 :duration 10)))
(def audio-clip (-> audio (segment :start 60 :duration duration)))
(def energy-data (-> audio-clip (analyze energy))))
;; Stage 2: Process
@@ -37,8 +39,8 @@
:requires [:analyze]
:inputs [energy-data]
:outputs [result audio-clip]
(def clip (-> video (segment :start 0 :duration 10)))
(def audio-clip (-> audio (segment :start 60 :duration 10)))
(def clip (-> video (segment :start 0 :duration duration)))
(def audio-clip (-> audio (segment :start 60 :duration duration)))
(def result (-> clip
(effect ascii_fx_zone

10
effects/ascii_dual_blend.sexp
View File

@@ -19,6 +19,8 @@
:desc "Blend opacity (0=video-a only, 1=video-b only)")
(blend_mode :type string :default "overlay"
:desc "Blend mode: alpha, add, multiply, screen, overlay, difference")
(duration :type float :default 10 :range [1 300]
:desc "Clip duration in seconds")
)
;; Registry - effects and analyzers
@@ -35,7 +37,7 @@
;; Stage 1: Analysis
(stage :analyze
:outputs [energy-data]
(def audio-clip (-> audio (segment :start 60 :duration 10)))
(def audio-clip (-> audio (segment :start 60 :duration duration)))
(def energy-data (-> audio-clip (analyze energy))))
;; Stage 2: Process both videos
@@ -45,10 +47,10 @@
:outputs [blended audio-clip]
;; Get audio clip for final mux
(def audio-clip (-> audio (segment :start 60 :duration 10)))
(def audio-clip (-> audio (segment :start 60 :duration duration)))
;; Process video A with ASCII effect
(def clip-a (-> video-a (segment :start 0 :duration 10)))
(def clip-a (-> video-a (segment :start 0 :duration duration)))
(def ascii-a (-> clip-a
(effect ascii_fx_zone
:cols cols
@@ -66,7 +68,7 @@
(- 1 (get zone "row-norm")))))))))))
;; Process video B with ASCII effect
(def clip-b (-> video-b (segment :start 0 :duration 10)))
(def clip-b (-> video-b (segment :start 0 :duration duration)))
(def ascii-b (-> clip-b
(effect ascii_fx_zone
:cols cols

178
effects/quick_test.sexp Normal file
View File

@@ -0,0 +1,178 @@
;; Quick Test Recipe
;;
;; Cycles between three video pairs (monday, duel, ecstacy) with smooth zoom-based crossfade.
;; Each pair is two copies of the same source with opposite rotations.
;; Each pair rotates in its own direction (per-pair rotation via template).
;; Cycle: active pair plays -> crossfade -> new pair plays -> advance and repeat.
;; Ripple drops on the final combined output only.
(recipe "quick_test"
:version "1.0"
:description "Cycling crossfade between three video pairs"
:minimal-primitives true
:encoding (:codec "libx264" :crf 23 :preset "ultrafast" :audio-codec "aac" :fps 30)
:params (
(audio_start :type float :default 60 :range [0 300]
:desc "Audio start time in seconds")
(audio_duration :type float :default nil
:desc "Audio duration (nil = full remaining)")
(blend_opacity :type float :default 0.5 :range [0 1]
:desc "Blend opacity within each pair")
(seed :type int :default 42 :desc "Master random seed")
)
;; Registry
(effect rotate :path "../sexp_effects/effects/rotate.sexp")
(effect zoom :path "../sexp_effects/effects/zoom.sexp")
(effect blend :path "../sexp_effects/effects/blend.sexp")
(effect invert :path "../sexp_effects/effects/invert.sexp")
(effect hue_shift :path "../sexp_effects/effects/hue_shift.sexp")
(effect ascii_art :path "../sexp_effects/effects/ascii_art.sexp")
(effect ripple :path "../sexp_effects/effects/ripple.sexp")
(effect blend_multi :path "../sexp_effects/effects/blend_multi.sexp")
(analyzer energy :path "../../artdag-analyzers/energy/analyzer.py")
(analyzer beats :path "../../artdag-analyzers/beats/analyzer.py")
;; Sources
(def video-1 (source :path "../1.mp4"))
(def video-2 (source :path "../2.webm"))
(def video-4 (source :path "../4.mp4"))
(def video-5 (source :path "../5.mp4"))
(def video-a (source :path "../monday.webm"))
(def video-b (source :path "../escher.webm"))
(def video-c (source :path "../dopple.webm"))
(def video-d (source :path "../disruptors.webm"))
(def video-e (source :path "../ecstacy.mp4"))
(def audio (source :path "../dizzy.mp3"))
;; Templates: reusable video-pair processor and cycle-crossfade
(include :path "../templates/process-pair.sexp")
(include :path "../templates/cycle-crossfade.sexp")
;; Unified RNG: auto-derives unique seeds for all scans
(def rng (make-rng seed))
;; Stage 1: Analysis - energy, beats, and global-level scans
(stage :analyze
:outputs [energy-data beat-data whole-spin
ripple-gate ripple-cx ripple-cy]
(def audio-clip (-> audio (segment :start audio_start :duration audio_duration)))
(def energy-data (-> audio-clip (analyze energy)))
(def beat-data (-> audio-clip (analyze beats)))
;; --- Whole-video continuous spin: cumulative rotation that reverses direction periodically ---
(def whole-spin (scan beat-data :rng rng
:init (dict :beat 0 :clen 25 :dir 1 :angle 0)
:step (if (< (+ beat 1) clen)
(dict :beat (+ beat 1) :clen clen :dir dir
:angle (+ angle (* dir (/ 360 clen))))
(dict :beat 0 :clen (rand-int 20 30) :dir (* dir -1)
:angle angle))
:emit angle))
;; --- Ripple drops on final output ---
(def ripple (scan beat-data :rng rng
:init (dict :rem 0 :cx 0.5 :cy 0.5)
:step (if (> rem 0)
(dict :rem (- rem 1) :cx cx :cy cy)
(if (< (rand) 0.05)
(dict :rem (rand-int 1 20) :cx (rand-range 0.1 0.9) :cy (rand-range 0.1 0.9))
(dict :rem 0 :cx 0.5 :cy 0.5)))
:emit {:gate (if (> rem 0) 1 0) :cx cx :cy cy})))
;; Stage 2: Process videos via template
;; Per-pair scans (inv/hue/ascii triggers, pair-mix, pair-rot) are now
;; defined inside the process-pair template using seed offsets.
(stage :process
:requires [:analyze]
:inputs [energy-data beat-data whole-spin
ripple-gate ripple-cx ripple-cy]
:outputs [final-video audio-clip]
;; Re-segment audio for final mux
(def audio-clip (-> audio (segment :start audio_start :duration audio_duration)))
;; --- Process each pair via template ---
(def monday-blend (process-pair
:video video-a :energy energy-data :beat-data beat-data
:rng rng :rot-dir -1
:rot-a [0 45] :rot-b [0 -45]
:zoom-a [1 1.5] :zoom-b [1 0.5]))
(def escher-blend (process-pair
:video video-b :energy energy-data :beat-data beat-data
:rng rng :rot-dir 1
:rot-a [0 45] :rot-b [0 -45]
:zoom-a [1 1.5] :zoom-b [1 0.5]))
(def duel-blend (process-pair
:video video-d :energy energy-data :beat-data beat-data
:rng rng :rot-dir -1
:rot-a [0 -45] :rot-b [0 45]
:zoom-a [1 0.5] :zoom-b [1 1.5]))
(def blend-2 (process-pair
:video video-2 :energy energy-data :beat-data beat-data
:rng rng :rot-dir 1
:rot-a [0 45] :rot-b [0 -45]
:zoom-a [1 1.5] :zoom-b [1 0.5]))
(def dopple-blend (process-pair
:video video-c :energy energy-data :beat-data beat-data
:rng rng :rot-dir -1
:rot-a [0 -45] :rot-b [0 45]
:zoom-a [1 0.5] :zoom-b [1 1.5]))
(def blend-4 (process-pair
:video video-4 :energy energy-data :beat-data beat-data
:rng rng :rot-dir -1
:rot-a [0 45] :rot-b [0 -45]
:zoom-a [1 1.5] :zoom-b [1 0.5]))
(def ext-blend (process-pair
:video video-e :energy energy-data :beat-data beat-data
:rng rng :rot-dir 1
:rot-a [0 30] :rot-b [0 -30]
:zoom-a [1 1.3] :zoom-b [1 0.7]))
(def blend-5 (process-pair
:video video-5 :energy energy-data :beat-data beat-data
:rng rng :rot-dir 1
:rot-a [0 45] :rot-b [0 -45]
:zoom-a [1 1.5] :zoom-b [1 0.5]))
;; --- Cycle zoom + crossfade via template ---
(def combined (cycle-crossfade
:beat-data beat-data
:input-videos [monday-blend escher-blend blend-2 duel-blend blend-4 ext-blend dopple-blend blend-5]
:init-clen 60))
;; --- Final output: sporadic spin + ripple ---
(def final-video (-> combined
(effect rotate :angle (bind whole-spin values))
(effect ripple
:amplitude (* (bind ripple-gate values) (bind energy-data values :range [5 50]))
:center_x (bind ripple-cx values)
:center_y (bind ripple-cy values)
:frequency 8
:decay 2
:speed 5))))
;; Stage 3: Output
(stage :output
:requires [:process]
:inputs [final-video audio-clip]
(mux final-video audio-clip)))