art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add IPFS HLS streaming and GPU optimizations

Browse Source 
- Add IPFSHLSOutput class that uploads segments to IPFS as they're created
- Update streaming task to use IPFS HLS output for distributed streaming
- Add /ipfs-stream endpoint to get IPFS playlist URL
- Update /stream endpoint to redirect to IPFS when available
- Add GPU persistence mode (STREAMING_GPU_PERSIST=1) to keep frames on GPU
- Add hardware video decoding (NVDEC) support for faster video processing
- Add GPU-accelerated primitive libraries: blending_gpu, color_ops_gpu, geometry_gpu
- Add streaming_gpu module with GPUFrame class for tracking CPU/GPU data location
- Add Dockerfile.gpu for building GPU-enabled worker image

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
giles
2026-02-03 20:23:16 +00:00
parent 5bc655f8c8
commit 86830019ad
24 changed files with 4025 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
app/routers/runs.py
View File

@@ -227,16 +227,19 @@ async def create_stream_run(
logger.warning(f"Failed to store recipe in cache: {e}")
# Continue anyway - run will still work, just won't appear in /recipes
# Submit Celery task
task = run_stream.delay(
run_id=run_id,
recipe_sexp=request.recipe_sexp,
output_name=request.output_name,
duration=request.duration,
fps=request.fps,
actor_id=actor_id,
sources_sexp=request.sources_sexp,
audio_sexp=request.audio_sexp,
# Submit Celery task to GPU queue for hardware-accelerated rendering
task = run_stream.apply_async(
kwargs=dict(
run_id=run_id,
recipe_sexp=request.recipe_sexp,
output_name=request.output_name,
duration=request.duration,
fps=request.fps,
actor_id=actor_id,
sources_sexp=request.sources_sexp,
audio_sexp=request.audio_sexp,
),
queue='gpu',
)
# Store in database for durability
@@ -396,7 +399,7 @@ async def get_run(
artifacts = []
output_media_type = None
if run.get("output_cid"):
# Detect media type using magic bytes
# Detect media type using magic bytes, fall back to database item_type
output_cid = run["output_cid"]
media_type = None
try:
@@ -408,6 +411,16 @@ async def get_run(
output_media_type = media_type
except Exception:
pass
# Fall back to database item_type if local detection failed
if not media_type:
try:
import database
item_types = await database.get_item_types(output_cid, run.get("actor_id"))
if item_types:
media_type = type_to_mime(item_types[0].get("type"))
output_media_type = media_type
except Exception:
pass
artifacts.append({
"cid": output_cid,
"step_name": "Output",
@@ -963,18 +976,44 @@ async def stream_run_output(
request: Request,
):
"""Stream the video output of a running render.
Returns the partial video file as it's being written,
allowing live preview of the render progress.
For IPFS HLS streams, redirects to the IPFS gateway playlist.
For local HLS streams, redirects to the m3u8 playlist.
For legacy MP4 streams, returns the file directly.
"""
from fastapi.responses import StreamingResponse, FileResponse
from fastapi.responses import StreamingResponse, FileResponse, RedirectResponse
from pathlib import Path
import os
import database
from celery_app import app as celery_app
await database.init_db()
# Check for IPFS HLS streaming first (distributed P2P streaming)
pending = await database.get_pending_run(run_id)
if pending and pending.get("celery_task_id"):
task_id = pending["celery_task_id"]
result = celery_app.AsyncResult(task_id)
if result.ready() and isinstance(result.result, dict):
ipfs_playlist_url = result.result.get("ipfs_playlist_url")
if ipfs_playlist_url:
logger.info(f"Redirecting to IPFS stream: {ipfs_playlist_url}")
return RedirectResponse(url=ipfs_playlist_url, status_code=302)
# Check for the streaming output file in the shared cache
cache_dir = os.environ.get("CACHE_DIR", "/data/cache")
stream_path = Path(cache_dir) / "streaming" / run_id / "output.mp4"
stream_dir = Path(cache_dir) / "streaming" / run_id
# Check for local HLS output
hls_playlist = stream_dir / "stream.m3u8"
if hls_playlist.exists():
# Redirect to the HLS playlist endpoint
return RedirectResponse(
url=f"/runs/{run_id}/hls/stream.m3u8",
status_code=302
)
# Fall back to legacy MP4 streaming
stream_path = stream_dir / "output.mp4"
if not stream_path.exists():
raise HTTPException(404, "Stream not available yet")
@@ -982,7 +1021,6 @@ async def stream_run_output(
if file_size == 0:
raise HTTPException(404, "Stream not ready")
# Return the file with headers that allow streaming of growing file
return FileResponse(
path=str(stream_path),
media_type="video/mp4",
@@ -992,3 +1030,139 @@ async def stream_run_output(
"X-Content-Size": str(file_size),
}
)
@router.get("/{run_id}/hls/{filename:path}")
async def serve_hls_content(
run_id: str,
filename: str,
request: Request,
):
"""Serve HLS playlist and segments for live streaming.
Serves stream.m3u8 (playlist) and segment_*.ts files.
The playlist updates as new segments are rendered.
If files aren't found locally, proxies to the GPU worker (if configured).
"""
from fastapi.responses import FileResponse, StreamingResponse
from pathlib import Path
import os
import httpx
cache_dir = os.environ.get("CACHE_DIR", "/data/cache")
stream_dir = Path(cache_dir) / "streaming" / run_id
file_path = stream_dir / filename
# Security: ensure we're only serving files within stream_dir
try:
file_path_resolved = file_path.resolve()
stream_dir_resolved = stream_dir.resolve()
if stream_dir.exists() and not str(file_path_resolved).startswith(str(stream_dir_resolved)):
raise HTTPException(403, "Invalid path")
except Exception:
pass # Allow proxy fallback
# Determine content type
if filename.endswith(".m3u8"):
media_type = "application/vnd.apple.mpegurl"
headers = {
"Cache-Control": "no-cache, no-store, must-revalidate",
"Access-Control-Allow-Origin": "*",
}
elif filename.endswith(".ts"):
media_type = "video/mp2t"
headers = {
"Cache-Control": "public, max-age=3600",
"Access-Control-Allow-Origin": "*",
}
else:
raise HTTPException(400, "Invalid file type")
# Try local file first
if file_path.exists():
return FileResponse(
path=str(file_path),
media_type=media_type,
headers=headers,
)
# Fallback: proxy to GPU worker if configured
gpu_worker_url = os.environ.get("GPU_WORKER_STREAM_URL")
if gpu_worker_url:
# Proxy request to GPU worker
proxy_url = f"{gpu_worker_url}/{run_id}/{filename}"
try:
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.get(proxy_url)
if resp.status_code == 200:
return StreamingResponse(
content=iter([resp.content]),
media_type=media_type,
headers=headers,
)
except Exception as e:
logger.warning(f"GPU worker proxy failed: {e}")
raise HTTPException(404, f"File not found: {filename}")
@router.get("/{run_id}/ipfs-stream")
async def get_ipfs_stream_info(run_id: str, request: Request):
"""Get IPFS streaming info for a run.
Returns the IPFS playlist URL and segment info if available.
This allows clients to stream directly from IPFS gateways.
"""
from celery_app import app as celery_app
import database
import os
await database.init_db()
# Try to get pending run to find the Celery task ID
pending = await database.get_pending_run(run_id)
if not pending:
# Try completed runs
run = await database.get_run_cache(run_id)
if not run:
raise HTTPException(404, "Run not found")
# For completed runs, check if we have IPFS info stored
ipfs_cid = run.get("ipfs_cid")
if ipfs_cid:
gateway = os.environ.get("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs")
return {
"run_id": run_id,
"status": "completed",
"ipfs_video_url": f"{gateway}/{ipfs_cid}",
}
raise HTTPException(404, "No IPFS stream info available")
task_id = pending.get("celery_task_id")
if not task_id:
raise HTTPException(404, "No task ID for this run")
# Get the Celery task result
result = celery_app.AsyncResult(task_id)
if result.ready():
# Task is complete - check the result for IPFS playlist info
task_result = result.result
if isinstance(task_result, dict):
ipfs_playlist_cid = task_result.get("ipfs_playlist_cid")
ipfs_playlist_url = task_result.get("ipfs_playlist_url")
if ipfs_playlist_url:
return {
"run_id": run_id,
"status": "completed",
"ipfs_playlist_cid": ipfs_playlist_cid,
"ipfs_playlist_url": ipfs_playlist_url,
"segment_count": task_result.get("ipfs_segment_count", 0),
}
# Task is still running or no IPFS info available
return {
"run_id": run_id,
"status": pending.get("status", "pending"),
"message": "IPFS streaming info not yet available"
}