art-dag/celery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add IPFS HLS streaming and GPU optimizations

Browse Source 
- Add IPFSHLSOutput class that uploads segments to IPFS as they're created
- Update streaming task to use IPFS HLS output for distributed streaming
- Add /ipfs-stream endpoint to get IPFS playlist URL
- Update /stream endpoint to redirect to IPFS when available
- Add GPU persistence mode (STREAMING_GPU_PERSIST=1) to keep frames on GPU
- Add hardware video decoding (NVDEC) support for faster video processing
- Add GPU-accelerated primitive libraries: blending_gpu, color_ops_gpu, geometry_gpu
- Add streaming_gpu module with GPUFrame class for tracking CPU/GPU data location
- Add Dockerfile.gpu for building GPU-enabled worker image

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
giles
2026-02-03 20:23:16 +00:00
parent 5bc655f8c8
commit 86830019ad
24 changed files with 4025 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
app/routers/runs.py
View File

@@ -227,16 +227,19 @@ async def create_stream_run(
logger.warning(f"Failed to store recipe in cache: {e}")
# Continue anyway - run will still work, just won't appear in /recipes
# Submit Celery task
task = run_stream.delay(
run_id=run_id,
recipe_sexp=request.recipe_sexp,
output_name=request.output_name,
duration=request.duration,
fps=request.fps,
actor_id=actor_id,
sources_sexp=request.sources_sexp,
audio_sexp=request.audio_sexp,
# Submit Celery task to GPU queue for hardware-accelerated rendering
task = run_stream.apply_async(
kwargs=dict(
run_id=run_id,
recipe_sexp=request.recipe_sexp,
output_name=request.output_name,
duration=request.duration,
fps=request.fps,
actor_id=actor_id,
sources_sexp=request.sources_sexp,
audio_sexp=request.audio_sexp,
),
queue='gpu',
)
# Store in database for durability
@@ -396,7 +399,7 @@ async def get_run(
artifacts = []
output_media_type = None
if run.get("output_cid"):
# Detect media type using magic bytes
# Detect media type using magic bytes, fall back to database item_type
output_cid = run["output_cid"]
media_type = None
try:
@@ -408,6 +411,16 @@ async def get_run(
output_media_type = media_type
except Exception:
pass
# Fall back to database item_type if local detection failed
if not media_type:
try:
import database
item_types = await database.get_item_types(output_cid, run.get("actor_id"))
if item_types:
media_type = type_to_mime(item_types[0].get("type"))
output_media_type = media_type
except Exception:
pass
artifacts.append({
"cid": output_cid,
"step_name": "Output",
@@ -963,18 +976,44 @@ async def stream_run_output(
request: Request,
):
"""Stream the video output of a running render.
Returns the partial video file as it's being written,
allowing live preview of the render progress.
For IPFS HLS streams, redirects to the IPFS gateway playlist.
For local HLS streams, redirects to the m3u8 playlist.
For legacy MP4 streams, returns the file directly.
"""
from fastapi.responses import StreamingResponse, FileResponse
from fastapi.responses import StreamingResponse, FileResponse, RedirectResponse
from pathlib import Path
import os
import database
from celery_app import app as celery_app
await database.init_db()
# Check for IPFS HLS streaming first (distributed P2P streaming)
pending = await database.get_pending_run(run_id)
if pending and pending.get("celery_task_id"):
task_id = pending["celery_task_id"]
result = celery_app.AsyncResult(task_id)
if result.ready() and isinstance(result.result, dict):
ipfs_playlist_url = result.result.get("ipfs_playlist_url")
if ipfs_playlist_url:
logger.info(f"Redirecting to IPFS stream: {ipfs_playlist_url}")
return RedirectResponse(url=ipfs_playlist_url, status_code=302)
# Check for the streaming output file in the shared cache
cache_dir = os.environ.get("CACHE_DIR", "/data/cache")
stream_path = Path(cache_dir) / "streaming" / run_id / "output.mp4"
stream_dir = Path(cache_dir) / "streaming" / run_id
# Check for local HLS output
hls_playlist = stream_dir / "stream.m3u8"
if hls_playlist.exists():
# Redirect to the HLS playlist endpoint
return RedirectResponse(
url=f"/runs/{run_id}/hls/stream.m3u8",
status_code=302
)
# Fall back to legacy MP4 streaming
stream_path = stream_dir / "output.mp4"
if not stream_path.exists():
raise HTTPException(404, "Stream not available yet")
@@ -982,7 +1021,6 @@ async def stream_run_output(
if file_size == 0:
raise HTTPException(404, "Stream not ready")
# Return the file with headers that allow streaming of growing file
return FileResponse(
path=str(stream_path),
media_type="video/mp4",
@@ -992,3 +1030,139 @@ async def stream_run_output(
"X-Content-Size": str(file_size),
}
)
@router.get("/{run_id}/hls/{filename:path}")
async def serve_hls_content(
run_id: str,
filename: str,
request: Request,
):
"""Serve HLS playlist and segments for live streaming.
Serves stream.m3u8 (playlist) and segment_*.ts files.
The playlist updates as new segments are rendered.
If files aren't found locally, proxies to the GPU worker (if configured).
"""
from fastapi.responses import FileResponse, StreamingResponse
from pathlib import Path
import os
import httpx
cache_dir = os.environ.get("CACHE_DIR", "/data/cache")
stream_dir = Path(cache_dir) / "streaming" / run_id
file_path = stream_dir / filename
# Security: ensure we're only serving files within stream_dir
try:
file_path_resolved = file_path.resolve()
stream_dir_resolved = stream_dir.resolve()
if stream_dir.exists() and not str(file_path_resolved).startswith(str(stream_dir_resolved)):
raise HTTPException(403, "Invalid path")
except Exception:
pass # Allow proxy fallback
# Determine content type
if filename.endswith(".m3u8"):
media_type = "application/vnd.apple.mpegurl"
headers = {
"Cache-Control": "no-cache, no-store, must-revalidate",
"Access-Control-Allow-Origin": "*",
}
elif filename.endswith(".ts"):
media_type = "video/mp2t"
headers = {
"Cache-Control": "public, max-age=3600",
"Access-Control-Allow-Origin": "*",
}
else:
raise HTTPException(400, "Invalid file type")
# Try local file first
if file_path.exists():
return FileResponse(
path=str(file_path),
media_type=media_type,
headers=headers,
)
# Fallback: proxy to GPU worker if configured
gpu_worker_url = os.environ.get("GPU_WORKER_STREAM_URL")
if gpu_worker_url:
# Proxy request to GPU worker
proxy_url = f"{gpu_worker_url}/{run_id}/{filename}"
try:
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.get(proxy_url)
if resp.status_code == 200:
return StreamingResponse(
content=iter([resp.content]),
media_type=media_type,
headers=headers,
)
except Exception as e:
logger.warning(f"GPU worker proxy failed: {e}")
raise HTTPException(404, f"File not found: {filename}")
@router.get("/{run_id}/ipfs-stream")
async def get_ipfs_stream_info(run_id: str, request: Request):
"""Get IPFS streaming info for a run.
Returns the IPFS playlist URL and segment info if available.
This allows clients to stream directly from IPFS gateways.
"""
from celery_app import app as celery_app
import database
import os
await database.init_db()
# Try to get pending run to find the Celery task ID
pending = await database.get_pending_run(run_id)
if not pending:
# Try completed runs
run = await database.get_run_cache(run_id)
if not run:
raise HTTPException(404, "Run not found")
# For completed runs, check if we have IPFS info stored
ipfs_cid = run.get("ipfs_cid")
if ipfs_cid:
gateway = os.environ.get("IPFS_GATEWAY_URL", "https://ipfs.io/ipfs")
return {
"run_id": run_id,
"status": "completed",
"ipfs_video_url": f"{gateway}/{ipfs_cid}",
}
raise HTTPException(404, "No IPFS stream info available")
task_id = pending.get("celery_task_id")
if not task_id:
raise HTTPException(404, "No task ID for this run")
# Get the Celery task result
result = celery_app.AsyncResult(task_id)
if result.ready():
# Task is complete - check the result for IPFS playlist info
task_result = result.result
if isinstance(task_result, dict):
ipfs_playlist_cid = task_result.get("ipfs_playlist_cid")
ipfs_playlist_url = task_result.get("ipfs_playlist_url")
if ipfs_playlist_url:
return {
"run_id": run_id,
"status": "completed",
"ipfs_playlist_cid": ipfs_playlist_cid,
"ipfs_playlist_url": ipfs_playlist_url,
"segment_count": task_result.get("ipfs_segment_count", 0),
}
# Task is still running or no IPFS info available
return {
"run_id": run_id,
"status": pending.get("status", "pending"),
"message": "IPFS streaming info not yet available"
}

48
app/services/cache_service.py
View File

@@ -100,30 +100,52 @@ class CacheService:
async def get_cache_item(self, cid: str, actor_id: str = None) -> Optional[Dict[str, Any]]:
"""Get cached item with full metadata for display."""
# Check if content exists
if not self.cache.has_content(cid):
return None
path = self.cache.get_by_cid(cid)
if not path or not path.exists():
return None
# Get metadata from database
# Get metadata from database first
meta = await self.db.load_item_metadata(cid, actor_id)
cache_item = await self.db.get_cache_item(cid)
media_type = detect_media_type(path)
mime_type = get_mime_type(path)
size = path.stat().st_size
# Check if content exists locally
path = self.cache.get_by_cid(cid) if self.cache.has_content(cid) else None
if path and path.exists():
# Local file exists - detect type from file
media_type = detect_media_type(path)
mime_type = get_mime_type(path)
size = path.stat().st_size
else:
# File not local - check database for type info
# Try to get type from item_types table
media_type = "unknown"
mime_type = "application/octet-stream"
size = 0
if actor_id:
try:
item_types = await self.db.get_item_types(cid, actor_id)
if item_types:
media_type = item_types[0].get("type", "unknown")
if media_type == "video":
mime_type = "video/mp4"
elif media_type == "image":
mime_type = "image/png"
elif media_type == "audio":
mime_type = "audio/mpeg"
except Exception:
pass
# If no local path but we have IPFS CID, content is available remotely
if not cache_item:
return None
result = {
"cid": cid,
"path": str(path),
"path": str(path) if path else None,
"media_type": media_type,
"mime_type": mime_type,
"size": size,
"ipfs_cid": cache_item.get("ipfs_cid") if cache_item else None,
"meta": meta,
"remote_only": path is None or not path.exists(),
}
# Unpack meta fields to top level for template convenience

15
app/templates/cache/detail.html vendored
View File

@@ -13,17 +13,32 @@
<!-- Preview -->
<div class="bg-gray-800 rounded-lg border border-gray-700 mb-6 overflow-hidden">
{% if cache.mime_type and cache.mime_type.startswith('image/') %}
{% if cache.remote_only and cache.ipfs_cid %}
<img src="https://ipfs.io/ipfs/{{ cache.ipfs_cid }}" alt=""
class="w-full max-h-96 object-contain bg-gray-900">
{% else %}
<img src="/cache/{{ cache.cid }}/raw" alt=""
class="w-full max-h-96 object-contain bg-gray-900">
{% endif %}
{% elif cache.mime_type and cache.mime_type.startswith('video/') %}
{% if cache.remote_only and cache.ipfs_cid %}
<video src="https://ipfs.io/ipfs/{{ cache.ipfs_cid }}" controls
class="w-full max-h-96 bg-gray-900">
</video>
{% else %}
<video src="/cache/{{ cache.cid }}/raw" controls
class="w-full max-h-96 bg-gray-900">
</video>
{% endif %}
{% elif cache.mime_type and cache.mime_type.startswith('audio/') %}
<div class="p-8 bg-gray-900">
{% if cache.remote_only and cache.ipfs_cid %}
<audio src="https://ipfs.io/ipfs/{{ cache.ipfs_cid }}" controls class="w-full"></audio>
{% else %}
<audio src="/cache/{{ cache.cid }}/raw" controls class="w-full"></audio>
{% endif %}
</div>
{% elif cache.mime_type == 'application/json' %}

169
app/templates/runs/detail.html
View File

@@ -7,6 +7,7 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.23.0/cytoscape.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/dagre/0.8.5/dagre.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/cytoscape-dagre@2.5.0/cytoscape-dagre.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/hls.js@1.4.12/dist/hls.min.js"></script>
{% endblock %}
{% block content %}
@@ -73,6 +74,174 @@
</div>
</div>
<!-- Live Stream Player (shown during rendering) -->
{% if run.status == 'rendering' %}
<div id="live-stream-container" class="mb-6 bg-gray-800 rounded-lg p-4">
<div class="flex items-center justify-between mb-4">
<h3 class="text-lg font-semibold flex items-center">
<span class="w-3 h-3 bg-red-500 rounded-full mr-2 animate-pulse"></span>
Live Preview
</h3>
<div id="stream-status" class="text-sm text-gray-400">Connecting...</div>
</div>
<div class="relative bg-black rounded-lg overflow-hidden" style="aspect-ratio: 16/9;">
<video id="live-video" class="w-full h-full" controls autoplay muted playsinline></video>
<div id="stream-loading" class="absolute inset-0 flex items-center justify-center bg-gray-900/80">
<div class="text-center">
<div class="animate-spin w-8 h-8 border-2 border-blue-500 border-t-transparent rounded-full mx-auto mb-2"></div>
<div class="text-gray-400">Waiting for stream...</div>
</div>
</div>
</div>
<div class="mt-2 text-xs text-gray-500">
Stream URL: <code class="bg-gray-900 px-1 rounded">/runs/{{ run.run_id }}/hls/stream.m3u8</code>
</div>
</div>
<script>
(function() {
const video = document.getElementById('live-video');
const statusEl = document.getElementById('stream-status');
const loadingEl = document.getElementById('stream-loading');
const hlsUrl = '/runs/{{ run.run_id }}/hls/stream.m3u8';
let hls = null;
let retryCount = 0;
const maxRetries = 120; // Try for up to 4 minutes
let segmentsLoaded = 0;
function initHls() {
if (Hls.isSupported()) {
hls = new Hls({
// Stability over low latency - buffer more for smoother playback
liveSyncDurationCount: 4, // Stay 4 segments behind live edge
liveMaxLatencyDurationCount: 8, // Max 8 segments behind
liveDurationInfinity: true, // Treat as infinite live stream
// Large buffers to absorb rendering speed variations
maxBufferLength: 60, // Buffer up to 60s ahead
maxMaxBufferLength: 120, // Allow even more if needed
maxBufferSize: 60 * 1024 * 1024, // 60MB buffer
maxBufferHole: 0.5, // Tolerate small gaps
// Back buffer for smooth seeking
backBufferLength: 30,
// Playlist reload settings
manifestLoadingTimeOut: 10000,
manifestLoadingMaxRetry: 4,
levelLoadingTimeOut: 10000,
levelLoadingMaxRetry: 4,
fragLoadingTimeOut: 20000,
fragLoadingMaxRetry: 6,
// Start at lowest quality for faster start
startLevel: 0,
// Enable smooth level switching
abrEwmaDefaultEstimate: 500000,
});
hls.on(Hls.Events.MANIFEST_PARSED, function(event, data) {
loadingEl.classList.add('hidden');
statusEl.textContent = 'Buffering...';
statusEl.classList.remove('text-gray-400');
statusEl.classList.add('text-yellow-400');
video.play().catch(() => {});
});
hls.on(Hls.Events.FRAG_LOADED, function(event, data) {
retryCount = 0;
segmentsLoaded++;
statusEl.textContent = `Playing (${segmentsLoaded} segments)`;
statusEl.classList.remove('text-yellow-400', 'text-gray-400');
statusEl.classList.add('text-green-400');
});
hls.on(Hls.Events.BUFFER_APPENDED, function() {
// Hide loading once we have buffered content
loadingEl.classList.add('hidden');
});
hls.on(Hls.Events.ERROR, function(event, data) {
console.log('HLS error:', data.type, data.details, data.fatal);
if (data.fatal) {
switch (data.type) {
case Hls.ErrorTypes.NETWORK_ERROR:
if (retryCount < maxRetries) {
retryCount++;
statusEl.textContent = `Waiting for stream... (${retryCount})`;
statusEl.classList.remove('text-green-400');
statusEl.classList.add('text-yellow-400');
// Exponential backoff with jitter
const delay = Math.min(1000 * Math.pow(1.5, Math.min(retryCount, 6)), 10000);
setTimeout(() => {
hls.loadSource(hlsUrl);
}, delay + Math.random() * 1000);
} else {
statusEl.textContent = 'Stream unavailable';
statusEl.classList.add('text-red-400');
}
break;
case Hls.ErrorTypes.MEDIA_ERROR:
console.log('Media error, attempting recovery');
hls.recoverMediaError();
break;
default:
statusEl.textContent = 'Stream error';
statusEl.classList.add('text-red-400');
break;
}
} else {
// Non-fatal error - just log it
if (data.details === 'bufferStalledError') {
statusEl.textContent = 'Buffering...';
statusEl.classList.remove('text-green-400');
statusEl.classList.add('text-yellow-400');
}
}
});
// Handle video stalls
video.addEventListener('waiting', function() {
statusEl.textContent = 'Buffering...';
statusEl.classList.remove('text-green-400');
statusEl.classList.add('text-yellow-400');
});
video.addEventListener('playing', function() {
statusEl.textContent = `Playing (${segmentsLoaded} segments)`;
statusEl.classList.remove('text-yellow-400');
statusEl.classList.add('text-green-400');
});
hls.loadSource(hlsUrl);
hls.attachMedia(video);
} else if (video.canPlayType('application/vnd.apple.mpegurl')) {
// Native HLS support (Safari)
video.src = hlsUrl;
video.addEventListener('loadedmetadata', function() {
loadingEl.classList.add('hidden');
statusEl.textContent = 'Playing';
video.play().catch(() => {});
});
} else {
statusEl.textContent = 'HLS not supported';
statusEl.classList.add('text-red-400');
}
}
// Start trying to connect
initHls();
// Cleanup on page unload
window.addEventListener('beforeunload', function() {
if (hls) hls.destroy();
});
})();
</script>
{% endif %}
<!-- Tabs -->
<div class="border-b border-gray-700 mb-6">
<nav class="flex space-x-8">