Add IPFS HLS streaming and GPU optimizations
- Add IPFSHLSOutput class that uploads segments to IPFS as they're created - Update streaming task to use IPFS HLS output for distributed streaming - Add /ipfs-stream endpoint to get IPFS playlist URL - Update /stream endpoint to redirect to IPFS when available - Add GPU persistence mode (STREAMING_GPU_PERSIST=1) to keep frames on GPU - Add hardware video decoding (NVDEC) support for faster video processing - Add GPU-accelerated primitive libraries: blending_gpu, color_ops_gpu, geometry_gpu - Add streaming_gpu module with GPUFrame class for tracking CPU/GPU data location - Add Dockerfile.gpu for building GPU-enabled worker image Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
giles
77
scripts/cloud-init-gpu.sh
Normal file
77
scripts/cloud-init-gpu.sh
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# Cloud-init startup script for GPU droplet (RTX 6000 Ada, etc.)
|
||||
# Paste this into DigitalOcean "User data" field when creating droplet
|
||||
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
exec > /var/log/artdag-setup.log 2>&1
|
||||
|
||||
echo "=== ArtDAG GPU Setup Started $(date) ==="
|
||||
|
||||
# Update system (non-interactive, keep existing configs)
|
||||
apt-get update
|
||||
apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade
|
||||
|
||||
# Install essentials
|
||||
apt-get install -y \
|
||||
python3 python3-venv python3-pip \
|
||||
git curl wget \
|
||||
ffmpeg \
|
||||
vulkan-tools \
|
||||
build-essential
|
||||
|
||||
# Create venv
|
||||
VENV_DIR="/opt/artdag-gpu"
|
||||
python3 -m venv "$VENV_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
# Install Python packages
|
||||
pip install --upgrade pip
|
||||
pip install \
|
||||
numpy \
|
||||
opencv-python-headless \
|
||||
wgpu \
|
||||
httpx \
|
||||
pyyaml \
|
||||
celery[redis] \
|
||||
fastapi \
|
||||
uvicorn \
|
||||
asyncpg
|
||||
|
||||
# Create code directory
|
||||
mkdir -p "$VENV_DIR/celery/sexp_effects/effects"
|
||||
mkdir -p "$VENV_DIR/celery/sexp_effects/primitive_libs"
|
||||
mkdir -p "$VENV_DIR/celery/streaming"
|
||||
|
||||
# Add SSH key for easier access (optional - add your key here)
|
||||
# echo "ssh-ed25519 AAAA... your-key" >> /root/.ssh/authorized_keys
|
||||
|
||||
# Test GPU
|
||||
echo "=== GPU Info ==="
|
||||
nvidia-smi || echo "nvidia-smi not available yet"
|
||||
|
||||
echo "=== NVENC Check ==="
|
||||
ffmpeg -encoders 2>/dev/null | grep -E "nvenc|cuda" || echo "NVENC not detected"
|
||||
|
||||
echo "=== wgpu Check ==="
|
||||
"$VENV_DIR/bin/python3" -c "
|
||||
import wgpu
|
||||
try:
|
||||
adapter = wgpu.gpu.request_adapter_sync(power_preference='high-performance')
|
||||
print(f'GPU: {adapter.info}')
|
||||
except Exception as e:
|
||||
print(f'wgpu error: {e}')
|
||||
" || echo "wgpu test failed"
|
||||
|
||||
# Add environment setup
|
||||
cat >> /etc/profile.d/artdag-gpu.sh << 'ENVEOF'
|
||||
export WGPU_BACKEND_TYPE=Vulkan
|
||||
export PATH="/opt/artdag-gpu/bin:$PATH"
|
||||
ENVEOF
|
||||
|
||||
# Mark setup complete
|
||||
touch /opt/artdag-gpu/.setup-complete
|
||||
echo "=== Setup Complete $(date) ==="
|
||||
echo "Venv: /opt/artdag-gpu"
|
||||
echo "Activate: source /opt/artdag-gpu/bin/activate"
|
||||
echo "Vulkan: export WGPU_BACKEND_TYPE=Vulkan"
|
||||
51
scripts/deploy-to-gpu.sh
Executable file
51
scripts/deploy-to-gpu.sh
Executable file
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# Deploy art-dag GPU code to a remote droplet
|
||||
# Usage: ./deploy-to-gpu.sh <droplet-ip>
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "Usage: $0 <droplet-ip>"
|
||||
echo "Example: $0 159.223.7.100"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
DROPLET_IP="$1"
|
||||
REMOTE_DIR="/opt/artdag-gpu/celery"
|
||||
LOCAL_DIR="$(dirname "$0")/.."
|
||||
|
||||
echo "=== Deploying to $DROPLET_IP ==="
|
||||
|
||||
# Create remote directory
|
||||
echo "[1/4] Creating remote directory..."
|
||||
ssh "root@$DROPLET_IP" "mkdir -p $REMOTE_DIR/sexp_effects $REMOTE_DIR/streaming $REMOTE_DIR/scripts"
|
||||
|
||||
# Copy core files
|
||||
echo "[2/4] Copying core files..."
|
||||
scp "$LOCAL_DIR/sexp_effects/wgsl_compiler.py" "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/"
|
||||
scp "$LOCAL_DIR/sexp_effects/parser.py" "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/"
|
||||
scp "$LOCAL_DIR/sexp_effects/interpreter.py" "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/"
|
||||
scp "$LOCAL_DIR/sexp_effects/__init__.py" "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/"
|
||||
scp "$LOCAL_DIR/streaming/backends.py" "root@$DROPLET_IP:$REMOTE_DIR/streaming/"
|
||||
|
||||
# Copy effects
|
||||
echo "[3/4] Copying effects..."
|
||||
ssh "root@$DROPLET_IP" "mkdir -p $REMOTE_DIR/sexp_effects/effects $REMOTE_DIR/sexp_effects/primitive_libs"
|
||||
scp -r "$LOCAL_DIR/sexp_effects/effects/"*.sexp "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/effects/" 2>/dev/null || true
|
||||
scp -r "$LOCAL_DIR/sexp_effects/primitive_libs/"*.py "root@$DROPLET_IP:$REMOTE_DIR/sexp_effects/primitive_libs/" 2>/dev/null || true
|
||||
|
||||
# Test
|
||||
echo "[4/4] Testing deployment..."
|
||||
ssh "root@$DROPLET_IP" "cd $REMOTE_DIR && /opt/artdag-gpu/bin/python3 -c '
|
||||
import sys
|
||||
sys.path.insert(0, \".\")
|
||||
from sexp_effects.wgsl_compiler import compile_effect_file
|
||||
result = compile_effect_file(\"sexp_effects/effects/invert.sexp\")
|
||||
print(f\"Compiled effect: {result.name}\")
|
||||
print(\"Deployment OK\")
|
||||
'" || echo "Test failed - may need to run setup script first"
|
||||
|
||||
echo ""
|
||||
echo "=== Deployment complete ==="
|
||||
echo "SSH: ssh root@$DROPLET_IP"
|
||||
echo "Test: ssh root@$DROPLET_IP 'cd $REMOTE_DIR && /opt/artdag-gpu/bin/python3 -c \"from streaming.backends import get_backend; b=get_backend(\\\"wgpu\\\"); print(b)\"'"
|
||||
108
scripts/setup-gpu-droplet.sh
Executable file
108
scripts/setup-gpu-droplet.sh
Executable file
@@ -0,0 +1,108 @@
|
||||
#!/bin/bash
|
||||
# Setup script for GPU droplet with NVENC support
|
||||
# Run as root on a fresh Ubuntu droplet with NVIDIA GPU
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== ArtDAG GPU Droplet Setup ==="
|
||||
|
||||
# 1. System updates
|
||||
echo "[1/7] Updating system..."
|
||||
apt-get update
|
||||
apt-get upgrade -y
|
||||
|
||||
# 2. Install NVIDIA drivers (if not already installed)
|
||||
echo "[2/7] Checking NVIDIA drivers..."
|
||||
if ! command -v nvidia-smi &> /dev/null; then
|
||||
echo "Installing NVIDIA drivers..."
|
||||
apt-get install -y nvidia-driver-535 nvidia-utils-535
|
||||
echo "NVIDIA drivers installed. Reboot required."
|
||||
echo "After reboot, run this script again."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
nvidia-smi
|
||||
echo "NVIDIA drivers OK"
|
||||
|
||||
# 3. Install FFmpeg with NVENC support
|
||||
echo "[3/7] Installing FFmpeg with NVENC..."
|
||||
apt-get install -y ffmpeg
|
||||
|
||||
# Verify NVENC
|
||||
if ffmpeg -encoders 2>/dev/null | grep -q nvenc; then
|
||||
echo "NVENC available:"
|
||||
ffmpeg -encoders 2>/dev/null | grep nvenc
|
||||
else
|
||||
echo "WARNING: NVENC not available. GPU may not support hardware encoding."
|
||||
fi
|
||||
|
||||
# 4. Install Python and create venv
|
||||
echo "[4/7] Setting up Python environment..."
|
||||
apt-get install -y python3 python3-venv python3-pip git
|
||||
|
||||
VENV_DIR="/opt/artdag-gpu"
|
||||
python3 -m venv "$VENV_DIR"
|
||||
source "$VENV_DIR/bin/activate"
|
||||
|
||||
# 5. Install Python dependencies
|
||||
echo "[5/7] Installing Python packages..."
|
||||
pip install --upgrade pip
|
||||
pip install \
|
||||
numpy \
|
||||
opencv-python-headless \
|
||||
wgpu \
|
||||
httpx \
|
||||
pyyaml \
|
||||
celery[redis] \
|
||||
fastapi \
|
||||
uvicorn
|
||||
|
||||
# 6. Clone/update art-dag code
|
||||
echo "[6/7] Setting up art-dag code..."
|
||||
ARTDAG_DIR="$VENV_DIR/celery"
|
||||
if [ -d "$ARTDAG_DIR" ]; then
|
||||
echo "Updating existing code..."
|
||||
cd "$ARTDAG_DIR"
|
||||
git pull || true
|
||||
else
|
||||
echo "Cloning art-dag..."
|
||||
git clone https://git.rose-ash.com/art-dag/celery.git "$ARTDAG_DIR" || {
|
||||
echo "Git clone failed. You may need to copy code manually."
|
||||
}
|
||||
fi
|
||||
|
||||
# 7. Test GPU compute
|
||||
echo "[7/7] Testing GPU compute..."
|
||||
"$VENV_DIR/bin/python3" << 'PYTEST'
|
||||
import sys
|
||||
try:
|
||||
import wgpu
|
||||
adapter = wgpu.gpu.request_adapter_sync(power_preference="high-performance")
|
||||
print(f"GPU Adapter: {adapter.info.get('device', 'unknown')}")
|
||||
device = adapter.request_device_sync()
|
||||
print("wgpu device created successfully")
|
||||
|
||||
# Check for NVENC via FFmpeg
|
||||
import subprocess
|
||||
result = subprocess.run(['ffmpeg', '-encoders'], capture_output=True, text=True)
|
||||
if 'h264_nvenc' in result.stdout:
|
||||
print("NVENC H.264 encoder: AVAILABLE")
|
||||
else:
|
||||
print("NVENC H.264 encoder: NOT AVAILABLE")
|
||||
if 'hevc_nvenc' in result.stdout:
|
||||
print("NVENC HEVC encoder: AVAILABLE")
|
||||
else:
|
||||
print("NVENC HEVC encoder: NOT AVAILABLE")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
PYTEST
|
||||
|
||||
echo ""
|
||||
echo "=== Setup Complete ==="
|
||||
echo "Venv: $VENV_DIR"
|
||||
echo "Code: $ARTDAG_DIR"
|
||||
echo ""
|
||||
echo "To activate: source $VENV_DIR/bin/activate"
|
||||
echo "To test: cd $ARTDAG_DIR && python -c 'from streaming.backends import get_backend; print(get_backend(\"wgpu\"))'"
|
||||
Reference in New Issue
Block a user