art-dag/mono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configure monorepo build: unified CI, local deps, .dockerignore

Browse Source 
- Dockerfiles use monorepo root as build context
- common/ and core/ installed as local packages (no git+https)
- Client tarball built from local client/ dir
- Unified CI with change detection: common/core -> rebuild both
- Per-repo CI workflows removed

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
giles
2026-02-24 23:16:23 +00:00
parent 5173167f3e
commit a13c361dee
10 changed files with 153 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.dockerignore Normal file
View File

@@ -0,0 +1,8 @@
.git
.gitea
**/.env
**/.env.gpu
**/__pycache__
**/.pytest_cache
**/*.pyc
test/

114
.gitea/workflows/ci.yml Normal file
View File

@@ -0,0 +1,114 @@
name: Build and Deploy
on:
push:
branches: [main]
env:
REGISTRY: registry.rose-ash.com:5000
ARTDAG_DIR: /root/art-dag-mono
jobs:
build-and-deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install tools
run: |
apt-get update && apt-get install -y --no-install-recommends openssh-client
- name: Set up SSH
env:
SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
mkdir -p ~/.ssh
echo "$SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
- name: Build and deploy
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd ${{ env.ARTDAG_DIR }}
OLD_HEAD=\$(git rev-parse HEAD 2>/dev/null || echo none)
git fetch origin main
git reset --hard origin/main
NEW_HEAD=\$(git rev-parse HEAD)
# Change detection
BUILD_L1=false
BUILD_L2=false
if [ \"\$OLD_HEAD\" = \"none\" ] || [ \"\$OLD_HEAD\" = \"\$NEW_HEAD\" ]; then
BUILD_L1=true
BUILD_L2=true
else
CHANGED=\$(git diff --name-only \$OLD_HEAD \$NEW_HEAD)
# common/ or core/ change -> rebuild both
if echo \"\$CHANGED\" | grep -qE '^(common|core)/'; then
BUILD_L1=true
BUILD_L2=true
fi
if echo \"\$CHANGED\" | grep -q '^l1/'; then
BUILD_L1=true
fi
if echo \"\$CHANGED\" | grep -q '^l2/'; then
BUILD_L2=true
fi
if echo \"\$CHANGED\" | grep -q '^client/'; then
BUILD_L1=true
fi
fi
# Build L1
if [ \"\$BUILD_L1\" = true ]; then
echo 'Building L1...'
docker build \
--build-arg CACHEBUST=\$(date +%s) \
-f l1/Dockerfile \
-t ${{ env.REGISTRY }}/celery-l1-server:latest \
-t ${{ env.REGISTRY }}/celery-l1-server:${{ github.sha }} \
.
docker push ${{ env.REGISTRY }}/celery-l1-server:latest
docker push ${{ env.REGISTRY }}/celery-l1-server:${{ github.sha }}
else
echo 'Skipping L1 (no changes)'
fi
# Build L2
if [ \"\$BUILD_L2\" = true ]; then
echo 'Building L2...'
docker build \
--build-arg CACHEBUST=\$(date +%s) \
-f l2/Dockerfile \
-t ${{ env.REGISTRY }}/l2-server:latest \
-t ${{ env.REGISTRY }}/l2-server:${{ github.sha }} \
.
docker push ${{ env.REGISTRY }}/l2-server:latest
docker push ${{ env.REGISTRY }}/l2-server:${{ github.sha }}
else
echo 'Skipping L2 (no changes)'
fi
# Deploy stacks
if [ \"\$BUILD_L1\" = true ]; then
cd l1 && source .env && docker stack deploy -c docker-compose.yml celery && cd ..
echo 'L1 stack deployed'
fi
if [ \"\$BUILD_L2\" = true ]; then
cd l2 && source .env && docker stack deploy -c docker-compose.yml activitypub && cd ..
echo 'L2 stack deployed'
fi
sleep 10
echo '=== L1 Services ==='
docker stack services celery
echo '=== L2 Services ==='
docker stack services activitypub
"

63
l1/.gitea/workflows/ci.yml
View File

@@ -1,63 +0,0 @@
name: Build and Deploy
on:
push:
env:
REGISTRY: registry.rose-ash.com:5000
IMAGE_CPU: celery-l1-server
jobs:
build-and-deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install tools
run: |
apt-get update && apt-get install -y --no-install-recommends openssh-client
- name: Set up SSH
env:
SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
mkdir -p ~/.ssh
echo "$SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
- name: Pull latest code on server
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
BRANCH: ${{ github.ref_name }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/celery
git fetch origin $BRANCH
git checkout $BRANCH
git reset --hard origin/$BRANCH
"
- name: Build and push image
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/celery
docker build --build-arg CACHEBUST=\$(date +%s) -t ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:latest -t ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:${{ github.sha }} .
docker push ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:latest
docker push ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:${{ github.sha }}
"
- name: Deploy stack
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/celery
docker stack deploy -c docker-compose.yml celery
echo 'Waiting for services to update...'
sleep 10
docker stack services celery
"

26
l1/Dockerfile
View File

@@ -1,25 +1,28 @@
FROM python:3.11-slim
WORKDIR /app
# Install git and ffmpeg (for video transcoding)
RUN apt-get update && apt-get install -y --no-install-recommends git ffmpeg && rm -rf /var/lib/apt/lists/*
# Install dependencies
COPY requirements.txt .
# Install common + core as local packages (no more git+https)
COPY common/ /tmp/common/
COPY core/ /tmp/core/
RUN pip install --no-cache-dir /tmp/common/ /tmp/core/ && rm -rf /tmp/common /tmp/core
# Install L1 dependencies
COPY l1/requirements.txt .
ARG CACHEBUST=1
RUN pip install --no-cache-dir -r requirements.txt
# Copy application
COPY . .
# Copy L1 application
COPY l1/ .
# Clone effects repo
# Build client tarball from local client/ dir
COPY client/ /tmp/artdag-client/
RUN cd /tmp && tar -czf /app/artdag-client.tar.gz artdag-client && rm -rf /tmp/artdag-client
# Clone effects repo (still external)
RUN git clone https://git.rose-ash.com/art-dag/effects.git /app/artdag-effects
# Build client tarball for download
RUN ./build-client.sh
# Create cache directory
RUN mkdir -p /data/cache
ENV PYTHONUNBUFFERED=1
@@ -27,5 +30,4 @@ ENV PYTHONDONTWRITEBYTECODE=1
ENV EFFECTS_PATH=/app/artdag-effects
ENV PYTHONPATH=/app
# Default command runs the server
CMD ["python", "server.py"]

9
l1/Dockerfile.gpu
View File

@@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
RUN python3 -m pip install --upgrade pip
# Install CPU dependencies first
COPY requirements.txt .
COPY l1/requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# Install GPU-specific dependencies (CuPy for CUDA 12.x)
@@ -74,11 +74,16 @@ COPY --from=builder /decord-install /usr/local/lib/python3.11/dist-packages/
COPY --from=builder /tmp/decord/build/libdecord.so /usr/local/lib/
RUN ldconfig
# Install monorepo shared packages
COPY common/ /tmp/common/
COPY core/ /tmp/core/
RUN pip install --no-cache-dir /tmp/common/ /tmp/core/ && rm -rf /tmp/common /tmp/core
# Clone effects repo (before COPY so it gets cached)
RUN git clone https://git.rose-ash.com/art-dag/effects.git /app/artdag-effects
# Copy application (this invalidates cache for any code change)
COPY . .
COPY l1/ .
# Create cache directory
RUN mkdir -p /data/cache

37
l1/build-client.sh
View File

@@ -1,37 +0,0 @@
#!/bin/bash
# Build the artdag-client tarball
# This script is run during deployment to create the downloadable client package
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLIENT_REPO="https://git.rose-ash.com/art-dag/client.git"
TEMP_DIR=$(mktemp -d)
OUTPUT_FILE="$SCRIPT_DIR/artdag-client.tar.gz"
echo "Building artdag-client.tar.gz..."
# Clone the client repo
git clone --depth 1 "$CLIENT_REPO" "$TEMP_DIR/artdag-client" 2>/dev/null || {
echo "Failed to clone client repo, trying alternative..."
# Try GitHub if internal git fails
git clone --depth 1 "https://github.com/gilesbradshaw/art-client.git" "$TEMP_DIR/artdag-client" 2>/dev/null || {
echo "Error: Could not clone client repository"
rm -rf "$TEMP_DIR"
exit 1
}
}
# Remove .git directory
rm -rf "$TEMP_DIR/artdag-client/.git"
rm -rf "$TEMP_DIR/artdag-client/__pycache__"
# Create tarball
cd "$TEMP_DIR"
tar -czf "$OUTPUT_FILE" artdag-client
# Cleanup
rm -rf "$TEMP_DIR"
echo "Created: $OUTPUT_FILE"
ls -lh "$OUTPUT_FILE"

5
l1/requirements.txt
View File

@@ -13,9 +13,6 @@ markdown>=3.5.0
# Common effect dependencies (used by uploaded effects)
numpy>=1.24.0
opencv-python-headless>=4.8.0
# Core artdag from GitHub (tracks main branch)
git+https://github.com/gilesbradshaw/art-dag.git@main
# Shared components (tracks master branch)
git+https://git.rose-ash.com/art-dag/common.git@master
# core (artdag) and common (artdag_common) installed from local dirs in Dockerfile
psycopg2-binary
nest_asyncio

62
l2/.gitea/workflows/ci.yml
View File

@@ -1,62 +0,0 @@
name: Build and Deploy
on:
push:
branches: [main]
env:
REGISTRY: registry.rose-ash.com:5000
IMAGE: l2-server
jobs:
build-and-deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install tools
run: |
apt-get update && apt-get install -y --no-install-recommends openssh-client
- name: Set up SSH
env:
SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
mkdir -p ~/.ssh
echo "$SSH_KEY" > ~/.ssh/id_rsa
chmod 600 ~/.ssh/id_rsa
ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true
- name: Pull latest code on server
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/activity-pub
git fetch origin main
git reset --hard origin/main
"
- name: Build and push image
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/activity-pub
docker build --build-arg CACHEBUST=\$(date +%s) -t ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest -t ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }} .
docker push ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest
docker push ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }}
"
- name: Deploy stack
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
run: |
ssh "root@$DEPLOY_HOST" "
cd /root/art-dag/activity-pub
docker stack deploy -c docker-compose.yml activitypub
echo 'Waiting for services to update...'
sleep 10
docker stack services activitypub
"

16
l2/Dockerfile
View File

@@ -1,23 +1,23 @@
FROM python:3.11-slim
WORKDIR /app
# Install git for pip to clone dependencies
RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
# Install dependencies
COPY requirements.txt .
# Install common as local package (no more git+https)
COPY common/ /tmp/common/
RUN pip install --no-cache-dir /tmp/common/ && rm -rf /tmp/common
# Install L2 dependencies
COPY l2/requirements.txt .
ARG CACHEBUST=1
RUN pip install --no-cache-dir -r requirements.txt
# Copy application
COPY . .
# Copy L2 application
COPY l2/ .
# Create data directory
RUN mkdir -p /data/l2
ENV PYTHONUNBUFFERED=1
ENV ARTDAG_DATA=/data/l2
# Default command runs the server
CMD ["python", "server.py"]

3
l2/requirements.txt
View File

@@ -9,5 +9,4 @@ markdown>=3.5.0
python-multipart>=0.0.6
asyncpg>=0.29.0
boto3>=1.34.0
# Shared components
git+https://git.rose-ash.com/art-dag/common.git@889ea98
# common (artdag_common) installed from local dir in Dockerfile