diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..01c3870 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.gitea +**/.env +**/.env.gpu +**/__pycache__ +**/.pytest_cache +**/*.pyc +test/ diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..c2bf29e --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,114 @@ +name: Build and Deploy + +on: + push: + branches: [main] + +env: + REGISTRY: registry.rose-ash.com:5000 + ARTDAG_DIR: /root/art-dag-mono + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install tools + run: | + apt-get update && apt-get install -y --no-install-recommends openssh-client + + - name: Set up SSH + env: + SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + run: | + mkdir -p ~/.ssh + echo "$SSH_KEY" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true + + - name: Build and deploy + env: + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + run: | + ssh "root@$DEPLOY_HOST" " + cd ${{ env.ARTDAG_DIR }} + + OLD_HEAD=\$(git rev-parse HEAD 2>/dev/null || echo none) + + git fetch origin main + git reset --hard origin/main + + NEW_HEAD=\$(git rev-parse HEAD) + + # Change detection + BUILD_L1=false + BUILD_L2=false + if [ \"\$OLD_HEAD\" = \"none\" ] || [ \"\$OLD_HEAD\" = \"\$NEW_HEAD\" ]; then + BUILD_L1=true + BUILD_L2=true + else + CHANGED=\$(git diff --name-only \$OLD_HEAD \$NEW_HEAD) + # common/ or core/ change -> rebuild both + if echo \"\$CHANGED\" | grep -qE '^(common|core)/'; then + BUILD_L1=true + BUILD_L2=true + fi + if echo \"\$CHANGED\" | grep -q '^l1/'; then + BUILD_L1=true + fi + if echo \"\$CHANGED\" | grep -q '^l2/'; then + BUILD_L2=true + fi + if echo \"\$CHANGED\" | grep -q '^client/'; then + BUILD_L1=true + fi + fi + + # Build L1 + if [ \"\$BUILD_L1\" = true ]; then + echo 'Building L1...' + docker build \ + --build-arg CACHEBUST=\$(date +%s) \ + -f l1/Dockerfile \ + -t ${{ env.REGISTRY }}/celery-l1-server:latest \ + -t ${{ env.REGISTRY }}/celery-l1-server:${{ github.sha }} \ + . + docker push ${{ env.REGISTRY }}/celery-l1-server:latest + docker push ${{ env.REGISTRY }}/celery-l1-server:${{ github.sha }} + else + echo 'Skipping L1 (no changes)' + fi + + # Build L2 + if [ \"\$BUILD_L2\" = true ]; then + echo 'Building L2...' + docker build \ + --build-arg CACHEBUST=\$(date +%s) \ + -f l2/Dockerfile \ + -t ${{ env.REGISTRY }}/l2-server:latest \ + -t ${{ env.REGISTRY }}/l2-server:${{ github.sha }} \ + . + docker push ${{ env.REGISTRY }}/l2-server:latest + docker push ${{ env.REGISTRY }}/l2-server:${{ github.sha }} + else + echo 'Skipping L2 (no changes)' + fi + + # Deploy stacks + if [ \"\$BUILD_L1\" = true ]; then + cd l1 && source .env && docker stack deploy -c docker-compose.yml celery && cd .. + echo 'L1 stack deployed' + fi + if [ \"\$BUILD_L2\" = true ]; then + cd l2 && source .env && docker stack deploy -c docker-compose.yml activitypub && cd .. + echo 'L2 stack deployed' + fi + + sleep 10 + echo '=== L1 Services ===' + docker stack services celery + echo '=== L2 Services ===' + docker stack services activitypub + " diff --git a/l1/.gitea/workflows/ci.yml b/l1/.gitea/workflows/ci.yml deleted file mode 100644 index a79f66e..0000000 --- a/l1/.gitea/workflows/ci.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build and Deploy - -on: - push: - -env: - REGISTRY: registry.rose-ash.com:5000 - IMAGE_CPU: celery-l1-server - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install tools - run: | - apt-get update && apt-get install -y --no-install-recommends openssh-client - - - name: Set up SSH - env: - SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Pull latest code on server - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - BRANCH: ${{ github.ref_name }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/celery - git fetch origin $BRANCH - git checkout $BRANCH - git reset --hard origin/$BRANCH - " - - - name: Build and push image - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/celery - docker build --build-arg CACHEBUST=\$(date +%s) -t ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:latest -t ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:${{ github.sha }} . - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:latest - docker push ${{ env.REGISTRY }}/${{ env.IMAGE_CPU }}:${{ github.sha }} - " - - - name: Deploy stack - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/celery - docker stack deploy -c docker-compose.yml celery - echo 'Waiting for services to update...' - sleep 10 - docker stack services celery - " diff --git a/l1/Dockerfile b/l1/Dockerfile index 90a770d..25de9a8 100644 --- a/l1/Dockerfile +++ b/l1/Dockerfile @@ -1,25 +1,28 @@ FROM python:3.11-slim - WORKDIR /app -# Install git and ffmpeg (for video transcoding) RUN apt-get update && apt-get install -y --no-install-recommends git ffmpeg && rm -rf /var/lib/apt/lists/* -# Install dependencies -COPY requirements.txt . +# Install common + core as local packages (no more git+https) +COPY common/ /tmp/common/ +COPY core/ /tmp/core/ +RUN pip install --no-cache-dir /tmp/common/ /tmp/core/ && rm -rf /tmp/common /tmp/core + +# Install L1 dependencies +COPY l1/requirements.txt . ARG CACHEBUST=1 RUN pip install --no-cache-dir -r requirements.txt -# Copy application -COPY . . +# Copy L1 application +COPY l1/ . -# Clone effects repo +# Build client tarball from local client/ dir +COPY client/ /tmp/artdag-client/ +RUN cd /tmp && tar -czf /app/artdag-client.tar.gz artdag-client && rm -rf /tmp/artdag-client + +# Clone effects repo (still external) RUN git clone https://git.rose-ash.com/art-dag/effects.git /app/artdag-effects -# Build client tarball for download -RUN ./build-client.sh - -# Create cache directory RUN mkdir -p /data/cache ENV PYTHONUNBUFFERED=1 @@ -27,5 +30,4 @@ ENV PYTHONDONTWRITEBYTECODE=1 ENV EFFECTS_PATH=/app/artdag-effects ENV PYTHONPATH=/app -# Default command runs the server CMD ["python", "server.py"] diff --git a/l1/Dockerfile.gpu b/l1/Dockerfile.gpu index 967f788..8f764da 100644 --- a/l1/Dockerfile.gpu +++ b/l1/Dockerfile.gpu @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN python3 -m pip install --upgrade pip # Install CPU dependencies first -COPY requirements.txt . +COPY l1/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Install GPU-specific dependencies (CuPy for CUDA 12.x) @@ -74,11 +74,16 @@ COPY --from=builder /decord-install /usr/local/lib/python3.11/dist-packages/ COPY --from=builder /tmp/decord/build/libdecord.so /usr/local/lib/ RUN ldconfig +# Install monorepo shared packages +COPY common/ /tmp/common/ +COPY core/ /tmp/core/ +RUN pip install --no-cache-dir /tmp/common/ /tmp/core/ && rm -rf /tmp/common /tmp/core + # Clone effects repo (before COPY so it gets cached) RUN git clone https://git.rose-ash.com/art-dag/effects.git /app/artdag-effects # Copy application (this invalidates cache for any code change) -COPY . . +COPY l1/ . # Create cache directory RUN mkdir -p /data/cache diff --git a/l1/build-client.sh b/l1/build-client.sh deleted file mode 100755 index c9443b6..0000000 --- a/l1/build-client.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -# Build the artdag-client tarball -# This script is run during deployment to create the downloadable client package - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -CLIENT_REPO="https://git.rose-ash.com/art-dag/client.git" -TEMP_DIR=$(mktemp -d) -OUTPUT_FILE="$SCRIPT_DIR/artdag-client.tar.gz" - -echo "Building artdag-client.tar.gz..." - -# Clone the client repo -git clone --depth 1 "$CLIENT_REPO" "$TEMP_DIR/artdag-client" 2>/dev/null || { - echo "Failed to clone client repo, trying alternative..." - # Try GitHub if internal git fails - git clone --depth 1 "https://github.com/gilesbradshaw/art-client.git" "$TEMP_DIR/artdag-client" 2>/dev/null || { - echo "Error: Could not clone client repository" - rm -rf "$TEMP_DIR" - exit 1 - } -} - -# Remove .git directory -rm -rf "$TEMP_DIR/artdag-client/.git" -rm -rf "$TEMP_DIR/artdag-client/__pycache__" - -# Create tarball -cd "$TEMP_DIR" -tar -czf "$OUTPUT_FILE" artdag-client - -# Cleanup -rm -rf "$TEMP_DIR" - -echo "Created: $OUTPUT_FILE" -ls -lh "$OUTPUT_FILE" diff --git a/l1/requirements.txt b/l1/requirements.txt index deab545..be6950f 100644 --- a/l1/requirements.txt +++ b/l1/requirements.txt @@ -13,9 +13,6 @@ markdown>=3.5.0 # Common effect dependencies (used by uploaded effects) numpy>=1.24.0 opencv-python-headless>=4.8.0 -# Core artdag from GitHub (tracks main branch) -git+https://github.com/gilesbradshaw/art-dag.git@main -# Shared components (tracks master branch) -git+https://git.rose-ash.com/art-dag/common.git@master +# core (artdag) and common (artdag_common) installed from local dirs in Dockerfile psycopg2-binary nest_asyncio diff --git a/l2/.gitea/workflows/ci.yml b/l2/.gitea/workflows/ci.yml deleted file mode 100644 index 30d34ea..0000000 --- a/l2/.gitea/workflows/ci.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Build and Deploy - -on: - push: - branches: [main] - -env: - REGISTRY: registry.rose-ash.com:5000 - IMAGE: l2-server - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install tools - run: | - apt-get update && apt-get install -y --no-install-recommends openssh-client - - - name: Set up SSH - env: - SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - mkdir -p ~/.ssh - echo "$SSH_KEY" > ~/.ssh/id_rsa - chmod 600 ~/.ssh/id_rsa - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true - - - name: Pull latest code on server - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/activity-pub - git fetch origin main - git reset --hard origin/main - " - - - name: Build and push image - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/activity-pub - docker build --build-arg CACHEBUST=\$(date +%s) -t ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest -t ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }} . - docker push ${{ env.REGISTRY }}/${{ env.IMAGE }}:latest - docker push ${{ env.REGISTRY }}/${{ env.IMAGE }}:${{ github.sha }} - " - - - name: Deploy stack - env: - DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} - run: | - ssh "root@$DEPLOY_HOST" " - cd /root/art-dag/activity-pub - docker stack deploy -c docker-compose.yml activitypub - echo 'Waiting for services to update...' - sleep 10 - docker stack services activitypub - " diff --git a/l2/Dockerfile b/l2/Dockerfile index 409aadf..085a695 100644 --- a/l2/Dockerfile +++ b/l2/Dockerfile @@ -1,23 +1,23 @@ FROM python:3.11-slim - WORKDIR /app -# Install git for pip to clone dependencies RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* -# Install dependencies -COPY requirements.txt . +# Install common as local package (no more git+https) +COPY common/ /tmp/common/ +RUN pip install --no-cache-dir /tmp/common/ && rm -rf /tmp/common + +# Install L2 dependencies +COPY l2/requirements.txt . ARG CACHEBUST=1 RUN pip install --no-cache-dir -r requirements.txt -# Copy application -COPY . . +# Copy L2 application +COPY l2/ . -# Create data directory RUN mkdir -p /data/l2 ENV PYTHONUNBUFFERED=1 ENV ARTDAG_DATA=/data/l2 -# Default command runs the server CMD ["python", "server.py"] diff --git a/l2/requirements.txt b/l2/requirements.txt index 94d1e5a..5d228c5 100644 --- a/l2/requirements.txt +++ b/l2/requirements.txt @@ -9,5 +9,4 @@ markdown>=3.5.0 python-multipart>=0.0.6 asyncpg>=0.29.0 boto3>=1.34.0 -# Shared components -git+https://git.rose-ash.com/art-dag/common.git@889ea98 +# common (artdag_common) installed from local dir in Dockerfile