art-dag/mono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tier 0 scalability: PgBouncer, Redis split, DB split, workers

Browse Source 
T0.1: Separate redis-auth service (64mb, noeviction) for auth state
T0.2: Bump data Redis from 256mb to 1gb
T0.3: Per-app DATABASE_URL via PgBouncer to per-domain databases
T0.4: PgBouncer service (transaction mode, pool=20, max_conn=300);
      session.py pools reduced to 3+5 with timeout and recycle
T0.5: Hypercorn --workers 2 --keep-alive 75 on all 6 entrypoints

Deploy requires running split-databases.sh first to create per-domain
databases from the existing appdb.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
giles
2026-02-25 10:10:18 +00:00
parent 0ccf897f74
commit 961067841e
9 changed files with 62 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
_config/split-databases.sh
View File

@@ -128,9 +128,9 @@ for target_db in db_account db_blog db_market db_cart db_events db_federation; d
CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num)
);
DELETE FROM alembic_version;
INSERT INTO alembic_version (version_num) VALUES ('v2t0p8q9r0');
INSERT INTO alembic_version (version_num) VALUES ('w3u1q9r0s1');
SQL
echo " $target_db stamped at v2t0p8q9r0"
echo " $target_db stamped at w3u1q9r0s1"
done
echo ""

2
account/entrypoint.sh
View File

@@ -23,4 +23,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

2
blog/entrypoint.sh
View File

@@ -29,4 +29,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

2
cart/entrypoint.sh
View File

@@ -26,4 +26,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

52
docker-compose.yml
View File

@@ -10,10 +10,8 @@ x-app-common: &app-common
- /root/rose-ash/_config/app-config.yaml:/app/config/app-config.yaml:ro
x-app-env: &app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@db:5432/appdb
DATABASE_URL_ACCOUNT: postgresql+asyncpg://postgres:change-me@db:5432/appdb
DATABASE_URL_FEDERATION: postgresql+asyncpg://postgres:change-me@db:5432/appdb
ALEMBIC_DATABASE_URL: postgresql+psycopg://postgres:change-me@db:5432/appdb
DATABASE_URL_ACCOUNT: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_account
DATABASE_URL_FEDERATION: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_federation
SMTP_HOST: ${SMTP_HOST}
SMTP_PORT: ${SMTP_PORT}
MAIL_FROM: ${MAIL_FROM}
@@ -25,7 +23,7 @@ x-app-env: &app-env
GHOST_CONTENT_API_KEY: ${GHOST_CONTENT_API_KEY}
GHOST_WEBHOOK_SECRET: ${GHOST_WEBHOOK_SECRET}
GHOST_ADMIN_API_KEY: ${GHOST_ADMIN_API_KEY}
REDIS_AUTH_URL: redis://redis:6379/15
REDIS_AUTH_URL: redis://redis-auth:6379/0
SECRET_KEY: ${SECRET_KEY}
SUMUP_API_KEY: ${SUMUP_API_KEY}
APP_URL_BLOG: https://blog.rose-ash.com
@@ -58,6 +56,8 @@ services:
dockerfile: blog/Dockerfile
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_blog
ALEMBIC_DATABASE_URL: postgresql+psycopg://postgres:change-me@db:5432/db_blog
REDIS_URL: redis://redis:6379/0
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -74,6 +74,7 @@ services:
- /root/rose-ash/_snapshot:/app/_snapshot
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_market
REDIS_URL: redis://redis:6379/1
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -86,6 +87,7 @@ services:
dockerfile: cart/Dockerfile
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_cart
REDIS_URL: redis://redis:6379/2
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -98,6 +100,7 @@ services:
dockerfile: events/Dockerfile
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_events
REDIS_URL: redis://redis:6379/3
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -110,6 +113,7 @@ services:
dockerfile: federation/Dockerfile
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_federation
REDIS_URL: redis://redis:6379/4
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -122,6 +126,7 @@ services:
dockerfile: account/Dockerfile
environment:
<<: *app-env
DATABASE_URL: postgresql+asyncpg://postgres:change-me@pgbouncer:6432/db_account
REDIS_URL: redis://redis:6379/5
DATABASE_HOST: db
DATABASE_PORT: "5432"
@@ -145,6 +150,25 @@ services:
constraints:
- node.labels.gpu != true
pgbouncer:
image: bitnami/pgbouncer:latest
environment:
POSTGRESQL_HOST: db
POSTGRESQL_PORT: "5432"
POSTGRESQL_USERNAME: ${POSTGRES_USER:-postgres}
POSTGRESQL_PASSWORD: ${POSTGRES_PASSWORD:-change-me}
POSTGRESQL_DATABASE: "*"
PGBOUNCER_POOL_MODE: transaction
PGBOUNCER_DEFAULT_POOL_SIZE: "20"
PGBOUNCER_MAX_CLIENT_CONN: "300"
PGBOUNCER_MIN_POOL_SIZE: "5"
networks:
appnet:
deploy:
placement:
constraints:
- node.labels.gpu != true
adminer:
image: adminer
networks:
@@ -164,16 +188,32 @@ services:
appnet:
command:
redis-server
--maxmemory 256mb
--maxmemory 1gb
--maxmemory-policy allkeys-lru
deploy:
placement:
constraints:
- node.labels.gpu != true
redis-auth:
image: redis:7-alpine
volumes:
- redis_auth_data:/data
networks:
appnet:
command:
redis-server
--maxmemory 64mb
--maxmemory-policy noeviction
deploy:
placement:
constraints:
- node.labels.gpu != true
volumes:
db_data_1:
redis_data:
redis_auth_data:
networks:
appnet:
driver: overlay

2
events/entrypoint.sh
View File

@@ -26,4 +26,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

2
federation/entrypoint.sh
View File

@@ -29,4 +29,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

2
market/entrypoint.sh
View File

@@ -26,4 +26,4 @@ fi
# Start the app
echo "Starting Hypercorn (${APP_MODULE:-app:app})..."
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000}
PYTHONUNBUFFERED=1 exec hypercorn "${APP_MODULE:-app:app}" --bind 0.0.0.0:${PORT:-8000} --workers ${WORKERS:-2} --keep-alive 75

12
shared/db/session.py
View File

@@ -15,8 +15,10 @@ _engine = create_async_engine(
future=True,
echo=False,
pool_pre_ping=True,
pool_size=5,
max_overflow=10,
pool_size=int(os.getenv("DB_POOL_SIZE", "3")),
max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "5")),
pool_timeout=10,
pool_recycle=1800,
)
_Session = async_sessionmaker(
@@ -57,7 +59,8 @@ _account_engine = (
else create_async_engine(
DATABASE_URL_ACCOUNT,
future=True, echo=False, pool_pre_ping=True,
pool_size=3, max_overflow=5,
pool_size=2, max_overflow=3,
pool_timeout=10, pool_recycle=1800,
)
)
_AccountSession = async_sessionmaker(
@@ -71,7 +74,8 @@ _federation_engine = (
else create_async_engine(
DATABASE_URL_FEDERATION,
future=True, echo=False, pool_pre_ping=True,
pool_size=3, max_overflow=5,
pool_size=2, max_overflow=3,
pool_timeout=10, pool_recycle=1800,
)
)
_FederationSession = async_sessionmaker(