feat(phase8.1-8.2): dynamic worker concurrency via worker_configs

- Migration 054: worker_configs table (queue_name PK, max/min_concurrency,
  enabled, updated_at); seeds step_processing(8/2), thumbnail_rendering(1/1),
  ai_validation(4/1)
- WorkerConfig SQLAlchemy model
- apply_worker_concurrency beat task: reads enabled configs, broadcasts
  pool_grow to all Celery workers every 5min
- GET/PUT /api/worker/configs (admin): list + update per-queue concurrency
- docker-compose.yml: worker uses --autoscale=${MAX_CONCURRENCY:-8},${MIN_CONCURRENCY:-2};
  render-worker uses --autoscale=1,1 --concurrency=1
- WorkerManagement.tsx: "Concurrency Settings" section with +/- steppers
  and Save button per queue

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-08 20:41:57 +01:00
parent b41e70cdad
commit 07e3d1e026
9 changed files with 344 additions and 5 deletions
+44
View File
@@ -10,6 +10,50 @@ from celery import shared_task
logger = logging.getLogger(__name__)
@shared_task(name="app.tasks.beat_tasks.apply_worker_concurrency", queue="step_processing")
def apply_worker_concurrency() -> None:
"""Read worker_configs from DB and broadcast pool_grow to workers.
Runs every 5 minutes via Celery Beat. Signals all workers to adjust their
pool size to match the max_concurrency setting for each enabled queue.
"""
try:
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from app.config import settings as app_settings
from app.models.worker_config import WorkerConfig
from app.tasks.celery_app import celery_app
sync_url = app_settings.database_url.replace("+asyncpg", "")
eng = create_engine(sync_url)
try:
with Session(eng) as session:
configs = session.execute(
select(WorkerConfig).where(WorkerConfig.enabled == True) # noqa: E712
).scalars().all()
for cfg in configs:
try:
celery_app.control.broadcast(
"pool_grow",
arguments={"n": cfg.max_concurrency},
destination=None, # all workers
reply=False,
)
logger.info(
"[WORKER_SCALE] Signalled pool_grow n=%d for queue %s",
cfg.max_concurrency, cfg.queue_name,
)
except Exception as exc:
logger.warning(
"[WORKER_SCALE] pool_grow failed for %s: %s",
cfg.queue_name, exc,
)
finally:
eng.dispose()
except Exception as exc:
logger.error("apply_worker_concurrency failed: %s", exc)
@shared_task(name="app.tasks.beat_tasks.broadcast_queue_status", queue="step_processing")
def broadcast_queue_status() -> None:
"""Broadcast current queue depths to all WebSocket clients every 10s.
+4
View File
@@ -46,5 +46,9 @@ celery_app.conf.update(
"task": "app.tasks.beat_tasks.recover_stalled_renders",
"schedule": 300.0, # every 5 minutes
},
"apply-worker-concurrency-every-5m": {
"task": "app.tasks.beat_tasks.apply_worker_concurrency",
"schedule": 300.0, # every 5 minutes
},
},
)