feat(phase8.1-8.2): dynamic worker concurrency via worker_configs

- Migration 054: worker_configs table (queue_name PK, max/min_concurrency,
  enabled, updated_at); seeds step_processing(8/2), thumbnail_rendering(1/1),
  ai_validation(4/1)
- WorkerConfig SQLAlchemy model
- apply_worker_concurrency beat task: reads enabled configs, broadcasts
  pool_grow to all Celery workers every 5min
- GET/PUT /api/worker/configs (admin): list + update per-queue concurrency
- docker-compose.yml: worker uses --autoscale=${MAX_CONCURRENCY:-8},${MIN_CONCURRENCY:-2};
  render-worker uses --autoscale=1,1 --concurrency=1
- WorkerManagement.tsx: "Concurrency Settings" section with +/- steppers
  and Save button per queue

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-08 20:41:57 +01:00
parent b41e70cdad
commit 07e3d1e026
9 changed files with 344 additions and 5 deletions
+87 -2
View File
@@ -1,7 +1,8 @@
"""Worker activity router — exposes recent background task status."""
from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select
from sqlalchemy.orm import selectinload
@@ -14,7 +15,8 @@ from app.models.order import Order
from app.models.order_line import OrderLine
from app.models.product import Product
from app.models.user import User
from app.utils.auth import get_current_user, require_admin_or_pm
from app.models.worker_config import WorkerConfig
from app.utils.auth import get_current_user, require_admin_or_pm, require_admin
router = APIRouter(prefix="/worker", tags=["worker"])
@@ -569,3 +571,86 @@ async def render_health(
last_render_age_minutes=last_render_age_minutes,
details=details,
)
# ---------------------------------------------------------------------------
# Worker concurrency configuration
# ---------------------------------------------------------------------------
class WorkerConfigOut(BaseModel):
queue_name: str
max_concurrency: int
min_concurrency: int
enabled: bool
updated_at: str
class WorkerConfigUpdate(BaseModel):
max_concurrency: Optional[int] = None
min_concurrency: Optional[int] = None
enabled: Optional[bool] = None
@router.get("/configs", response_model=list[WorkerConfigOut])
async def list_worker_configs(
user: User = Depends(require_admin),
db: AsyncSession = Depends(get_db),
):
"""List all worker concurrency configurations (admin only)."""
result = await db.execute(select(WorkerConfig).order_by(WorkerConfig.queue_name))
configs = result.scalars().all()
return [
WorkerConfigOut(
queue_name=cfg.queue_name,
max_concurrency=cfg.max_concurrency,
min_concurrency=cfg.min_concurrency,
enabled=cfg.enabled,
updated_at=cfg.updated_at.isoformat() if cfg.updated_at else datetime.utcnow().isoformat(),
)
for cfg in configs
]
@router.put("/configs/{queue_name}", response_model=WorkerConfigOut)
async def update_worker_config(
queue_name: str,
body: WorkerConfigUpdate,
user: User = Depends(require_admin),
db: AsyncSession = Depends(get_db),
):
"""Update concurrency settings for a specific queue (admin only)."""
result = await db.execute(
select(WorkerConfig).where(WorkerConfig.queue_name == queue_name)
)
cfg = result.scalar_one_or_none()
if not cfg:
raise HTTPException(404, detail=f"No worker config found for queue '{queue_name}'")
if body.max_concurrency is not None:
if body.max_concurrency < 1:
raise HTTPException(400, detail="max_concurrency must be >= 1")
cfg.max_concurrency = body.max_concurrency
if body.min_concurrency is not None:
if body.min_concurrency < 1:
raise HTTPException(400, detail="min_concurrency must be >= 1")
cfg.min_concurrency = body.min_concurrency
if body.enabled is not None:
cfg.enabled = body.enabled
# Validate min <= max after updates
if cfg.min_concurrency > cfg.max_concurrency:
raise HTTPException(400, detail="min_concurrency cannot exceed max_concurrency")
cfg.updated_at = datetime.utcnow()
await db.commit()
await db.refresh(cfg)
return WorkerConfigOut(
queue_name=cfg.queue_name,
max_concurrency=cfg.max_concurrency,
min_concurrency=cfg.min_concurrency,
enabled=cfg.enabled,
updated_at=cfg.updated_at.isoformat(),
)