Hartmut/HartOMat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(A1): remove Flamenco, simplify render pipeline to Celery-only

Browse Source 
- Remove flamenco-manager and flamenco-worker from docker-compose.yml
- Delete flamenco_client.py, flamenco_tasks.py, docker_scaler.py
- Simplify render_dispatcher.py to Celery-only (removes ~300 lines)
- Remove Flamenco beat schedule from celery_app.py
- Clean admin.py: remove flamenco settings, endpoints, threejs validation
- Clean orders.py cancel-render: Celery revoke only
- Clean worker.py: remove flamenco_job_id from activity response
- Migration 032: cancel lingering flamenco jobs, remove flamenco settings
- PLAN.md: mark all decisions confirmed, status IN UMSETZUNG

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Hartmut Nörenberg
2026-03-06 15:38:37 +01:00
parent 552922eb8a
commit 1d6864fb64
13 changed files with 1524 additions and 1151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routers/admin.py
+6 -110
View File
@@ -1,4 +1,3 @@
import asyncio
import json
import uuid
from datetime import datetime
@@ -17,27 +16,21 @@ from app.utils.auth import require_admin, hash_password
router = APIRouter(prefix="/admin", tags=["admin"])
VALID_RENDERERS = {"pillow", "blender", "threejs"}
VALID_ENGINES = {"cycles", "eevee"}
VALID_THREEJS_SIZES = {512, 1024, 2048}
VALID_FORMATS = {"jpg", "png"}
VALID_STL_QUALITIES = {"low", "high"}
VALID_RENDERERS = {"pillow", "blender"}
VALID_ENGINES = {"cycles", "eevee"}
VALID_FORMATS = {"jpg", "png"}
VALID_STL_QUALITIES = {"low", "high"}
VALID_CYCLES_DEVICES = {"auto", "gpu", "cpu"}
VALID_RENDER_BACKENDS = {"celery", "flamenco", "auto"}
SETTINGS_DEFAULTS: dict[str, str] = {
"thumbnail_renderer": "pillow",
"thumbnail_renderer": "blender",
"blender_engine": "cycles",
"blender_cycles_samples": "256",
"blender_eevee_samples": "64",
"threejs_render_size": "1024",
"thumbnail_format": "jpg",
"stl_quality": "low",
"blender_smooth_angle": "30",
"cycles_device": "auto",
"render_backend": "celery",
"flamenco_manager_url": "http://flamenco-manager:8080",
"flamenco_worker_count": "1",
"blender_max_concurrent_renders": "3",
"product_thumbnail_priority": '["latest_render","cad_thumbnail"]',
"render_stall_timeout_minutes": "120",
@@ -45,18 +38,15 @@ SETTINGS_DEFAULTS: dict[str, str] = {
class SettingsOut(BaseModel):
thumbnail_renderer: str = "pillow"
thumbnail_renderer: str = "blender"
blender_engine: str = "cycles"
blender_cycles_samples: int = 256
blender_eevee_samples: int = 64
threejs_render_size: int = 1024
thumbnail_format: str = "jpg"
stl_quality: str = "low"
blender_smooth_angle: int = 30
cycles_device: str = "auto"
render_backend: str = "celery"
flamenco_manager_url: str = "http://flamenco-manager:8080"
flamenco_worker_count: int = 1
blender_max_concurrent_renders: int = 3
product_thumbnail_priority: str = '["latest_render","cad_thumbnail"]'
render_stall_timeout_minutes: int = 120
@@ -67,14 +57,11 @@ class SettingsUpdate(BaseModel):
blender_engine: str | None = None
blender_cycles_samples: int | None = None
blender_eevee_samples: int | None = None
threejs_render_size: int | None = None
thumbnail_format: str | None = None
stl_quality: str | None = None
blender_smooth_angle: int | None = None
cycles_device: str | None = None
render_backend: str | None = None
flamenco_manager_url: str | None = None
flamenco_worker_count: int | None = None
blender_max_concurrent_renders: int | None = None
product_thumbnail_priority: str | None = None
render_stall_timeout_minutes: int | None = None
@@ -171,14 +158,11 @@ def _settings_to_out(raw: dict[str, str]) -> SettingsOut:
blender_engine=raw["blender_engine"],
blender_cycles_samples=int(raw["blender_cycles_samples"]),
blender_eevee_samples=int(raw["blender_eevee_samples"]),
threejs_render_size=int(raw["threejs_render_size"]),
thumbnail_format=raw["thumbnail_format"],
stl_quality=raw["stl_quality"],
blender_smooth_angle=int(raw["blender_smooth_angle"]),
cycles_device=raw["cycles_device"],
render_backend=raw["render_backend"],
flamenco_manager_url=raw["flamenco_manager_url"],
flamenco_worker_count=int(raw["flamenco_worker_count"]),
blender_max_concurrent_renders=int(raw["blender_max_concurrent_renders"]),
product_thumbnail_priority=raw.get("product_thumbnail_priority", '["latest_render","cad_thumbnail"]'),
render_stall_timeout_minutes=int(raw.get("render_stall_timeout_minutes", "120")),
@@ -207,8 +191,6 @@ async def update_settings(
raise HTTPException(400, detail="blender_cycles_samples must be 1–4096")
if body.blender_eevee_samples is not None and not (1 <= body.blender_eevee_samples <= 1024):
raise HTTPException(400, detail="blender_eevee_samples must be 1–1024")
if body.threejs_render_size is not None and body.threejs_render_size not in VALID_THREEJS_SIZES:
raise HTTPException(400, detail=f"Invalid threejs_render_size. Choose: {', '.join(str(s) for s in sorted(VALID_THREEJS_SIZES))}")
if body.thumbnail_format is not None and body.thumbnail_format not in VALID_FORMATS:
raise HTTPException(400, detail=f"Invalid thumbnail_format. Choose: {', '.join(sorted(VALID_FORMATS))}")
if body.stl_quality is not None and body.stl_quality not in VALID_STL_QUALITIES:
@@ -217,10 +199,6 @@ async def update_settings(
raise HTTPException(400, detail="blender_smooth_angle must be 0–180 degrees")
if body.cycles_device is not None and body.cycles_device not in VALID_CYCLES_DEVICES:
raise HTTPException(400, detail=f"Invalid cycles_device. Choose: {', '.join(sorted(VALID_CYCLES_DEVICES))}")
if body.render_backend is not None and body.render_backend not in VALID_RENDER_BACKENDS:
raise HTTPException(400, detail=f"Invalid render_backend. Choose: {', '.join(sorted(VALID_RENDER_BACKENDS))}")
if body.flamenco_worker_count is not None and not (1 <= body.flamenco_worker_count <= 16):
raise HTTPException(400, detail="flamenco_worker_count must be 1–16")
if body.blender_max_concurrent_renders is not None and not (1 <= body.blender_max_concurrent_renders <= 16):
raise HTTPException(400, detail="blender_max_concurrent_renders must be 1–16")
if body.render_stall_timeout_minutes is not None and not (10 <= body.render_stall_timeout_minutes <= 10080):
@@ -252,8 +230,6 @@ async def update_settings(
updates["blender_cycles_samples"] = str(body.blender_cycles_samples)
if body.blender_eevee_samples is not None:
updates["blender_eevee_samples"] = str(body.blender_eevee_samples)
if body.threejs_render_size is not None:
updates["threejs_render_size"] = str(body.threejs_render_size)
if body.thumbnail_format is not None:
updates["thumbnail_format"] = body.thumbnail_format
if body.stl_quality is not None:
@@ -264,10 +240,6 @@ async def update_settings(
updates["cycles_device"] = body.cycles_device
if body.render_backend is not None:
updates["render_backend"] = body.render_backend
if body.flamenco_manager_url is not None:
updates["flamenco_manager_url"] = body.flamenco_manager_url
if body.flamenco_worker_count is not None:
updates["flamenco_worker_count"] = str(body.flamenco_worker_count)
if body.blender_max_concurrent_renders is not None:
updates["blender_max_concurrent_renders"] = str(body.blender_max_concurrent_renders)
if body.render_stall_timeout_minutes is not None:
@@ -392,7 +364,6 @@ async def renderer_status(
services = {
"pillow": {"url": None, "available": True, "note": "Built-in (always available)"},
"blender": {"url": "http://blender-renderer:8100/health", "available": False, "note": ""},
"threejs": {"url": "http://threejs-renderer:8101/health", "available": False, "note": ""},
}
async with httpx.AsyncClient(timeout=3.0) as client:
for name, info in services.items():
@@ -409,78 +380,3 @@ async def renderer_status(
return services
@router.get("/settings/flamenco-status")
async def flamenco_status(
admin: User = Depends(require_admin),
db: AsyncSession = Depends(get_db),
):
"""Check Flamenco Manager health and list workers."""
raw = await _load_settings(db)
manager_url = raw.get("flamenco_manager_url", "http://flamenco-manager:8080")
from app.services.flamenco_client import get_flamenco_client
client = get_flamenco_client(manager_url)
health = client.health_check()
workers: list[dict] = []
if health["available"]:
try:
workers = client.list_workers()
except Exception as exc:
workers = [{"error": str(exc)[:200]}]
return {
"manager": health,
"workers": workers,
"manager_url": manager_url,
}
class WorkerCountBody(BaseModel):
count: int
@router.get("/settings/flamenco-worker-actual")
async def get_flamenco_worker_actual(admin: User = Depends(require_admin)):
"""Return the number of flamenco-worker containers currently running."""
from app.services.docker_scaler import get_running_worker_count
count = await asyncio.get_event_loop().run_in_executor(None, get_running_worker_count)
return {"running": count, "available": count >= 0}
@router.post("/settings/flamenco-worker-count")
async def set_flamenco_worker_count(
body: WorkerCountBody,
admin: User = Depends(require_admin),
db: AsyncSession = Depends(get_db),
):
"""Scale Flamenco worker containers to the requested count via Docker socket."""
if not (1 <= body.count <= 16):
raise HTTPException(400, detail="Worker count must be 1–16")
# Save desired count to settings first
await _save_setting(db, "flamenco_worker_count", str(body.count))
await db.commit()
# Perform actual Docker scaling in a thread (blocking SDK call)
from app.services.docker_scaler import scale_workers
try:
result = await asyncio.get_event_loop().run_in_executor(None, scale_workers, body.count)
return {
"count": body.count,
"previous": result["previous"],
"current": result["current"],
"delta": result["delta"],
"message": result["message"],
}
except Exception as exc:
# Scaling failed — return a warning but keep the saved setting
return {
"count": body.count,
"previous": -1,
"current": -1,
"delta": 0,
"message": f"Setting saved, but Docker scaling failed: {exc}. "
f"Run `docker compose up -d --scale flamenco-worker={body.count}` manually.",
}
backend/app/api/routers/orders.py
+16 -69
View File
@@ -920,44 +920,17 @@ async def cancel_line_render(
if line.render_status not in ("processing", "pending"):
raise HTTPException(400, detail=f"Line render_status is '{line.render_status}', nothing to cancel")
cancelled_backend = line.render_backend_used or "unknown"
cancelled_backend = line.render_backend_used or "celery"
errors: list[str] = []
# Cancel Flamenco job if applicable
if line.render_backend_used == "flamenco" and line.flamenco_job_id:
try:
from app.services.flamenco_client import get_flamenco_client
from app.models.system_setting import SystemSetting
row = await db.execute(
select(SystemSetting).where(SystemSetting.key == "flamenco_manager_url")
)
setting = row.scalar_one_or_none()
url = setting.value if setting else "http://flamenco-manager:8080"
client = get_flamenco_client(url)
client.cancel_job(line.flamenco_job_id)
except Exception as exc:
errors.append(f"Flamenco cancel failed: {str(exc)[:200]}")
# Revoke Celery task if applicable
if line.render_backend_used == "celery" or not line.render_backend_used:
try:
from app.tasks.celery_app import celery_app
celery_app.control.revoke(
f"render-{line_id}", terminate=True, signal="SIGTERM"
)
except Exception as exc:
errors.append(f"Celery revoke failed: {str(exc)[:200]}")
# Also kill the Blender subprocess in the renderer microservice.
# The job_id sent to blender-renderer equals the order_line_id.
try:
import httpx as _httpx
_httpx.post(
f"http://blender-renderer:8100/cancel/{line_id}",
timeout=5.0,
)
except Exception:
pass # best-effort; renderer may not be running a job for this line
# Revoke Celery task (best-effort)
try:
from app.tasks.celery_app import celery_app
celery_app.control.revoke(
f"render-{line_id}", terminate=True, signal="SIGTERM"
)
except Exception as exc:
errors.append(f"Celery revoke failed: {str(exc)[:200]}")
# Mark line as cancelled
from sqlalchemy import update as sql_update
@@ -1013,47 +986,21 @@ async def cancel_order_renders(
if not lines:
raise HTTPException(400, detail="No active renders to cancel")
from app.services.flamenco_client import get_flamenco_client
from app.models.system_setting import SystemSetting
from app.tasks.celery_app import celery_app
from sqlalchemy import update as sql_update
# Load Flamenco URL once
row = await db.execute(
select(SystemSetting).where(SystemSetting.key == "flamenco_manager_url")
)
setting = row.scalar_one_or_none()
flamenco_url = setting.value if setting else "http://flamenco-manager:8080"
now = datetime.utcnow()
cancelled_count = 0
errors: list[str] = []
for line in lines:
# Cancel Flamenco job
if line.render_backend_used == "flamenco" and line.flamenco_job_id:
try:
client = get_flamenco_client(flamenco_url)
client.cancel_job(line.flamenco_job_id)
except Exception as exc:
errors.append(f"Line {line.id}: Flamenco cancel failed: {str(exc)[:100]}")
# Revoke Celery task + kill Blender subprocess in renderer service
if line.render_backend_used == "celery" or not line.render_backend_used:
try:
celery_app.control.revoke(
f"render-{line.id}", terminate=True, signal="SIGTERM"
)
except Exception:
pass # Celery revoke is best-effort
try:
import httpx as _httpx
_httpx.post(
f"http://blender-renderer:8100/cancel/{line.id}",
timeout=5.0,
)
except Exception:
pass # best-effort
# Revoke Celery task (best-effort)
try:
celery_app.control.revoke(
f"render-{line.id}", terminate=True, signal="SIGTERM"
)
except Exception:
pass
await db.execute(
sql_update(OrderLine)
backend/app/api/routers/worker.py
-2
View File
@@ -38,7 +38,6 @@ class RenderJobEntry(BaseModel):
output_type_name: str | None
render_status: str
render_backend_used: str | None
flamenco_job_id: str | None
render_started_at: str | None
render_completed_at: str | None
updated_at: str
@@ -140,7 +139,6 @@ async def get_worker_activity(
output_type_name=rl.output_type.name if rl.output_type else None,
render_status=rl.render_status,
render_backend_used=rl.render_backend_used,
flamenco_job_id=rl.flamenco_job_id,
render_started_at=rl.render_started_at.isoformat() if rl.render_started_at else None,
render_completed_at=rl.render_completed_at.isoformat() if rl.render_completed_at else None,
updated_at=rl.updated_at.isoformat(),