From 23605783bd42be24244fdbfce68463401eb73dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hartmut=20N=C3=B6renberg?= Date: Wed, 22 Jul 2026 15:12:07 +0200 Subject: [PATCH] feat: multi-GPU queue routing for legacy still renders (M2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dispatch_order_line_render now routes still renders (non-animation, non-cinematic) to a secondary GPU queue when MULTI_GPU_LIGHT_RENDER_QUEUE is configured and that queue has active Celery workers. - config.py: multi_gpu_light_render_queue setting (default "" = disabled) - render_order_line.py: implement the routing stub — load output_type via selectinload, check render_settings.animation + .cinematic flags, call _inspect_active_worker_queues (reused from workflow_graph_runtime) with 0.5s timeout to check if the light queue is live No behaviour change when MULTI_GPU_LIGHT_RENDER_QUEUE is not set. Enable by setting it to "asset_pipeline_light" and adding a render-worker-light service with concurrency=1 to docker-compose. docs: learnings erfasst — multi-GPU queue routing M2 Co-Authored-By: Claude Sonnet 4.6 --- LEARNINGS.md | 3 ++ backend/app/config.py | 7 +++++ .../pipeline/tasks/render_order_line.py | 29 +++++++++++++++---- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/LEARNINGS.md b/LEARNINGS.md index 28721dd..0506e8d 100644 --- a/LEARNINGS.md +++ b/LEARNINGS.md @@ -7,6 +7,9 @@ ## Learnings +### 2026-07-22 | Architecture | Multi-GPU Queue Routing für Legacy-Dispatch implementiert +M2: `dispatch_order_line_render` leitet Still-Renders an eine zweite GPU-Queue um wenn `MULTI_GPU_LIGHT_RENDER_QUEUE` gesetzt ist UND diese Queue aktive Worker hat (Celery inspect). Animationen und Cinematics bleiben immer auf `asset_pipeline`. Neue Settings-Option `multi_gpu_light_render_queue: str = ""` in config.py — Default leer = disabled. `_inspect_active_worker_queues()` aus `workflow_graph_runtime.py` wiederverwendet (timeout=0.5s). Kein Breaking-Change: ohne die Env-Variable verhält sich die Dispatch-Funktion exakt wie vorher. + ### 2026-07-22 | Architecture | BLENDER_CINEMATIC Workflow-Graph-Node implementiert Der cinematic Render-Pfad hatte keinen eigenen Workflow-Graph-Node — Migration 071 hatte alle cinematic Output-Types auf `legacy_only` gezwungen als Sicherheitsnetz. M1 fügt jetzt `StepName.BLENDER_CINEMATIC` hinzu, zusammen mit: (1) Node-Definition im `workflow_node_registry.py` mit denselben Szene/Camera/Material-Feldern wie BLENDER_STILL, aber ohne Animations-Params (frame_count/fps sind im cinematic_render.py-Script hartkodiert auf 250 @ 25fps), (2) `render_cinematic_task` in `tasks.py` — folgt dem Pattern von `render_order_line_still_task`, gibt mp4 aus, nutzt `_finalize_graph_turntable_output`/`_finalize_shadow_turntable_output` da cinematic = mp4, published mit `asset_type="turntable"`, (3) STEP_TASK_MAP + `_ORDER_LINE_RENDER_STEPS` + `_build_task_kwargs` + `_predict_render_output_artifact` + `_artifact_kind_override_for_step` in `workflow_graph_runtime.py` alle aktualisiert. Kein neues DB-Migration nötig: Admins können cinematic Output-Types jetzt manuell von `legacy_only` auf `graph` umstellen und ein Workflow-Definition mit BLENDER_CINEMATIC-Node zuweisen. diff --git a/backend/app/config.py b/backend/app/config.py index 126e41c..5aada25 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -75,7 +75,14 @@ class Settings(BaseSettings): # Redis / Celery redis_url: str = "redis://localhost:6379/0" + # Queue for shadow-mode workflow renders (second GPU worker). workflow_shadow_render_queue: str = "asset_pipeline_light" + # When non-empty AND that queue has active workers, still renders from the + # legacy dispatch path are routed here instead of asset_pipeline, enabling + # concurrent still rendering on multi-GPU setups. + # Set MULTI_GPU_LIGHT_RENDER_QUEUE=asset_pipeline_light in docker-compose + # for the render-worker-light service and restart the workers. + multi_gpu_light_render_queue: str = "" @model_validator(mode="after") def normalize_runtime_hosts(self) -> "Settings": diff --git a/backend/app/domains/pipeline/tasks/render_order_line.py b/backend/app/domains/pipeline/tasks/render_order_line.py index 2af7484..e4e4d4c 100644 --- a/backend/app/domains/pipeline/tasks/render_order_line.py +++ b/backend/app/domains/pipeline/tasks/render_order_line.py @@ -42,13 +42,32 @@ def dispatch_order_line_render(order_line_id: str): logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching") return - # All renders go to asset_pipeline (single-GPU default). - # For multi-GPU setups: enable render-worker-light in docker-compose - # and change target_queue logic below to route small stills to - # asset_pipeline_light for concurrent rendering. - pass + # Multi-GPU routing: route still renders to a secondary GPU queue when + # MULTI_GPU_LIGHT_RENDER_QUEUE is configured and that queue is active. + is_still_render = False + if line: + from sqlalchemy.orm import selectinload + line_full = session.execute( + select(OrderLine) + .options(selectinload(OrderLine.output_type)) + .where(OrderLine.id == order_line_id) + ).scalar_one_or_none() + if line_full and line_full.output_type: + rs = line_full.output_type.render_settings or {} + is_still_render = not rs.get("animation") and not rs.get("cinematic") + light_queue = app_settings.multi_gpu_light_render_queue.strip() target_queue = "asset_pipeline" + if light_queue and is_still_render: + from app.domains.rendering.workflow_graph_runtime import _inspect_active_worker_queues + active_queues = _inspect_active_worker_queues(timeout=0.5) + if light_queue in active_queues: + target_queue = light_queue + logger.info( + "Multi-GPU routing: order_line %s (still) -> queue=%s", + order_line_id, + target_queue, + ) logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}") render_order_line_task.apply_async(args=[order_line_id], queue=target_queue)