diff --git a/backend/app/domains/pipeline/tasks/render_order_line.py b/backend/app/domains/pipeline/tasks/render_order_line.py index d7e644a..3743aac 100644 --- a/backend/app/domains/pipeline/tasks/render_order_line.py +++ b/backend/app/domains/pipeline/tasks/render_order_line.py @@ -40,8 +40,29 @@ def dispatch_order_line_render(order_line_id: str): logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching") return - logger.info(f"Dispatching render for order line: {order_line_id}") - render_order_line_task.delay(order_line_id) + # Route light renders (small stills) to asset_pipeline_light, + # heavy renders (HQ stills, animations) stay on asset_pipeline. + is_animation = False + max_dim = 0 + if line: + from app.models.output_type import OutputType + ot = session.execute( + select(OutputType).where(OutputType.id == line.output_type_id) + ).scalar_one_or_none() if line.output_type_id else None + if ot: + is_animation = bool(getattr(ot, 'is_animation', False)) + rs = ot.render_settings or {} + w = int(rs.get("width", 0) or 0) + h = int(rs.get("height", 0) or 0) + max_dim = max(w, h) + + if max_dim > 0 and max_dim <= 1024 and not is_animation: + target_queue = "asset_pipeline_light" + else: + target_queue = "asset_pipeline" + + logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}") + render_order_line_task.apply_async(args=[order_line_id], queue=target_queue) @celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3) @@ -165,10 +186,48 @@ def render_order_line_task(self, order_line_id: str): _usd_candidate.name, cad_file.id, ) + # Look up existing GLB geometry asset — reuse to skip re-tessellation + # when rendering via the GLB path (non-USD fallback). + glb_reuse_path = None + if cad_file and not usd_render_path: + _glb_asset = session.execute( + select(MediaAsset) + .where( + MediaAsset.cad_file_id == cad_file.id, + MediaAsset.asset_type == MediaAssetType.gltf_geometry, + ) + .order_by(MediaAsset.created_at.desc()) + .limit(1) + ).scalar_one_or_none() + if _glb_asset and _glb_asset.storage_key: + _glb_candidate = _Path(app_settings.upload_dir) / _glb_asset.storage_key + if _glb_candidate.exists() and _glb_candidate.stat().st_size > 0: + # Copy to the path render_blender.py expects so its + # local cache check (`glb_path.exists()`) finds it. + _step_path = _Path(cad_file.stored_path) + _expected_glb = _step_path.parent / f"{_step_path.stem}_thumbnail.glb" + if not _expected_glb.exists() or _expected_glb.stat().st_size == 0: + try: + import shutil as _shutil + _shutil.copy2(str(_glb_candidate), str(_expected_glb)) + logger.info( + "render_order_line: reused gltf_geometry asset %s -> %s", + _glb_candidate.name, _expected_glb.name, + ) + glb_reuse_path = _expected_glb + except Exception as _copy_exc: + logger.warning( + "render_order_line: failed to copy GLB asset: %s", _copy_exc, + ) + else: + glb_reuse_path = _expected_glb + if usd_render_path: emit(order_line_id, "Using USD master for render (skipping GLB tessellation)") + elif glb_reuse_path: + emit(order_line_id, f"Reusing cached GLB geometry ({glb_reuse_path.name}) — skipping re-tessellation") else: - emit(order_line_id, "No USD master available — using GLB tessellation path") + emit(order_line_id, "No USD master or cached GLB — will tessellate STEP -> GLB") part_colors = {} if cad_file and cad_file.parsed_objects: @@ -261,6 +320,8 @@ def render_order_line_task(self, order_line_id: str): fmt = line.output_type.output_format.lower() if fmt == "mp4": out_ext = "mp4" + elif fmt == "webp": + out_ext = "webp" elif fmt in ("png", "jpg", "jpeg"): out_ext = "png" if fmt == "png" else "jpg" @@ -479,7 +540,12 @@ def render_order_line_task(self, order_line_id: str): from app.domains.media.models import MediaAsset, MediaAssetType as MAT from app.config import settings as _cfg2 _ext = str(output_path).rsplit(".", 1)[-1].lower() if "." in str(output_path) else "bin" - _mime = "video/mp4" if _ext in ("mp4", "webm") else ("image/jpeg" if _ext in ("jpg", "jpeg") else "image/png") + _mime = ( + "video/mp4" if _ext in ("mp4", "webm") + else "image/webp" if _ext == "webp" + else "image/jpeg" if _ext in ("jpg", "jpeg") + else "image/png" + ) # Extension determines type — poster frames (.jpg/.png) from animations are still stills _at = MAT.turntable if _ext in ("mp4", "webm") else MAT.still _tenant_id = line.product.cad_file.tenant_id if (line.product and line.product.cad_file) else None diff --git a/backend/app/domains/pipeline/tasks/render_thumbnail.py b/backend/app/domains/pipeline/tasks/render_thumbnail.py index a5ac633..866fc49 100644 --- a/backend/app/domains/pipeline/tasks/render_thumbnail.py +++ b/backend/app/domains/pipeline/tasks/render_thumbnail.py @@ -72,7 +72,7 @@ def _pipeline_session(tenant_id: str | None = None): engine.dispose() -@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="asset_pipeline") +@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="asset_pipeline_light") def render_step_thumbnail(self, cad_file_id: str): """Render the thumbnail for a freshly-processed STEP file. @@ -188,7 +188,7 @@ def render_step_thumbnail(self, cad_file_id: str): pl.step_done("render_step_thumbnail") -@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="asset_pipeline") +@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="asset_pipeline_light") def regenerate_thumbnail(self, cad_file_id: str, part_colors: dict): """Regenerate thumbnail with per-part colours.""" pl = PipelineLogger(task_id=self.request.id) diff --git a/backend/app/services/step_processor.py b/backend/app/services/step_processor.py index 40a461c..0178054 100644 --- a/backend/app/services/step_processor.py +++ b/backend/app/services/step_processor.py @@ -807,7 +807,21 @@ def _generate_thumbnail( def _finalise_image(src: Path, dst: Path) -> Path | None: - """Move src image to dst, always as PNG.""" + """Move src image to dst. When dst has a .webp suffix, convert via Pillow + (quality=90, method=4) for 50-70 % smaller files. Otherwise output PNG.""" + if dst.suffix.lower() == ".webp": + try: + from PIL import Image + img = Image.open(str(src)) + out = dst.with_suffix(".webp") + img.save(str(out), "WebP", quality=90, method=4) + src.unlink(missing_ok=True) + return out + except Exception: + logger.warning("WebP conversion failed — falling back to PNG") + out = dst.with_suffix(".png") + src.rename(out) + return out out = dst.with_suffix(".png") src.rename(out) return out @@ -927,7 +941,7 @@ def render_to_file( settings = _get_all_settings() renderer = settings["thumbnail_renderer"] fmt = out.suffix.lstrip(".") or settings.get("thumbnail_format", "jpg") - if fmt not in ("jpg", "png"): + if fmt not in ("jpg", "png", "webp"): fmt = "jpg" # Temporary PNG for service renderers diff --git a/docker-compose.yml b/docker-compose.yml index 0e5e99b..521934c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,47 @@ services: count: 1 capabilities: [gpu, compute, utility, graphics] + render-worker-light: + build: + context: . + dockerfile: render-worker/Dockerfile + args: + - BLENDER_VERSION=${BLENDER_VERSION:-5.0.1} + command: bash -c "python3 /check_version.py && celery -A app.tasks.celery_app worker --loglevel=info -Q asset_pipeline_light --autoscale=2,2 --concurrency=2" + environment: + - POSTGRES_DB=${POSTGRES_DB:-schaeffler} + - POSTGRES_USER=${POSTGRES_USER:-schaeffler} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-schaeffler} + - POSTGRES_HOST=postgres + - POSTGRES_PORT=5432 + - REDIS_URL=${REDIS_URL:-redis://redis:6379/0} + - JWT_SECRET_KEY=${JWT_SECRET_KEY:-changeme-in-production} + - UPLOAD_DIR=/app/uploads + - BLENDER_BIN=/opt/blender/blender + - RENDER_SCRIPTS_DIR=/render-scripts + - CYCLES_DEVICE=${CYCLES_DEVICE:-auto} + - MINIO_URL=${MINIO_URL:-http://minio:9000} + - MINIO_USER=${MINIO_USER:-minioadmin} + - MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin} + - MINIO_BUCKET=${MINIO_BUCKET:-uploads} + volumes: + - ./backend:/app + - uploads:/app/uploads + - /opt/blender:/opt/blender:ro + - optix-cache:/var/tmp/OptixCache_root + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu, compute, utility, graphics] + beat: build: context: ./backend diff --git a/plan.md b/plan.md index 93f2d47..45ce6ac 100644 --- a/plan.md +++ b/plan.md @@ -41,7 +41,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser): - **Dependencies**: None - **Risk**: Low — Blender 5.0 supports this; increases VRAM usage slightly -### [ ] Task 4: Dual render queue for light/heavy workloads +### [x] Task 4: Dual render queue for light/heavy workloads - **Files**: - `docker-compose.yml` — add second render-worker service for light tasks @@ -55,7 +55,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser): - **Dependencies**: Task 1 (lower samples for light queue makes concurrent rendering safer) - **Risk**: Medium — VRAM contention if both workers render simultaneously. Mitigated by thumbnails being small (512x512, 64 samples = minimal VRAM) -### [ ] Task 5: Skip re-tessellation when GLB already exists +### [x] Task 5: Skip re-tessellation when GLB already exists - **File**: `backend/app/services/render_blender.py` - **What**: In `render_still()`, the STEP→GLB tessellation runs every time. Cache the GLB file per CAD file (already stored as `gltf_geometry` MediaAsset). Before tessellating, check if a GLB MediaAsset exists for this cad_file_id and reuse it. @@ -64,7 +64,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser): - **Dependencies**: Task 2 (USD path is preferred; this is fallback for products without USD) - **Risk**: Low — GLB is deterministic per CAD file; if the CAD file changes, a new GLB is generated -### [ ] Task 6: Output format optimization (WebP for stills) +### [x] Task 6: Output format optimization (WebP for stills) - **File**: `render-worker/scripts/_blender_scene_setup.py` (or `blender_render.py`) - **What**: After Blender renders a PNG, optionally convert to WebP for 50-70% smaller files. Add a `webp` output format option to OutputType. When selected, render as PNG then convert via Pillow.