perf: dual queue, GLB caching, WebP output, persistent BVH
Task 4: Dual render queue - render-worker: heavy (asset_pipeline, concurrency=1) — HQ 2048x2048, animations - render-worker-light: light (asset_pipeline_light, concurrency=2) — thumbnails, <=1024 - Thumbnails routed to light queue automatically - Order line renders routed by resolution at dispatch time Task 5: GLB caching (skip re-tessellation) - Before tessellating, check if gltf_geometry MediaAsset exists for the cad_file_id - If found, copy to expected path — render_blender.py finds it and skips tessellation - Saves 7-11s per re-render of the same product Task 6: WebP output format - New 'webp' option in output_format (OutputType admin) - Blender renders PNG intermediate, Pillow converts to WebP (quality=90, method=4) - 50-70% smaller files with no visible quality loss - Correct MIME type (image/webp) in MediaAsset Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,8 +40,29 @@ def dispatch_order_line_render(order_line_id: str):
|
||||
logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching")
|
||||
return
|
||||
|
||||
logger.info(f"Dispatching render for order line: {order_line_id}")
|
||||
render_order_line_task.delay(order_line_id)
|
||||
# Route light renders (small stills) to asset_pipeline_light,
|
||||
# heavy renders (HQ stills, animations) stay on asset_pipeline.
|
||||
is_animation = False
|
||||
max_dim = 0
|
||||
if line:
|
||||
from app.models.output_type import OutputType
|
||||
ot = session.execute(
|
||||
select(OutputType).where(OutputType.id == line.output_type_id)
|
||||
).scalar_one_or_none() if line.output_type_id else None
|
||||
if ot:
|
||||
is_animation = bool(getattr(ot, 'is_animation', False))
|
||||
rs = ot.render_settings or {}
|
||||
w = int(rs.get("width", 0) or 0)
|
||||
h = int(rs.get("height", 0) or 0)
|
||||
max_dim = max(w, h)
|
||||
|
||||
if max_dim > 0 and max_dim <= 1024 and not is_animation:
|
||||
target_queue = "asset_pipeline_light"
|
||||
else:
|
||||
target_queue = "asset_pipeline"
|
||||
|
||||
logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}")
|
||||
render_order_line_task.apply_async(args=[order_line_id], queue=target_queue)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3)
|
||||
@@ -165,10 +186,48 @@ def render_order_line_task(self, order_line_id: str):
|
||||
_usd_candidate.name, cad_file.id,
|
||||
)
|
||||
|
||||
# Look up existing GLB geometry asset — reuse to skip re-tessellation
|
||||
# when rendering via the GLB path (non-USD fallback).
|
||||
glb_reuse_path = None
|
||||
if cad_file and not usd_render_path:
|
||||
_glb_asset = session.execute(
|
||||
select(MediaAsset)
|
||||
.where(
|
||||
MediaAsset.cad_file_id == cad_file.id,
|
||||
MediaAsset.asset_type == MediaAssetType.gltf_geometry,
|
||||
)
|
||||
.order_by(MediaAsset.created_at.desc())
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if _glb_asset and _glb_asset.storage_key:
|
||||
_glb_candidate = _Path(app_settings.upload_dir) / _glb_asset.storage_key
|
||||
if _glb_candidate.exists() and _glb_candidate.stat().st_size > 0:
|
||||
# Copy to the path render_blender.py expects so its
|
||||
# local cache check (`glb_path.exists()`) finds it.
|
||||
_step_path = _Path(cad_file.stored_path)
|
||||
_expected_glb = _step_path.parent / f"{_step_path.stem}_thumbnail.glb"
|
||||
if not _expected_glb.exists() or _expected_glb.stat().st_size == 0:
|
||||
try:
|
||||
import shutil as _shutil
|
||||
_shutil.copy2(str(_glb_candidate), str(_expected_glb))
|
||||
logger.info(
|
||||
"render_order_line: reused gltf_geometry asset %s -> %s",
|
||||
_glb_candidate.name, _expected_glb.name,
|
||||
)
|
||||
glb_reuse_path = _expected_glb
|
||||
except Exception as _copy_exc:
|
||||
logger.warning(
|
||||
"render_order_line: failed to copy GLB asset: %s", _copy_exc,
|
||||
)
|
||||
else:
|
||||
glb_reuse_path = _expected_glb
|
||||
|
||||
if usd_render_path:
|
||||
emit(order_line_id, "Using USD master for render (skipping GLB tessellation)")
|
||||
elif glb_reuse_path:
|
||||
emit(order_line_id, f"Reusing cached GLB geometry ({glb_reuse_path.name}) — skipping re-tessellation")
|
||||
else:
|
||||
emit(order_line_id, "No USD master available — using GLB tessellation path")
|
||||
emit(order_line_id, "No USD master or cached GLB — will tessellate STEP -> GLB")
|
||||
|
||||
part_colors = {}
|
||||
if cad_file and cad_file.parsed_objects:
|
||||
@@ -261,6 +320,8 @@ def render_order_line_task(self, order_line_id: str):
|
||||
fmt = line.output_type.output_format.lower()
|
||||
if fmt == "mp4":
|
||||
out_ext = "mp4"
|
||||
elif fmt == "webp":
|
||||
out_ext = "webp"
|
||||
elif fmt in ("png", "jpg", "jpeg"):
|
||||
out_ext = "png" if fmt == "png" else "jpg"
|
||||
|
||||
@@ -479,7 +540,12 @@ def render_order_line_task(self, order_line_id: str):
|
||||
from app.domains.media.models import MediaAsset, MediaAssetType as MAT
|
||||
from app.config import settings as _cfg2
|
||||
_ext = str(output_path).rsplit(".", 1)[-1].lower() if "." in str(output_path) else "bin"
|
||||
_mime = "video/mp4" if _ext in ("mp4", "webm") else ("image/jpeg" if _ext in ("jpg", "jpeg") else "image/png")
|
||||
_mime = (
|
||||
"video/mp4" if _ext in ("mp4", "webm")
|
||||
else "image/webp" if _ext == "webp"
|
||||
else "image/jpeg" if _ext in ("jpg", "jpeg")
|
||||
else "image/png"
|
||||
)
|
||||
# Extension determines type — poster frames (.jpg/.png) from animations are still stills
|
||||
_at = MAT.turntable if _ext in ("mp4", "webm") else MAT.still
|
||||
_tenant_id = line.product.cad_file.tenant_id if (line.product and line.product.cad_file) else None
|
||||
|
||||
@@ -72,7 +72,7 @@ def _pipeline_session(tenant_id: str | None = None):
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="asset_pipeline")
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="asset_pipeline_light")
|
||||
def render_step_thumbnail(self, cad_file_id: str):
|
||||
"""Render the thumbnail for a freshly-processed STEP file.
|
||||
|
||||
@@ -188,7 +188,7 @@ def render_step_thumbnail(self, cad_file_id: str):
|
||||
pl.step_done("render_step_thumbnail")
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="asset_pipeline")
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="asset_pipeline_light")
|
||||
def regenerate_thumbnail(self, cad_file_id: str, part_colors: dict):
|
||||
"""Regenerate thumbnail with per-part colours."""
|
||||
pl = PipelineLogger(task_id=self.request.id)
|
||||
|
||||
@@ -807,7 +807,21 @@ def _generate_thumbnail(
|
||||
|
||||
|
||||
def _finalise_image(src: Path, dst: Path) -> Path | None:
|
||||
"""Move src image to dst, always as PNG."""
|
||||
"""Move src image to dst. When dst has a .webp suffix, convert via Pillow
|
||||
(quality=90, method=4) for 50-70 % smaller files. Otherwise output PNG."""
|
||||
if dst.suffix.lower() == ".webp":
|
||||
try:
|
||||
from PIL import Image
|
||||
img = Image.open(str(src))
|
||||
out = dst.with_suffix(".webp")
|
||||
img.save(str(out), "WebP", quality=90, method=4)
|
||||
src.unlink(missing_ok=True)
|
||||
return out
|
||||
except Exception:
|
||||
logger.warning("WebP conversion failed — falling back to PNG")
|
||||
out = dst.with_suffix(".png")
|
||||
src.rename(out)
|
||||
return out
|
||||
out = dst.with_suffix(".png")
|
||||
src.rename(out)
|
||||
return out
|
||||
@@ -927,7 +941,7 @@ def render_to_file(
|
||||
settings = _get_all_settings()
|
||||
renderer = settings["thumbnail_renderer"]
|
||||
fmt = out.suffix.lstrip(".") or settings.get("thumbnail_format", "jpg")
|
||||
if fmt not in ("jpg", "png"):
|
||||
if fmt not in ("jpg", "png", "webp"):
|
||||
fmt = "jpg"
|
||||
|
||||
# Temporary PNG for service renderers
|
||||
|
||||
@@ -156,6 +156,47 @@ services:
|
||||
count: 1
|
||||
capabilities: [gpu, compute, utility, graphics]
|
||||
|
||||
render-worker-light:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: render-worker/Dockerfile
|
||||
args:
|
||||
- BLENDER_VERSION=${BLENDER_VERSION:-5.0.1}
|
||||
command: bash -c "python3 /check_version.py && celery -A app.tasks.celery_app worker --loglevel=info -Q asset_pipeline_light --autoscale=2,2 --concurrency=2"
|
||||
environment:
|
||||
- POSTGRES_DB=${POSTGRES_DB:-schaeffler}
|
||||
- POSTGRES_USER=${POSTGRES_USER:-schaeffler}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-schaeffler}
|
||||
- POSTGRES_HOST=postgres
|
||||
- POSTGRES_PORT=5432
|
||||
- REDIS_URL=${REDIS_URL:-redis://redis:6379/0}
|
||||
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-changeme-in-production}
|
||||
- UPLOAD_DIR=/app/uploads
|
||||
- BLENDER_BIN=/opt/blender/blender
|
||||
- RENDER_SCRIPTS_DIR=/render-scripts
|
||||
- CYCLES_DEVICE=${CYCLES_DEVICE:-auto}
|
||||
- MINIO_URL=${MINIO_URL:-http://minio:9000}
|
||||
- MINIO_USER=${MINIO_USER:-minioadmin}
|
||||
- MINIO_PASSWORD=${MINIO_PASSWORD:-minioadmin}
|
||||
- MINIO_BUCKET=${MINIO_BUCKET:-uploads}
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
- uploads:/app/uploads
|
||||
- /opt/blender:/opt/blender:ro
|
||||
- optix-cache:/var/tmp/OptixCache_root
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: 1
|
||||
capabilities: [gpu, compute, utility, graphics]
|
||||
|
||||
beat:
|
||||
build:
|
||||
context: ./backend
|
||||
|
||||
@@ -41,7 +41,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser):
|
||||
- **Dependencies**: None
|
||||
- **Risk**: Low — Blender 5.0 supports this; increases VRAM usage slightly
|
||||
|
||||
### [ ] Task 4: Dual render queue for light/heavy workloads
|
||||
### [x] Task 4: Dual render queue for light/heavy workloads
|
||||
|
||||
- **Files**:
|
||||
- `docker-compose.yml` — add second render-worker service for light tasks
|
||||
@@ -55,7 +55,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser):
|
||||
- **Dependencies**: Task 1 (lower samples for light queue makes concurrent rendering safer)
|
||||
- **Risk**: Medium — VRAM contention if both workers render simultaneously. Mitigated by thumbnails being small (512x512, 64 samples = minimal VRAM)
|
||||
|
||||
### [ ] Task 5: Skip re-tessellation when GLB already exists
|
||||
### [x] Task 5: Skip re-tessellation when GLB already exists
|
||||
|
||||
- **File**: `backend/app/services/render_blender.py`
|
||||
- **What**: In `render_still()`, the STEP→GLB tessellation runs every time. Cache the GLB file per CAD file (already stored as `gltf_geometry` MediaAsset). Before tessellating, check if a GLB MediaAsset exists for this cad_file_id and reuse it.
|
||||
@@ -64,7 +64,7 @@ Current baseline (2048x2048, 256 samples, Cycles GPU, OIDN denoiser):
|
||||
- **Dependencies**: Task 2 (USD path is preferred; this is fallback for products without USD)
|
||||
- **Risk**: Low — GLB is deterministic per CAD file; if the CAD file changes, a new GLB is generated
|
||||
|
||||
### [ ] Task 6: Output format optimization (WebP for stills)
|
||||
### [x] Task 6: Output format optimization (WebP for stills)
|
||||
|
||||
- **File**: `render-worker/scripts/_blender_scene_setup.py` (or `blender_render.py`)
|
||||
- **What**: After Blender renders a PNG, optionally convert to WebP for 50-70% smaller files. Add a `webp` output format option to OutputType. When selected, render as PNG then convert via Pillow.
|
||||
|
||||
Reference in New Issue
Block a user