"""Order-line render tasks. Covers: - dispatch_order_line_render — thin router that queues render_order_line_task - render_order_line_task — full still/turntable render pipeline for one order line """ import logging from datetime import datetime from app.tasks.celery_app import celery_app from app.core.task_logs import log_task_event from app.core.pipeline_logger import PipelineLogger logger = logging.getLogger(__name__) @celery_app.task(name="app.tasks.step_tasks.dispatch_order_line_render", queue="step_processing") def dispatch_order_line_render(order_line_id: str): """Route an order-line render to render_order_line_task.""" # Pre-check: skip if line is already cancelled or order is rejected from sqlalchemy import create_engine, select from sqlalchemy.orm import Session from app.config import settings as app_settings from app.models.order_line import OrderLine from app.domains.orders.models import Order, OrderStatus sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: line = session.execute( select(OrderLine).where(OrderLine.id == order_line_id) ).scalar_one_or_none() if line and line.render_status == "cancelled": logger.info(f"OrderLine {order_line_id} cancelled — not dispatching") return if line: order = session.execute( select(Order).where(Order.id == line.order_id) ).scalar_one_or_none() if order and order.status in (OrderStatus.rejected, OrderStatus.completed): logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching") return # All renders go to asset_pipeline (single-GPU default). # For multi-GPU setups: enable render-worker-light in docker-compose # and change target_queue logic below to route small stills to # asset_pipeline_light for concurrent rendering. pass target_queue = "asset_pipeline" logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}") render_order_line_task.apply_async(args=[order_line_id], queue=target_queue) @celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3) def render_order_line_task(self, order_line_id: str): """Render a specific output type for an order line. Loads OrderLine → Product → CadFile → OutputType.render_settings. Merges with system render settings. Stores result at order_line.result_path. """ pl = PipelineLogger(task_id=self.request.id, order_line_id=order_line_id) pl.step_start("render_order_line_task", {"order_line_id": order_line_id}) logger.info(f"Rendering order line: {order_line_id}") # Resolve and log tenant context at task start (required for RLS) from app.core.tenant_context import resolve_tenant_id_for_order_line, set_tenant_context_sync _tenant_id = resolve_tenant_id_for_order_line(order_line_id) from app.services.render_log import emit emit(order_line_id, "Celery render task started") try: from sqlalchemy import create_engine from sqlalchemy.orm import Session from app.config import settings as app_settings from app.domains.rendering.workflow_runtime_services import ( emit_order_line_render_notifications, persist_order_line_output, prepare_order_line_render_context, resolve_order_line_template_context, resolve_render_position_context, ) # Use sync session for Celery (no async event loop) sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: set_tenant_context_sync(session, _tenant_id) from pathlib import Path as _Path setup = prepare_order_line_render_context( session, order_line_id, emit=emit, ) if not setup.is_ready: return line = setup.order_line cad_file = setup.cad_file materials_source = setup.materials_source usd_render_path = setup.usd_render_path glb_reuse_path = setup.glb_reuse_path part_colors = setup.part_colors render_start = setup.render_start template_context = resolve_order_line_template_context( session, setup, emit=emit, ) template = template_context.template material_library = template_context.material_library material_map = template_context.material_map use_materials = template_context.use_materials override_mat = template_context.override_material cad_name = cad_file.original_name if cad_file else "?" position_context = resolve_render_position_context(session, line, emit=emit) rotation_x = position_context.rotation_x rotation_y = position_context.rotation_y rotation_z = position_context.rotation_z focal_length_mm = position_context.focal_length_mm sensor_width_mm = position_context.sensor_width_mm emit(order_line_id, f"Starting render for {cad_name} ({len(part_colors)} coloured parts)") # Determine if this is an animation output type is_animation = bool(line.output_type and getattr(line.output_type, 'is_animation', False)) # Detect cinematic render type (render_settings.cinematic flag) is_cinematic = bool( line.output_type and line.output_type.render_settings and line.output_type.render_settings.get("cinematic") ) # Determine output format/extension out_ext = "jpg" if line.output_type and line.output_type.output_format: fmt = line.output_type.output_format.lower() if fmt == "mp4": out_ext = "mp4" elif fmt == "webp": out_ext = "webp" elif fmt in ("png", "jpg", "jpeg"): out_ext = "png" if fmt == "png" else "jpg" # Build meaningful output filename import re def _sanitize(s: str) -> str: return re.sub(r'[^\w\-.]', '_', s.strip())[:100] product_name = line.product.name or line.product.pim_id or "product" ot_name = line.output_type.name if line.output_type else "render" filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}" # Render to per-line output directory from pathlib import Path as _Path render_dir = _Path(app_settings.upload_dir) / "renders" / order_line_id render_dir.mkdir(parents=True, exist_ok=True) output_path = str(render_dir / filename) # Extract per-output-type render settings render_width = None render_height = None render_engine = None render_samples = None noise_threshold = "" denoiser = "" denoising_input_passes = "" denoising_prefilter = "" denoising_quality = "" denoising_use_gpu = "" frame_count = 24 fps = 25 bg_color = "" turntable_axis = "world_z" if line.output_type and line.output_type.render_settings: rs = line.output_type.render_settings if rs.get("width"): render_width = int(rs["width"]) if rs.get("height"): render_height = int(rs["height"]) if rs.get("engine"): render_engine = rs["engine"] if rs.get("samples"): render_samples = int(rs["samples"]) if rs.get("frame_count"): frame_count = int(rs["frame_count"]) if rs.get("fps"): fps = int(rs["fps"]) bg_color = rs.get("bg_color", "") turntable_axis = rs.get("turntable_axis", "world_z") noise_threshold = str(rs.get("noise_threshold", "")) denoiser = str(rs.get("denoiser", "")) denoising_input_passes = str(rs.get("denoising_input_passes", "")) denoising_prefilter = str(rs.get("denoising_prefilter", "")) denoising_quality = str(rs.get("denoising_quality", "")) denoising_use_gpu = str(rs.get("denoising_use_gpu", "")) # Auto-scale samples for lower resolutions (thumbnails/previews). # Only applies when the output type provides both samples and dimensions. if render_samples and render_width and render_height: max_dim = max(render_width, render_height) if max_dim <= 1024: scaled = max(32, int(render_samples * max_dim / 2048)) if scaled < render_samples: emit(order_line_id, f"Auto-scaled samples {render_samples} \u2192 {scaled} for {render_width}x{render_height}") render_samples = scaled transparent_bg = bool(line.output_type and line.output_type.transparent_bg) cycles_device_val = (line.output_type.cycles_device or "auto") if line.output_type else "auto" # Apply per-line render overrides (format, resolution, samples, etc.) _render_overrides = getattr(line, 'render_overrides', None) if _render_overrides and isinstance(_render_overrides, dict): if 'width' in _render_overrides: render_width = int(_render_overrides['width']) if 'height' in _render_overrides: render_height = int(_render_overrides['height']) if 'samples' in _render_overrides: render_samples = int(_render_overrides['samples']) if 'engine' in _render_overrides: render_engine = _render_overrides['engine'] if 'frame_count' in _render_overrides: frame_count = int(_render_overrides['frame_count']) if 'fps' in _render_overrides: fps = int(_render_overrides['fps']) if 'bg_color' in _render_overrides: bg_color = _render_overrides['bg_color'] if 'turntable_axis' in _render_overrides: turntable_axis = _render_overrides['turntable_axis'] if 'noise_threshold' in _render_overrides: noise_threshold = str(_render_overrides['noise_threshold']) if 'denoiser' in _render_overrides: denoiser = str(_render_overrides['denoiser']) if 'denoising_input_passes' in _render_overrides: denoising_input_passes = str(_render_overrides['denoising_input_passes']) if 'denoising_prefilter' in _render_overrides: denoising_prefilter = str(_render_overrides['denoising_prefilter']) if 'denoising_quality' in _render_overrides: denoising_quality = str(_render_overrides['denoising_quality']) if 'denoising_use_gpu' in _render_overrides: denoising_use_gpu = str(_render_overrides['denoising_use_gpu']) if 'transparent_bg' in _render_overrides: transparent_bg = bool(_render_overrides['transparent_bg']) if 'cycles_device' in _render_overrides: cycles_device_val = _render_overrides['cycles_device'] emit(order_line_id, f"Render overrides active: {_render_overrides}") # Apply output_format override (affects out_ext and filename) if 'output_format' in _render_overrides: fmt_override = _render_overrides['output_format'].lower() if fmt_override == "mp4": out_ext = "mp4" elif fmt_override == "webp": out_ext = "webp" elif fmt_override in ("png", "jpg", "jpeg"): out_ext = "png" if fmt_override == "png" else "jpg" # Rebuild filename with new extension filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}" output_path = str(render_dir / filename) # Build ordered part names list for index-based Blender matching part_names_ordered = None if cad_file and cad_file.parsed_objects: part_names_ordered = cad_file.parsed_objects.get("objects", []) or None tmpl_info = f" template={template.name}" if template else "" if is_cinematic: # ── Cinematic highlight animation path ────────────────────── # Use frame_count/fps from output_type.render_settings (already extracted above) _cine_fps = fps # extracted from render_settings, default 25 _cine_frames = frame_count # extracted from render_settings, default 24 emit(order_line_id, f"Starting cinematic render: {_cine_frames} frames @ {_cine_fps}fps, {render_width or 1920}x{render_height or 1080}{tmpl_info}") pl.step_start("blender_cinematic", {"frame_count": _cine_frames, "fps": _cine_fps}) from app.services.render_blender import is_blender_available, render_cinematic_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: service_data = render_cinematic_to_file( step_path=_Path(cad_file.stored_path), output_path=_Path(output_path), width=render_width or 1920, height=render_height or 1080, engine=render_engine or _sys.get("blender_engine", "cycles"), samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), cycles_device=cycles_device_val, transparent_bg=transparent_bg, part_colors=part_colors or None, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, usd_path=usd_render_path, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, log_callback=lambda line: emit(order_line_id, line), ) success = True render_log = { "renderer": "blender", "type": "cinematic", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1080, "frame_count": service_data.get("frame_count", _cine_frames), "fps": _cine_fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_cinematic") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "cinematic", "error": str(exc)[:500]} pl.step_error("blender_cinematic", str(exc), exc) logger.error("Cinematic render failed for %s: %s", order_line_id, exc) elif is_animation: # ── Turntable animation path ──────────────────────────────── emit(order_line_id, f"Starting turntable render: {frame_count} frames @ {fps}fps, {render_width or 1920}x{render_height or 1920}{tmpl_info}") pl.step_start("blender_turntable", {"frame_count": frame_count, "fps": fps}) from app.services.render_blender import is_blender_available, render_turntable_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: service_data = render_turntable_to_file( step_path=_Path(cad_file.stored_path), output_path=_Path(output_path), frame_count=frame_count, fps=fps, width=render_width or 1920, height=render_height or 1920, engine=render_engine or _sys.get("blender_engine", "cycles"), samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), cycles_device=cycles_device_val, transparent_bg=transparent_bg, bg_color=bg_color, turntable_axis=turntable_axis, part_colors=part_colors or None, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, camera_orbit=bool(template.camera_orbit) if template else True, usd_path=usd_render_path, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, ) success = True render_log = { "renderer": "blender", "type": "turntable", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1920, "frame_count": service_data.get("frame_count", frame_count), "fps": fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_turntable") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "turntable", "error": str(exc)[:500]} pl.step_error("blender_turntable", str(exc), exc) logger.error("Turntable render failed for %s: %s", order_line_id, exc) else: # ── Still image path ──────────────────────────────────────── _render_path_label = "USD → Blender" if usd_render_path else "STEP → GLB → Blender" emit(order_line_id, f"Calling renderer ({_render_path_label}) {render_width or 'default'}x{render_height or 'default'}{' [transparent]' if transparent_bg else ''}{f' engine={render_engine}' if render_engine else ''}{f' samples={render_samples}' if render_samples else ''}{tmpl_info}") pl.step_start("blender_still", {"width": render_width, "height": render_height}) from app.services.step_processor import render_to_file success, render_log = render_to_file( step_path=cad_file.stored_path, output_path=output_path, part_colors=part_colors, width=render_width, height=render_height, transparent_bg=transparent_bg, engine=render_engine, samples=render_samples, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, cycles_device=line.output_type.cycles_device if line.output_type else None, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, job_id=order_line_id, order_line_id=order_line_id, noise_threshold=noise_threshold, denoiser=denoiser, denoising_input_passes=denoising_input_passes, denoising_prefilter=denoising_prefilter, denoising_quality=denoising_quality, denoising_use_gpu=denoising_use_gpu, usd_path=usd_render_path, ) if success: pl.step_done("blender_still") else: pl.step_error("blender_still", "render_to_file returned False") render_end = datetime.utcnow() elapsed = (render_end - render_start).total_seconds() try: persist_order_line_output( session, line, success=success, output_path=output_path, render_log=render_log if isinstance(render_log, dict) else None, render_completed_at=render_end, ) except Exception: logger.exception("Failed to persist render output for order_line %s", order_line_id) raise if success: emit(order_line_id, f"Render completed in {elapsed:.1f}s", "success") else: emit(order_line_id, f"Render failed after {elapsed:.1f}s", "error") emit_order_line_render_notifications( success=success, order_line_id=order_line_id, tenant_id=str(line.product.cad_file.tenant_id) if ( line.product and line.product.cad_file and line.product.cad_file.tenant_id ) else None, product_name=product_name, output_type_name=ot_name, render_log=render_log if isinstance(render_log, dict) else None, session=session, line=line, ) # Check if all lines for this order are done → auto-advance order_id_str = str(line.order_id) engine.dispose() from app.services.order_status_service import check_order_completion check_order_completion(order_id_str) pl.step_done("render_order_line_task") except Exception as exc: logger.error(f"render_order_line_task failed for {order_line_id}: {exc}") # If retries exhausted, mark as failed so the line doesn't stay stuck if self.request.retries >= self.max_retries: logger.error(f"Max retries reached for {order_line_id}, marking as failed") try: from sqlalchemy import create_engine, update as sql_update2 from sqlalchemy.orm import Session as SyncSession from app.config import settings as app_settings from app.models.order_line import OrderLine as OL2 sync_url2 = app_settings.database_url.replace("+asyncpg", "") eng2 = create_engine(sync_url2) with SyncSession(eng2) as s2: set_tenant_context_sync(s2, _tenant_id) from datetime import datetime as dt2 s2.execute( sql_update2(OL2).where(OL2.id == order_line_id) .values( render_status="failed", render_completed_at=dt2.utcnow(), render_log={"error": str(exc)[:500]}, ) ) s2.commit() eng2.dispose() from app.services.order_status_service import check_order_completion # Try to get order_id from DB eng3 = create_engine(sync_url2) with SyncSession(eng3) as s3: set_tenant_context_sync(s3, _tenant_id) from sqlalchemy import select as sel row = s3.execute(sel(OL2.order_id).where(OL2.id == order_line_id)).scalar_one_or_none() if row: check_order_completion(str(row)) eng3.dispose() # Notify the order creator about the failure try: from sqlalchemy import select as sel2 from app.models.order import Order as OrderModel2 from app.domains.rendering.workflow_runtime_services import ( emit_order_line_render_notifications, ) eng4 = create_engine(sync_url2) with SyncSession(eng4) as s4: set_tenant_context_sync(s4, _tenant_id) order_row2 = s4.execute( sel2(OrderModel2.created_by, OrderModel2.order_number) .join(OL2, OL2.order_id == OrderModel2.id) .where(OL2.id == order_line_id) ).one_or_none() eng4.dispose() if order_row2: emit_order_line_render_notifications( success=False, order_line_id=order_line_id, order_number=order_row2[1], order_creator_id=str(order_row2[0]), product_name="unknown", output_type_name="unknown", render_log={"error": str(exc)}, emit_websocket=False, activity_entity_id=None, ) except Exception: logger.exception("Failed to emit render failure activity event") except Exception: logger.exception(f"Failed to mark {order_line_id} as failed in DB") raise raise self.retry(exc=exc, countdown=60)