"""Order-line render tasks. Covers: - dispatch_order_line_render — thin router that queues render_order_line_task - render_order_line_task — full still/turntable render pipeline for one order line """ import logging from datetime import datetime from app.tasks.celery_app import celery_app from app.core.task_logs import log_task_event from app.core.pipeline_logger import PipelineLogger logger = logging.getLogger(__name__) @celery_app.task(name="app.tasks.step_tasks.dispatch_order_line_render", queue="step_processing") def dispatch_order_line_render(order_line_id: str): """Route an order-line render to render_order_line_task.""" # Pre-check: skip if line is already cancelled or order is rejected from sqlalchemy import create_engine, select from sqlalchemy.orm import Session from app.config import settings as app_settings from app.models.order_line import OrderLine from app.domains.orders.models import Order, OrderStatus sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: line = session.execute( select(OrderLine).where(OrderLine.id == order_line_id) ).scalar_one_or_none() if line and line.render_status == "cancelled": logger.info(f"OrderLine {order_line_id} cancelled — not dispatching") return if line: order = session.execute( select(Order).where(Order.id == line.order_id) ).scalar_one_or_none() if order and order.status in (OrderStatus.rejected, OrderStatus.completed): logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching") return # All renders go to asset_pipeline (single-GPU default). # For multi-GPU setups: enable render-worker-light in docker-compose # and change target_queue logic below to route small stills to # asset_pipeline_light for concurrent rendering. pass target_queue = "asset_pipeline" logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}") render_order_line_task.apply_async(args=[order_line_id], queue=target_queue) @celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3) def render_order_line_task(self, order_line_id: str): """Render a specific output type for an order line. Loads OrderLine → Product → CadFile → OutputType.render_settings. Merges with system render settings. Stores result at order_line.result_path. """ pl = PipelineLogger(task_id=self.request.id, order_line_id=order_line_id) pl.step_start("render_order_line_task", {"order_line_id": order_line_id}) logger.info(f"Rendering order line: {order_line_id}") # Resolve and log tenant context at task start (required for RLS) from app.core.tenant_context import resolve_tenant_id_for_order_line, set_tenant_context_sync _tenant_id = resolve_tenant_id_for_order_line(order_line_id) from app.services.render_log import emit emit(order_line_id, "Celery render task started") try: from sqlalchemy import create_engine from sqlalchemy.orm import Session from app.config import settings as app_settings from app.domains.rendering.workflow_runtime_services import ( build_order_line_render_invocation, emit_order_line_render_notifications, persist_order_line_output, prepare_order_line_render_context, resolve_order_line_template_context, resolve_render_position_context, ) # Use sync session for Celery (no async event loop) sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: set_tenant_context_sync(session, _tenant_id) from pathlib import Path as _Path setup = prepare_order_line_render_context( session, order_line_id, emit=emit, ) if not setup.is_ready: return line = setup.order_line cad_file = setup.cad_file part_colors = setup.part_colors render_start = setup.render_start template_context = resolve_order_line_template_context( session, setup, emit=emit, ) template = template_context.template material_map = template_context.material_map cad_name = cad_file.original_name if cad_file else "?" position_context = resolve_render_position_context(session, line, emit=emit) rotation_x = position_context.rotation_x rotation_y = position_context.rotation_y rotation_z = position_context.rotation_z focal_length_mm = position_context.focal_length_mm sensor_width_mm = position_context.sensor_width_mm render_invocation = build_order_line_render_invocation( setup, template_context=template_context, position_context=position_context, ) emit(order_line_id, f"Starting render for {cad_name} ({len(part_colors)} coloured parts)") if getattr(line, "render_overrides", None): emit(order_line_id, f"Render overrides active: {line.render_overrides}") if ( line.output_type and line.output_type.render_settings and render_invocation.samples is not None and line.output_type.render_settings.get("samples") and render_invocation.width is not None and render_invocation.height is not None ): base_samples = int(line.output_type.render_settings["samples"]) if render_invocation.samples < base_samples: emit( order_line_id, f"Auto-scaled samples {base_samples} -> {render_invocation.samples} " f"for {render_invocation.width}x{render_invocation.height}", ) is_animation = render_invocation.is_animation is_cinematic = render_invocation.is_cinematic product_name = render_invocation.product_name ot_name = render_invocation.output_type_name output_path = render_invocation.output_path _Path(output_path).parent.mkdir(parents=True, exist_ok=True) render_width = render_invocation.width render_height = render_invocation.height render_engine = render_invocation.engine render_samples = render_invocation.samples frame_count = render_invocation.frame_count fps = render_invocation.fps bg_color = render_invocation.bg_color turntable_axis = render_invocation.turntable_axis noise_threshold = render_invocation.noise_threshold denoiser = render_invocation.denoiser denoising_input_passes = render_invocation.denoising_input_passes denoising_prefilter = render_invocation.denoising_prefilter denoising_quality = render_invocation.denoising_quality denoising_use_gpu = render_invocation.denoising_use_gpu transparent_bg = render_invocation.transparent_bg cycles_device_val = render_invocation.cycles_device part_names_ordered = render_invocation.part_names_ordered material_override = render_invocation.material_override target_collection = render_invocation.target_collection template_path = render_invocation.template_path material_library_path = render_invocation.material_library_path camera_orbit = render_invocation.camera_orbit lighting_only = render_invocation.lighting_only shadow_catcher = render_invocation.shadow_catcher usd_path = render_invocation.usd_path step_path = _Path(cad_file.stored_path) tmpl_info = f" template={template.name}" if template else "" if is_cinematic: # ── Cinematic highlight animation path ────────────────────── # Use frame_count/fps from output_type.render_settings (already extracted above) _cine_fps = fps # extracted from render_settings, default 25 _cine_frames = frame_count # extracted from render_settings, default 24 emit(order_line_id, f"Starting cinematic render: {_cine_frames} frames @ {_cine_fps}fps, {render_width or 1920}x{render_height or 1080}{tmpl_info}") pl.step_start("blender_cinematic", {"frame_count": _cine_frames, "fps": _cine_fps}) from app.services.render_blender import is_blender_available, render_cinematic_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: cinematic_kwargs = render_invocation.as_cinematic_renderer_kwargs( step_path=step_path, output_path=_Path(output_path), default_width=1920, default_height=1080, default_engine=_sys.get("blender_engine", "cycles"), default_samples=int( _sys.get( f"blender_{render_engine or _sys.get('blender_engine', 'cycles')}_samples", 128, ) ), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), log_callback=lambda line: emit(order_line_id, line), ) service_data = render_cinematic_to_file(**cinematic_kwargs) success = True render_log = { "renderer": "blender", "type": "cinematic", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1080, "frame_count": service_data.get("frame_count", _cine_frames), "fps": _cine_fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_cinematic") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "cinematic", "error": str(exc)[:500]} pl.step_error("blender_cinematic", str(exc), exc) logger.error("Cinematic render failed for %s: %s", order_line_id, exc) elif is_animation: # ── Turntable animation path ──────────────────────────────── emit(order_line_id, f"Starting turntable render: {frame_count} frames @ {fps}fps, {render_width or 1920}x{render_height or 1920}{tmpl_info}") pl.step_start("blender_turntable", {"frame_count": frame_count, "fps": fps}) from app.services.render_blender import is_blender_available, render_turntable_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: turntable_kwargs = render_invocation.as_turntable_renderer_kwargs( step_path=step_path, output_path=_Path(output_path), default_width=1920, default_height=1920, default_engine=_sys.get("blender_engine", "cycles"), default_samples=int( _sys.get( f"blender_{render_engine or _sys.get('blender_engine', 'cycles')}_samples", 128, ) ), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), ) service_data = render_turntable_to_file(**turntable_kwargs) success = True render_log = { "renderer": "blender", "type": "turntable", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1920, "frame_count": service_data.get("frame_count", frame_count), "fps": fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_turntable") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "turntable", "error": str(exc)[:500]} pl.step_error("blender_turntable", str(exc), exc) logger.error("Turntable render failed for %s: %s", order_line_id, exc) else: # ── Still image path ──────────────────────────────────────── _render_path_label = "USD → Blender" if usd_path else "STEP → GLB → Blender" emit(order_line_id, f"Calling renderer ({_render_path_label}) {render_width or 'default'}x{render_height or 'default'}{' [transparent]' if transparent_bg else ''}{f' engine={render_engine}' if render_engine else ''}{f' samples={render_samples}' if render_samples else ''}{tmpl_info}") pl.step_start("blender_still", {"width": render_width, "height": render_height}) from app.services.step_processor import render_to_file success, render_log = render_to_file( **render_invocation.as_still_renderer_kwargs( step_path=cad_file.stored_path, output_path=output_path, job_id=order_line_id, order_line_id=order_line_id, ) ) if success: pl.step_done("blender_still") else: pl.step_error("blender_still", "render_to_file returned False") render_end = datetime.utcnow() elapsed = (render_end - render_start).total_seconds() try: persist_order_line_output( session, line, success=success, output_path=output_path, render_log=render_log if isinstance(render_log, dict) else None, render_completed_at=render_end, ) except Exception: logger.exception("Failed to persist render output for order_line %s", order_line_id) raise if success: emit(order_line_id, f"Render completed in {elapsed:.1f}s", "success") else: emit(order_line_id, f"Render failed after {elapsed:.1f}s", "error") emit_order_line_render_notifications( success=success, order_line_id=order_line_id, tenant_id=str(line.product.cad_file.tenant_id) if ( line.product and line.product.cad_file and line.product.cad_file.tenant_id ) else None, product_name=product_name, output_type_name=ot_name, render_log=render_log if isinstance(render_log, dict) else None, session=session, line=line, ) # Check if all lines for this order are done → auto-advance order_id_str = str(line.order_id) engine.dispose() from app.services.order_status_service import check_order_completion check_order_completion(order_id_str) pl.step_done("render_order_line_task") except Exception as exc: logger.error(f"render_order_line_task failed for {order_line_id}: {exc}") # If retries exhausted, mark as failed so the line doesn't stay stuck if self.request.retries >= self.max_retries: logger.error(f"Max retries reached for {order_line_id}, marking as failed") try: from sqlalchemy import create_engine, update as sql_update2 from sqlalchemy.orm import Session as SyncSession from app.config import settings as app_settings from app.models.order_line import OrderLine as OL2 sync_url2 = app_settings.database_url.replace("+asyncpg", "") eng2 = create_engine(sync_url2) with SyncSession(eng2) as s2: set_tenant_context_sync(s2, _tenant_id) from datetime import datetime as dt2 s2.execute( sql_update2(OL2).where(OL2.id == order_line_id) .values( render_status="failed", render_completed_at=dt2.utcnow(), render_log={"error": str(exc)[:500]}, ) ) s2.commit() eng2.dispose() from app.services.order_status_service import check_order_completion # Try to get order_id from DB eng3 = create_engine(sync_url2) with SyncSession(eng3) as s3: set_tenant_context_sync(s3, _tenant_id) from sqlalchemy import select as sel row = s3.execute(sel(OL2.order_id).where(OL2.id == order_line_id)).scalar_one_or_none() if row: check_order_completion(str(row)) eng3.dispose() # Notify the order creator about the failure try: from sqlalchemy import select as sel2 from app.models.order import Order as OrderModel2 from app.domains.rendering.workflow_runtime_services import ( emit_order_line_render_notifications, ) eng4 = create_engine(sync_url2) with SyncSession(eng4) as s4: set_tenant_context_sync(s4, _tenant_id) order_row2 = s4.execute( sel2(OrderModel2.created_by, OrderModel2.order_number) .join(OL2, OL2.order_id == OrderModel2.id) .where(OL2.id == order_line_id) ).one_or_none() eng4.dispose() if order_row2: emit_order_line_render_notifications( success=False, order_line_id=order_line_id, order_number=order_row2[1], order_creator_id=str(order_row2[0]), product_name="unknown", output_type_name="unknown", render_log={"error": str(exc)}, emit_websocket=False, activity_entity_id=None, ) except Exception: logger.exception("Failed to emit render failure activity event") except Exception: logger.exception(f"Failed to mark {order_line_id} as failed in DB") raise raise self.retry(exc=exc, countdown=60)