"""Order-line render tasks. Covers: - dispatch_order_line_render — thin router that queues render_order_line_task - render_order_line_task — full still/turntable render pipeline for one order line """ import logging from app.tasks.celery_app import celery_app from app.core.task_logs import log_task_event from app.core.pipeline_logger import PipelineLogger logger = logging.getLogger(__name__) @celery_app.task(name="app.tasks.step_tasks.dispatch_order_line_render", queue="step_processing") def dispatch_order_line_render(order_line_id: str): """Route an order-line render to render_order_line_task.""" # Pre-check: skip if line is already cancelled or order is rejected from sqlalchemy import create_engine, select from sqlalchemy.orm import Session from app.config import settings as app_settings from app.models.order_line import OrderLine from app.domains.orders.models import Order, OrderStatus sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: line = session.execute( select(OrderLine).where(OrderLine.id == order_line_id) ).scalar_one_or_none() if line and line.render_status == "cancelled": logger.info(f"OrderLine {order_line_id} cancelled — not dispatching") return if line: order = session.execute( select(Order).where(Order.id == line.order_id) ).scalar_one_or_none() if order and order.status in (OrderStatus.rejected, OrderStatus.completed): logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching") return # All renders go to asset_pipeline (single-GPU default). # For multi-GPU setups: enable render-worker-light in docker-compose # and change target_queue logic below to route small stills to # asset_pipeline_light for concurrent rendering. pass target_queue = "asset_pipeline" logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}") render_order_line_task.apply_async(args=[order_line_id], queue=target_queue) @celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3) def render_order_line_task(self, order_line_id: str): """Render a specific output type for an order line. Loads OrderLine → Product → CadFile → OutputType.render_settings. Merges with system render settings. Stores result at order_line.result_path. """ pl = PipelineLogger(task_id=self.request.id, order_line_id=order_line_id) pl.step_start("render_order_line_task", {"order_line_id": order_line_id}) logger.info(f"Rendering order line: {order_line_id}") # Resolve and log tenant context at task start (required for RLS) from app.core.tenant_context import resolve_tenant_id_for_order_line, set_tenant_context_sync _tenant_id = resolve_tenant_id_for_order_line(order_line_id) from app.services.render_log import emit emit(order_line_id, "Celery render task started") try: from sqlalchemy import create_engine, select, update as sql_update from sqlalchemy.orm import Session, joinedload from app.config import settings as app_settings # Use sync session for Celery (no async event loop) sync_url = app_settings.database_url.replace("+asyncpg", "") engine = create_engine(sync_url) with Session(engine) as session: set_tenant_context_sync(session, _tenant_id) from app.models.order_line import OrderLine from app.models.product import Product emit(order_line_id, "Loading order line from database") line = session.execute( select(OrderLine) .where(OrderLine.id == order_line_id) .options( joinedload(OrderLine.product).joinedload(Product.cad_file), joinedload(OrderLine.output_type), ) ).scalar_one_or_none() if line is None: emit(order_line_id, "Order line not found in database", "error") logger.error(f"OrderLine {order_line_id} not found") return # Skip if line was cancelled or order was rejected/completed if line.render_status == "cancelled": emit(order_line_id, "Order line already cancelled — skipping render") logger.info(f"OrderLine {order_line_id} cancelled — skipping") return from app.domains.orders.models import Order, OrderStatus order = session.execute( select(Order).where(Order.id == line.order_id) ).scalar_one_or_none() if order and order.status in (OrderStatus.rejected, OrderStatus.completed): emit(order_line_id, f"Order {order.status.value} — skipping render") logger.info(f"OrderLine {order_line_id}: order {order.status.value} — skipping") if line.render_status in ("pending", "processing"): session.execute( sql_update(OrderLine) .where(OrderLine.id == line.id) .values(render_status="cancelled") ) session.commit() return if line.product.cad_file_id is None: emit(order_line_id, "Product has no CAD file — marking as failed", "error") logger.warning(f"OrderLine {order_line_id}: product has no CAD file") session.execute( sql_update(OrderLine) .where(OrderLine.id == line.id) .values(render_status="failed") ) session.commit() return # Mark as processing with timing from datetime import datetime render_start = datetime.utcnow() session.execute( sql_update(OrderLine) .where(OrderLine.id == line.id) .values( render_status="processing", render_backend_used="celery", render_started_at=render_start, ) ) session.commit() cad_file = line.product.cad_file materials_source = line.product.cad_part_materials # Look up USD master asset for this CAD file — used when rendering # via USD path instead of production GLB from app.domains.media.models import MediaAsset, MediaAssetType from pathlib import Path as _Path usd_render_path = None if cad_file: _usd_asset = session.execute( select(MediaAsset) .where( MediaAsset.cad_file_id == cad_file.id, MediaAsset.asset_type == MediaAssetType.usd_master, ) .order_by(MediaAsset.created_at.desc()) .limit(1) ).scalar_one_or_none() if _usd_asset and _usd_asset.storage_key: _usd_candidate = _Path(app_settings.upload_dir) / _usd_asset.storage_key if _usd_candidate.exists(): usd_render_path = _usd_candidate logger.info( "render_order_line: using usd_master %s for cad %s", _usd_candidate.name, cad_file.id, ) # Look up existing GLB geometry asset — reuse to skip re-tessellation # when rendering via the GLB path (non-USD fallback). glb_reuse_path = None if cad_file and not usd_render_path: _glb_asset = session.execute( select(MediaAsset) .where( MediaAsset.cad_file_id == cad_file.id, MediaAsset.asset_type == MediaAssetType.gltf_geometry, ) .order_by(MediaAsset.created_at.desc()) .limit(1) ).scalar_one_or_none() if _glb_asset and _glb_asset.storage_key: _glb_candidate = _Path(app_settings.upload_dir) / _glb_asset.storage_key if _glb_candidate.exists() and _glb_candidate.stat().st_size > 0: # Copy to the path render_blender.py expects so its # local cache check (`glb_path.exists()`) finds it. _step_path = _Path(cad_file.stored_path) _expected_glb = _step_path.parent / f"{_step_path.stem}_thumbnail.glb" if not _expected_glb.exists() or _expected_glb.stat().st_size == 0: try: import shutil as _shutil _shutil.copy2(str(_glb_candidate), str(_expected_glb)) logger.info( "render_order_line: reused gltf_geometry asset %s -> %s", _glb_candidate.name, _expected_glb.name, ) glb_reuse_path = _expected_glb except Exception as _copy_exc: logger.warning( "render_order_line: failed to copy GLB asset: %s", _copy_exc, ) else: glb_reuse_path = _expected_glb if usd_render_path: emit(order_line_id, "Using USD master for render (skipping GLB tessellation)") elif glb_reuse_path: emit(order_line_id, f"Reusing cached GLB geometry ({glb_reuse_path.name}) — skipping re-tessellation") else: emit(order_line_id, "No USD master or cached GLB — will tessellate STEP -> GLB") part_colors = {} if cad_file and cad_file.parsed_objects: parsed_names = cad_file.parsed_objects.get("objects", []) if materials_source: from app.services.step_processor import build_part_colors part_colors = build_part_colors(parsed_names, materials_source) # Resolve render template + material library from app.services.template_service import resolve_template, get_material_library_path category_key = line.product.category_key if line.product else None ot_id = str(line.output_type_id) if line.output_type_id else None template = resolve_template(category_key=category_key, output_type_id=ot_id) material_library = get_material_library_path() # Build material_map (part_name → material_name) for material replacement. # Works with or without a render template — only suppressed if a # template explicitly has material_replace_enabled=False. material_map = None use_materials = bool(material_library and materials_source) if template and not template.material_replace_enabled: use_materials = False if use_materials: material_map = { m["part_name"]: m["material"] for m in materials_source if m.get("part_name") and m.get("material") } # Resolve raw material names to SCHAEFFLER library names via aliases from app.services.material_service import resolve_material_map material_map = resolve_material_map(material_map) # Apply material override: per-line override takes priority over output type override _line_override = getattr(line, 'material_override', None) _ot_override = line.output_type.material_override if line.output_type else None override_mat = _line_override or _ot_override if override_mat: # Build override map from existing material_map keys or from parsed STEP parts override_keys = set() if material_map: override_keys = set(material_map.keys()) if cad_file and cad_file.parsed_objects: for part_name in cad_file.parsed_objects.get("objects", []): override_keys.add(part_name) material_map = {k: override_mat for k in override_keys} use_materials = True emit(order_line_id, f"Material override active: {len(material_map)} parts → {override_mat}") if template: emit(order_line_id, f"Using render template: {template.name} (collection={template.target_collection}, material_replace={template.material_replace_enabled}, lighting_only={template.lighting_only})") logger.info(f"Render template resolved: '{template.name}' path={template.blend_file_path}, lighting_only={template.lighting_only}") else: emit(order_line_id, "No render template found — using factory settings (Mode A)") logger.info(f"No render template for category_key={category_key!r}, output_type_id={ot_id!r}") cad_name = cad_file.original_name if cad_file else "?" # Load render_position for rotation values (per-product takes priority, falls back to global) rotation_x = rotation_y = rotation_z = 0.0 focal_length_mm = None sensor_width_mm = None if line.render_position_id: from app.models.render_position import ProductRenderPosition rp = session.get(ProductRenderPosition, line.render_position_id) if rp: rotation_x, rotation_y, rotation_z = rp.rotation_x, rp.rotation_y, rp.rotation_z focal_length_mm = rp.focal_length_mm sensor_width_mm = rp.sensor_width_mm emit(order_line_id, f"Render position: '{rp.name}' ({rotation_x}°, {rotation_y}°, {rotation_z}°)" + (f" focal_length={focal_length_mm}mm" if focal_length_mm else "")) elif line.global_render_position_id: from app.models import GlobalRenderPosition grp = session.get(GlobalRenderPosition, line.global_render_position_id) if grp: rotation_x, rotation_y, rotation_z = grp.rotation_x, grp.rotation_y, grp.rotation_z focal_length_mm = grp.focal_length_mm sensor_width_mm = grp.sensor_width_mm emit(order_line_id, f"Global render position: '{grp.name}' ({rotation_x}°, {rotation_y}°, {rotation_z}°)" + (f" focal_length={focal_length_mm}mm" if focal_length_mm else "")) emit(order_line_id, f"Starting render for {cad_name} ({len(part_colors)} coloured parts)") # Determine if this is an animation output type is_animation = bool(line.output_type and getattr(line.output_type, 'is_animation', False)) # Detect cinematic render type (render_settings.cinematic flag) is_cinematic = bool( line.output_type and line.output_type.render_settings and line.output_type.render_settings.get("cinematic") ) # Determine output format/extension out_ext = "jpg" if line.output_type and line.output_type.output_format: fmt = line.output_type.output_format.lower() if fmt == "mp4": out_ext = "mp4" elif fmt == "webp": out_ext = "webp" elif fmt in ("png", "jpg", "jpeg"): out_ext = "png" if fmt == "png" else "jpg" # Build meaningful output filename import re def _sanitize(s: str) -> str: return re.sub(r'[^\w\-.]', '_', s.strip())[:100] product_name = line.product.name or line.product.pim_id or "product" ot_name = line.output_type.name if line.output_type else "render" filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}" # Render to per-line output directory from pathlib import Path as _Path render_dir = _Path(app_settings.upload_dir) / "renders" / order_line_id render_dir.mkdir(parents=True, exist_ok=True) output_path = str(render_dir / filename) # Extract per-output-type render settings render_width = None render_height = None render_engine = None render_samples = None noise_threshold = "" denoiser = "" denoising_input_passes = "" denoising_prefilter = "" denoising_quality = "" denoising_use_gpu = "" frame_count = 24 fps = 25 bg_color = "" turntable_axis = "world_z" if line.output_type and line.output_type.render_settings: rs = line.output_type.render_settings if rs.get("width"): render_width = int(rs["width"]) if rs.get("height"): render_height = int(rs["height"]) if rs.get("engine"): render_engine = rs["engine"] if rs.get("samples"): render_samples = int(rs["samples"]) if rs.get("frame_count"): frame_count = int(rs["frame_count"]) if rs.get("fps"): fps = int(rs["fps"]) bg_color = rs.get("bg_color", "") turntable_axis = rs.get("turntable_axis", "world_z") noise_threshold = str(rs.get("noise_threshold", "")) denoiser = str(rs.get("denoiser", "")) denoising_input_passes = str(rs.get("denoising_input_passes", "")) denoising_prefilter = str(rs.get("denoising_prefilter", "")) denoising_quality = str(rs.get("denoising_quality", "")) denoising_use_gpu = str(rs.get("denoising_use_gpu", "")) # Auto-scale samples for lower resolutions (thumbnails/previews). # Only applies when the output type provides both samples and dimensions. if render_samples and render_width and render_height: max_dim = max(render_width, render_height) if max_dim <= 1024: scaled = max(32, int(render_samples * max_dim / 2048)) if scaled < render_samples: emit(order_line_id, f"Auto-scaled samples {render_samples} \u2192 {scaled} for {render_width}x{render_height}") render_samples = scaled transparent_bg = bool(line.output_type and line.output_type.transparent_bg) cycles_device_val = (line.output_type.cycles_device or "auto") if line.output_type else "auto" # Apply per-line render overrides (format, resolution, samples, etc.) _render_overrides = getattr(line, 'render_overrides', None) if _render_overrides and isinstance(_render_overrides, dict): if 'width' in _render_overrides: render_width = int(_render_overrides['width']) if 'height' in _render_overrides: render_height = int(_render_overrides['height']) if 'samples' in _render_overrides: render_samples = int(_render_overrides['samples']) if 'engine' in _render_overrides: render_engine = _render_overrides['engine'] if 'frame_count' in _render_overrides: frame_count = int(_render_overrides['frame_count']) if 'fps' in _render_overrides: fps = int(_render_overrides['fps']) if 'bg_color' in _render_overrides: bg_color = _render_overrides['bg_color'] if 'turntable_axis' in _render_overrides: turntable_axis = _render_overrides['turntable_axis'] if 'noise_threshold' in _render_overrides: noise_threshold = str(_render_overrides['noise_threshold']) if 'denoiser' in _render_overrides: denoiser = str(_render_overrides['denoiser']) if 'denoising_input_passes' in _render_overrides: denoising_input_passes = str(_render_overrides['denoising_input_passes']) if 'denoising_prefilter' in _render_overrides: denoising_prefilter = str(_render_overrides['denoising_prefilter']) if 'denoising_quality' in _render_overrides: denoising_quality = str(_render_overrides['denoising_quality']) if 'denoising_use_gpu' in _render_overrides: denoising_use_gpu = str(_render_overrides['denoising_use_gpu']) if 'transparent_bg' in _render_overrides: transparent_bg = bool(_render_overrides['transparent_bg']) if 'cycles_device' in _render_overrides: cycles_device_val = _render_overrides['cycles_device'] emit(order_line_id, f"Render overrides active: {_render_overrides}") # Apply output_format override (affects out_ext and filename) if 'output_format' in _render_overrides: fmt_override = _render_overrides['output_format'].lower() if fmt_override == "mp4": out_ext = "mp4" elif fmt_override == "webp": out_ext = "webp" elif fmt_override in ("png", "jpg", "jpeg"): out_ext = "png" if fmt_override == "png" else "jpg" # Rebuild filename with new extension filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}" output_path = str(render_dir / filename) # Build ordered part names list for index-based Blender matching part_names_ordered = None if cad_file and cad_file.parsed_objects: part_names_ordered = cad_file.parsed_objects.get("objects", []) or None tmpl_info = f" template={template.name}" if template else "" if is_cinematic: # ── Cinematic highlight animation path ────────────────────── # Use frame_count/fps from output_type.render_settings (already extracted above) _cine_fps = fps # extracted from render_settings, default 25 _cine_frames = frame_count # extracted from render_settings, default 24 emit(order_line_id, f"Starting cinematic render: {_cine_frames} frames @ {_cine_fps}fps, {render_width or 1920}x{render_height or 1080}{tmpl_info}") pl.step_start("blender_cinematic", {"frame_count": _cine_frames, "fps": _cine_fps}) from app.services.render_blender import is_blender_available, render_cinematic_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: service_data = render_cinematic_to_file( step_path=_Path(cad_file.stored_path), output_path=_Path(output_path), width=render_width or 1920, height=render_height or 1080, engine=render_engine or _sys.get("blender_engine", "cycles"), samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), cycles_device=cycles_device_val, transparent_bg=transparent_bg, part_colors=part_colors or None, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, usd_path=usd_render_path, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, log_callback=lambda line: emit(order_line_id, line), ) success = True render_log = { "renderer": "blender", "type": "cinematic", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1080, "frame_count": service_data.get("frame_count", _cine_frames), "fps": _cine_fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_cinematic") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "cinematic", "error": str(exc)[:500]} pl.step_error("blender_cinematic", str(exc), exc) logger.error("Cinematic render failed for %s: %s", order_line_id, exc) elif is_animation: # ── Turntable animation path ──────────────────────────────── emit(order_line_id, f"Starting turntable render: {frame_count} frames @ {fps}fps, {render_width or 1920}x{render_height or 1920}{tmpl_info}") pl.step_start("blender_turntable", {"frame_count": frame_count, "fps": fps}) from app.services.render_blender import is_blender_available, render_turntable_to_file if not is_blender_available(): raise RuntimeError("Blender not available on this worker") from app.services.step_processor import _get_all_settings _sys = _get_all_settings() try: service_data = render_turntable_to_file( step_path=_Path(cad_file.stored_path), output_path=_Path(output_path), frame_count=frame_count, fps=fps, width=render_width or 1920, height=render_height or 1920, engine=render_engine or _sys.get("blender_engine", "cycles"), samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)), smooth_angle=int(_sys.get("blender_smooth_angle", 30)), cycles_device=cycles_device_val, transparent_bg=transparent_bg, bg_color=bg_color, turntable_axis=turntable_axis, part_colors=part_colors or None, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, camera_orbit=bool(template.camera_orbit) if template else True, usd_path=usd_render_path, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, ) success = True render_log = { "renderer": "blender", "type": "turntable", "format": "mp4", "engine": render_engine or _sys.get("blender_engine", "cycles"), "engine_used": service_data.get("engine_used", "cycles"), "samples": render_samples, "cycles_device": cycles_device_val, "width": render_width or 1920, "height": render_height or 1920, "frame_count": service_data.get("frame_count", frame_count), "fps": fps, "total_duration_s": service_data.get("total_duration_s"), "stl_duration_s": service_data.get("stl_duration_s"), "render_duration_s": service_data.get("render_duration_s"), "ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"), "stl_size_bytes": service_data.get("stl_size_bytes"), "output_size_bytes": service_data.get("output_size_bytes"), "log_lines": service_data.get("log_lines", []), } if template: render_log["template"] = template.blend_file_path pl.step_done("blender_turntable") except Exception as exc: success = False render_log = {"renderer": "blender", "type": "turntable", "error": str(exc)[:500]} pl.step_error("blender_turntable", str(exc), exc) logger.error("Turntable render failed for %s: %s", order_line_id, exc) else: # ── Still image path ──────────────────────────────────────── _render_path_label = "USD → Blender" if usd_render_path else "STEP → GLB → Blender" emit(order_line_id, f"Calling renderer ({_render_path_label}) {render_width or 'default'}x{render_height or 'default'}{' [transparent]' if transparent_bg else ''}{f' engine={render_engine}' if render_engine else ''}{f' samples={render_samples}' if render_samples else ''}{tmpl_info}") pl.step_start("blender_still", {"width": render_width, "height": render_height}) from app.services.step_processor import render_to_file success, render_log = render_to_file( step_path=cad_file.stored_path, output_path=output_path, part_colors=part_colors, width=render_width, height=render_height, transparent_bg=transparent_bg, engine=render_engine, samples=render_samples, template_path=template.blend_file_path if template else None, target_collection=template.target_collection if template else "Product", material_library_path=material_library if use_materials else None, material_map=material_map, part_names_ordered=part_names_ordered, lighting_only=bool(template.lighting_only) if template else False, shadow_catcher=bool(template.shadow_catcher_enabled) if template else False, cycles_device=line.output_type.cycles_device if line.output_type else None, rotation_x=rotation_x, rotation_y=rotation_y, rotation_z=rotation_z, focal_length_mm=focal_length_mm, sensor_width_mm=sensor_width_mm, material_override=override_mat, job_id=order_line_id, order_line_id=order_line_id, noise_threshold=noise_threshold, denoiser=denoiser, denoising_input_passes=denoising_input_passes, denoising_prefilter=denoising_prefilter, denoising_quality=denoising_quality, denoising_use_gpu=denoising_use_gpu, usd_path=usd_render_path, ) if success: pl.step_done("blender_still") else: pl.step_error("blender_still", "render_to_file returned False") new_status = "completed" if success else "failed" render_end = datetime.utcnow() elapsed = (render_end - render_start).total_seconds() update_values = dict( render_status=new_status, render_completed_at=render_end, render_log=render_log, ) if success: update_values["result_path"] = output_path session.execute( sql_update(OrderLine) .where(OrderLine.id == line.id) .values(**update_values) ) session.commit() if success: # Create MediaAsset so the render appears in the Media Browser try: import os as _os from app.domains.media.models import MediaAsset, MediaAssetType as MAT from app.config import settings as _cfg2 _ext = str(output_path).rsplit(".", 1)[-1].lower() if "." in str(output_path) else "bin" _mime = ( "video/mp4" if _ext in ("mp4", "webm") else "image/webp" if _ext == "webp" else "image/jpeg" if _ext in ("jpg", "jpeg") else "image/png" ) # Extension determines type — poster frames (.jpg/.png) from animations are still stills _at = MAT.turntable if _ext in ("mp4", "webm") else MAT.still _tenant_id = line.product.cad_file.tenant_id if (line.product and line.product.cad_file) else None # Normalize storage_key to relative path _raw_key = str(output_path) _upload_prefix = str(_cfg2.upload_dir).rstrip("/") + "/" _norm_key = _raw_key[len(_upload_prefix):] if _raw_key.startswith(_upload_prefix) else _raw_key _existing = session.execute( select(MediaAsset.id).where(MediaAsset.storage_key == _norm_key).limit(1) ).scalar_one_or_none() if not _existing: # Probe output file for metadata _file_size = None _width = None _height = None if _os.path.exists(output_path): try: _file_size = _os.path.getsize(output_path) except OSError: pass # Snapshot key render settings into render_config _render_config = None if isinstance(render_log, dict): _render_config = { k: render_log[k] for k in ( "renderer", "engine_used", "engine", "samples", "device_used", "compute_type", "total_duration_s", ) if k in render_log } _asset = MediaAsset( tenant_id=_tenant_id, order_line_id=line.id, product_id=line.product_id, asset_type=_at, storage_key=_norm_key, mime_type=_mime, file_size_bytes=_file_size, width=_width, height=_height, render_config=_render_config, ) session.add(_asset) session.commit() except Exception: logger.exception("Failed to create MediaAsset for order_line %s", order_line_id) if success: emit(order_line_id, f"Render completed in {elapsed:.1f}s", "success") else: emit(order_line_id, f"Render failed after {elapsed:.1f}s", "error") # Broadcast WebSocket event for live UI updates try: from app.core.websocket import publish_event_sync _tenant_id = str(line.product.cad_file.tenant_id) if ( line.product and line.product.cad_file and line.product.cad_file.tenant_id ) else None if _tenant_id: publish_event_sync(_tenant_id, { "type": "render_complete" if success else "render_failed", "order_line_id": order_line_id, "order_id": str(line.order_id), "status": new_status, }) except Exception: logger.debug("WebSocket publish skipped (non-fatal)") # Emit per-render activity event (channel=activity, not shown in bell dropdown) try: from app.models.order import Order as OrderModel order_row = session.execute( select(OrderModel.created_by, OrderModel.order_number) .where(OrderModel.id == line.order_id) ).one_or_none() if order_row: from app.services.notification_service import emit_notification_sync, CHANNEL_ACTIVITY details: dict = { "order_number": order_row[1], "product_name": product_name, "output_type": ot_name, } if not success and isinstance(render_log, dict): err = render_log.get("error") or render_log.get("stderr", "") if err: details["error"] = str(err)[:300] emit_notification_sync( actor_user_id=None, target_user_id=str(order_row[0]), action="render.completed" if success else "render.failed", entity_type="order", entity_id=str(line.order_id), details=details, channel=CHANNEL_ACTIVITY, ) except Exception: logger.exception("Failed to emit render activity event") # Check if all lines for this order are done → auto-advance order_id_str = str(line.order_id) engine.dispose() from app.services.order_status_service import check_order_completion check_order_completion(order_id_str) pl.step_done("render_order_line_task") except Exception as exc: logger.error(f"render_order_line_task failed for {order_line_id}: {exc}") # If retries exhausted, mark as failed so the line doesn't stay stuck if self.request.retries >= self.max_retries: logger.error(f"Max retries reached for {order_line_id}, marking as failed") try: from sqlalchemy import create_engine, update as sql_update2 from sqlalchemy.orm import Session as SyncSession from app.config import settings as app_settings from app.models.order_line import OrderLine as OL2 sync_url2 = app_settings.database_url.replace("+asyncpg", "") eng2 = create_engine(sync_url2) with SyncSession(eng2) as s2: set_tenant_context_sync(s2, _tenant_id) from datetime import datetime as dt2 s2.execute( sql_update2(OL2).where(OL2.id == order_line_id) .values( render_status="failed", render_completed_at=dt2.utcnow(), render_log={"error": str(exc)[:500]}, ) ) s2.commit() eng2.dispose() from app.services.order_status_service import check_order_completion # Try to get order_id from DB eng3 = create_engine(sync_url2) with SyncSession(eng3) as s3: set_tenant_context_sync(s3, _tenant_id) from sqlalchemy import select as sel row = s3.execute(sel(OL2.order_id).where(OL2.id == order_line_id)).scalar_one_or_none() if row: check_order_completion(str(row)) eng3.dispose() # Notify the order creator about the failure try: from sqlalchemy import select as sel2 from app.models.order import Order as OrderModel2 eng4 = create_engine(sync_url2) with SyncSession(eng4) as s4: set_tenant_context_sync(s4, _tenant_id) order_row2 = s4.execute( sel2(OrderModel2.created_by, OrderModel2.order_number) .join(OL2, OL2.order_id == OrderModel2.id) .where(OL2.id == order_line_id) ).one_or_none() eng4.dispose() if order_row2: from app.services.notification_service import emit_notification_sync, CHANNEL_ACTIVITY emit_notification_sync( actor_user_id=None, target_user_id=str(order_row2[0]), action="render.failed", entity_type="order", entity_id=None, details={ "order_number": order_row2[1], "product_name": "unknown", "output_type": "unknown", "error": str(exc)[:300], }, channel=CHANNEL_ACTIVITY, ) except Exception: logger.exception("Failed to emit render failure activity event") except Exception: logger.exception(f"Failed to mark {order_line_id} as failed in DB") raise raise self.retry(exc=exc, countdown=60)