605 lines
32 KiB
Python
605 lines
32 KiB
Python
"""Order-line render tasks.
|
|
|
|
Covers:
|
|
- dispatch_order_line_render — thin router that queues render_order_line_task
|
|
- render_order_line_task — full still/turntable render pipeline for one order line
|
|
"""
|
|
import logging
|
|
|
|
from app.tasks.celery_app import celery_app
|
|
from app.core.task_logs import log_task_event
|
|
from app.core.pipeline_logger import PipelineLogger
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@celery_app.task(name="app.tasks.step_tasks.dispatch_order_line_render", queue="step_processing")
|
|
def dispatch_order_line_render(order_line_id: str):
|
|
"""Route an order-line render to render_order_line_task."""
|
|
# Pre-check: skip if line is already cancelled or order is rejected
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session
|
|
from app.config import settings as app_settings
|
|
from app.models.order_line import OrderLine
|
|
from app.domains.orders.models import Order, OrderStatus
|
|
|
|
sync_url = app_settings.database_url.replace("+asyncpg", "")
|
|
engine = create_engine(sync_url)
|
|
with Session(engine) as session:
|
|
line = session.execute(
|
|
select(OrderLine).where(OrderLine.id == order_line_id)
|
|
).scalar_one_or_none()
|
|
if line and line.render_status == "cancelled":
|
|
logger.info(f"OrderLine {order_line_id} cancelled — not dispatching")
|
|
return
|
|
if line:
|
|
order = session.execute(
|
|
select(Order).where(Order.id == line.order_id)
|
|
).scalar_one_or_none()
|
|
if order and order.status in (OrderStatus.rejected, OrderStatus.completed):
|
|
logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching")
|
|
return
|
|
|
|
# All renders go to asset_pipeline (single-GPU default).
|
|
# For multi-GPU setups: enable render-worker-light in docker-compose
|
|
# and change target_queue logic below to route small stills to
|
|
# asset_pipeline_light for concurrent rendering.
|
|
pass
|
|
|
|
target_queue = "asset_pipeline"
|
|
logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}")
|
|
render_order_line_task.apply_async(args=[order_line_id], queue=target_queue)
|
|
|
|
|
|
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3)
|
|
def render_order_line_task(self, order_line_id: str):
|
|
"""Render a specific output type for an order line.
|
|
|
|
Loads OrderLine → Product → CadFile → OutputType.render_settings.
|
|
Merges with system render settings. Stores result at order_line.result_path.
|
|
"""
|
|
pl = PipelineLogger(task_id=self.request.id, order_line_id=order_line_id)
|
|
pl.step_start("render_order_line_task", {"order_line_id": order_line_id})
|
|
logger.info(f"Rendering order line: {order_line_id}")
|
|
|
|
# Resolve and log tenant context at task start (required for RLS)
|
|
from app.core.tenant_context import resolve_tenant_id_for_order_line, set_tenant_context_sync
|
|
_tenant_id = resolve_tenant_id_for_order_line(order_line_id)
|
|
|
|
from app.services.render_log import emit
|
|
|
|
emit(order_line_id, "Celery render task started")
|
|
try:
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session
|
|
from app.config import settings as app_settings
|
|
from app.domains.rendering.workflow_runtime_services import (
|
|
persist_order_line_output,
|
|
prepare_order_line_render_context,
|
|
resolve_order_line_template_context,
|
|
resolve_render_position_context,
|
|
)
|
|
|
|
# Use sync session for Celery (no async event loop)
|
|
sync_url = app_settings.database_url.replace("+asyncpg", "")
|
|
engine = create_engine(sync_url)
|
|
|
|
with Session(engine) as session:
|
|
set_tenant_context_sync(session, _tenant_id)
|
|
from pathlib import Path as _Path
|
|
|
|
setup = prepare_order_line_render_context(
|
|
session,
|
|
order_line_id,
|
|
emit=emit,
|
|
)
|
|
if not setup.is_ready:
|
|
return
|
|
|
|
line = setup.order_line
|
|
cad_file = setup.cad_file
|
|
materials_source = setup.materials_source
|
|
usd_render_path = setup.usd_render_path
|
|
glb_reuse_path = setup.glb_reuse_path
|
|
part_colors = setup.part_colors
|
|
render_start = setup.render_start
|
|
|
|
template_context = resolve_order_line_template_context(
|
|
session,
|
|
setup,
|
|
emit=emit,
|
|
)
|
|
template = template_context.template
|
|
material_library = template_context.material_library
|
|
material_map = template_context.material_map
|
|
use_materials = template_context.use_materials
|
|
override_mat = template_context.override_material
|
|
|
|
cad_name = cad_file.original_name if cad_file else "?"
|
|
position_context = resolve_render_position_context(session, line, emit=emit)
|
|
rotation_x = position_context.rotation_x
|
|
rotation_y = position_context.rotation_y
|
|
rotation_z = position_context.rotation_z
|
|
focal_length_mm = position_context.focal_length_mm
|
|
sensor_width_mm = position_context.sensor_width_mm
|
|
|
|
emit(order_line_id, f"Starting render for {cad_name} ({len(part_colors)} coloured parts)")
|
|
|
|
# Determine if this is an animation output type
|
|
is_animation = bool(line.output_type and getattr(line.output_type, 'is_animation', False))
|
|
|
|
# Detect cinematic render type (render_settings.cinematic flag)
|
|
is_cinematic = bool(
|
|
line.output_type and
|
|
line.output_type.render_settings and
|
|
line.output_type.render_settings.get("cinematic")
|
|
)
|
|
|
|
# Determine output format/extension
|
|
out_ext = "jpg"
|
|
if line.output_type and line.output_type.output_format:
|
|
fmt = line.output_type.output_format.lower()
|
|
if fmt == "mp4":
|
|
out_ext = "mp4"
|
|
elif fmt == "webp":
|
|
out_ext = "webp"
|
|
elif fmt in ("png", "jpg", "jpeg"):
|
|
out_ext = "png" if fmt == "png" else "jpg"
|
|
|
|
# Build meaningful output filename
|
|
import re
|
|
def _sanitize(s: str) -> str:
|
|
return re.sub(r'[^\w\-.]', '_', s.strip())[:100]
|
|
|
|
product_name = line.product.name or line.product.pim_id or "product"
|
|
ot_name = line.output_type.name if line.output_type else "render"
|
|
filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}"
|
|
|
|
# Render to per-line output directory
|
|
from pathlib import Path as _Path
|
|
render_dir = _Path(app_settings.upload_dir) / "renders" / order_line_id
|
|
render_dir.mkdir(parents=True, exist_ok=True)
|
|
output_path = str(render_dir / filename)
|
|
|
|
# Extract per-output-type render settings
|
|
render_width = None
|
|
render_height = None
|
|
render_engine = None
|
|
render_samples = None
|
|
noise_threshold = ""
|
|
denoiser = ""
|
|
denoising_input_passes = ""
|
|
denoising_prefilter = ""
|
|
denoising_quality = ""
|
|
denoising_use_gpu = ""
|
|
frame_count = 24
|
|
fps = 25
|
|
bg_color = ""
|
|
turntable_axis = "world_z"
|
|
if line.output_type and line.output_type.render_settings:
|
|
rs = line.output_type.render_settings
|
|
if rs.get("width"):
|
|
render_width = int(rs["width"])
|
|
if rs.get("height"):
|
|
render_height = int(rs["height"])
|
|
if rs.get("engine"):
|
|
render_engine = rs["engine"]
|
|
if rs.get("samples"):
|
|
render_samples = int(rs["samples"])
|
|
if rs.get("frame_count"):
|
|
frame_count = int(rs["frame_count"])
|
|
if rs.get("fps"):
|
|
fps = int(rs["fps"])
|
|
bg_color = rs.get("bg_color", "")
|
|
turntable_axis = rs.get("turntable_axis", "world_z")
|
|
noise_threshold = str(rs.get("noise_threshold", ""))
|
|
denoiser = str(rs.get("denoiser", ""))
|
|
denoising_input_passes = str(rs.get("denoising_input_passes", ""))
|
|
denoising_prefilter = str(rs.get("denoising_prefilter", ""))
|
|
denoising_quality = str(rs.get("denoising_quality", ""))
|
|
denoising_use_gpu = str(rs.get("denoising_use_gpu", ""))
|
|
|
|
# Auto-scale samples for lower resolutions (thumbnails/previews).
|
|
# Only applies when the output type provides both samples and dimensions.
|
|
if render_samples and render_width and render_height:
|
|
max_dim = max(render_width, render_height)
|
|
if max_dim <= 1024:
|
|
scaled = max(32, int(render_samples * max_dim / 2048))
|
|
if scaled < render_samples:
|
|
emit(order_line_id, f"Auto-scaled samples {render_samples} \u2192 {scaled} for {render_width}x{render_height}")
|
|
render_samples = scaled
|
|
|
|
transparent_bg = bool(line.output_type and line.output_type.transparent_bg)
|
|
cycles_device_val = (line.output_type.cycles_device or "auto") if line.output_type else "auto"
|
|
|
|
# Apply per-line render overrides (format, resolution, samples, etc.)
|
|
_render_overrides = getattr(line, 'render_overrides', None)
|
|
if _render_overrides and isinstance(_render_overrides, dict):
|
|
if 'width' in _render_overrides:
|
|
render_width = int(_render_overrides['width'])
|
|
if 'height' in _render_overrides:
|
|
render_height = int(_render_overrides['height'])
|
|
if 'samples' in _render_overrides:
|
|
render_samples = int(_render_overrides['samples'])
|
|
if 'engine' in _render_overrides:
|
|
render_engine = _render_overrides['engine']
|
|
if 'frame_count' in _render_overrides:
|
|
frame_count = int(_render_overrides['frame_count'])
|
|
if 'fps' in _render_overrides:
|
|
fps = int(_render_overrides['fps'])
|
|
if 'bg_color' in _render_overrides:
|
|
bg_color = _render_overrides['bg_color']
|
|
if 'turntable_axis' in _render_overrides:
|
|
turntable_axis = _render_overrides['turntable_axis']
|
|
if 'noise_threshold' in _render_overrides:
|
|
noise_threshold = str(_render_overrides['noise_threshold'])
|
|
if 'denoiser' in _render_overrides:
|
|
denoiser = str(_render_overrides['denoiser'])
|
|
if 'denoising_input_passes' in _render_overrides:
|
|
denoising_input_passes = str(_render_overrides['denoising_input_passes'])
|
|
if 'denoising_prefilter' in _render_overrides:
|
|
denoising_prefilter = str(_render_overrides['denoising_prefilter'])
|
|
if 'denoising_quality' in _render_overrides:
|
|
denoising_quality = str(_render_overrides['denoising_quality'])
|
|
if 'denoising_use_gpu' in _render_overrides:
|
|
denoising_use_gpu = str(_render_overrides['denoising_use_gpu'])
|
|
if 'transparent_bg' in _render_overrides:
|
|
transparent_bg = bool(_render_overrides['transparent_bg'])
|
|
if 'cycles_device' in _render_overrides:
|
|
cycles_device_val = _render_overrides['cycles_device']
|
|
emit(order_line_id, f"Render overrides active: {_render_overrides}")
|
|
|
|
# Apply output_format override (affects out_ext and filename)
|
|
if 'output_format' in _render_overrides:
|
|
fmt_override = _render_overrides['output_format'].lower()
|
|
if fmt_override == "mp4":
|
|
out_ext = "mp4"
|
|
elif fmt_override == "webp":
|
|
out_ext = "webp"
|
|
elif fmt_override in ("png", "jpg", "jpeg"):
|
|
out_ext = "png" if fmt_override == "png" else "jpg"
|
|
# Rebuild filename with new extension
|
|
filename = f"{_sanitize(product_name)}_{_sanitize(ot_name)}.{out_ext}"
|
|
output_path = str(render_dir / filename)
|
|
|
|
# Build ordered part names list for index-based Blender matching
|
|
part_names_ordered = None
|
|
if cad_file and cad_file.parsed_objects:
|
|
part_names_ordered = cad_file.parsed_objects.get("objects", []) or None
|
|
|
|
tmpl_info = f" template={template.name}" if template else ""
|
|
|
|
if is_cinematic:
|
|
# ── Cinematic highlight animation path ──────────────────────
|
|
# Use frame_count/fps from output_type.render_settings (already extracted above)
|
|
_cine_fps = fps # extracted from render_settings, default 25
|
|
_cine_frames = frame_count # extracted from render_settings, default 24
|
|
emit(order_line_id, f"Starting cinematic render: {_cine_frames} frames @ {_cine_fps}fps, {render_width or 1920}x{render_height or 1080}{tmpl_info}")
|
|
pl.step_start("blender_cinematic", {"frame_count": _cine_frames, "fps": _cine_fps})
|
|
from app.services.render_blender import is_blender_available, render_cinematic_to_file
|
|
if not is_blender_available():
|
|
raise RuntimeError("Blender not available on this worker")
|
|
|
|
from app.services.step_processor import _get_all_settings
|
|
_sys = _get_all_settings()
|
|
try:
|
|
service_data = render_cinematic_to_file(
|
|
step_path=_Path(cad_file.stored_path),
|
|
output_path=_Path(output_path),
|
|
width=render_width or 1920,
|
|
height=render_height or 1080,
|
|
engine=render_engine or _sys.get("blender_engine", "cycles"),
|
|
samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)),
|
|
smooth_angle=int(_sys.get("blender_smooth_angle", 30)),
|
|
cycles_device=cycles_device_val,
|
|
transparent_bg=transparent_bg,
|
|
part_colors=part_colors or None,
|
|
template_path=template.blend_file_path if template else None,
|
|
target_collection=template.target_collection if template else "Product",
|
|
material_library_path=material_library if use_materials else None,
|
|
material_map=material_map,
|
|
part_names_ordered=part_names_ordered,
|
|
lighting_only=bool(template.lighting_only) if template else False,
|
|
shadow_catcher=bool(template.shadow_catcher_enabled) if template else False,
|
|
rotation_x=rotation_x,
|
|
rotation_y=rotation_y,
|
|
rotation_z=rotation_z,
|
|
usd_path=usd_render_path,
|
|
focal_length_mm=focal_length_mm,
|
|
sensor_width_mm=sensor_width_mm,
|
|
material_override=override_mat,
|
|
log_callback=lambda line: emit(order_line_id, line),
|
|
)
|
|
success = True
|
|
render_log = {
|
|
"renderer": "blender",
|
|
"type": "cinematic",
|
|
"format": "mp4",
|
|
"engine": render_engine or _sys.get("blender_engine", "cycles"),
|
|
"engine_used": service_data.get("engine_used", "cycles"),
|
|
"samples": render_samples,
|
|
"cycles_device": cycles_device_val,
|
|
"width": render_width or 1920,
|
|
"height": render_height or 1080,
|
|
"frame_count": service_data.get("frame_count", _cine_frames),
|
|
"fps": _cine_fps,
|
|
"total_duration_s": service_data.get("total_duration_s"),
|
|
"stl_duration_s": service_data.get("stl_duration_s"),
|
|
"render_duration_s": service_data.get("render_duration_s"),
|
|
"ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"),
|
|
"stl_size_bytes": service_data.get("stl_size_bytes"),
|
|
"output_size_bytes": service_data.get("output_size_bytes"),
|
|
"log_lines": service_data.get("log_lines", []),
|
|
}
|
|
if template:
|
|
render_log["template"] = template.blend_file_path
|
|
pl.step_done("blender_cinematic")
|
|
except Exception as exc:
|
|
success = False
|
|
render_log = {"renderer": "blender", "type": "cinematic", "error": str(exc)[:500]}
|
|
pl.step_error("blender_cinematic", str(exc), exc)
|
|
logger.error("Cinematic render failed for %s: %s", order_line_id, exc)
|
|
elif is_animation:
|
|
# ── Turntable animation path ────────────────────────────────
|
|
emit(order_line_id, f"Starting turntable render: {frame_count} frames @ {fps}fps, {render_width or 1920}x{render_height or 1920}{tmpl_info}")
|
|
pl.step_start("blender_turntable", {"frame_count": frame_count, "fps": fps})
|
|
from app.services.render_blender import is_blender_available, render_turntable_to_file
|
|
if not is_blender_available():
|
|
raise RuntimeError("Blender not available on this worker")
|
|
|
|
from app.services.step_processor import _get_all_settings
|
|
_sys = _get_all_settings()
|
|
try:
|
|
service_data = render_turntable_to_file(
|
|
step_path=_Path(cad_file.stored_path),
|
|
output_path=_Path(output_path),
|
|
frame_count=frame_count,
|
|
fps=fps,
|
|
width=render_width or 1920,
|
|
height=render_height or 1920,
|
|
engine=render_engine or _sys.get("blender_engine", "cycles"),
|
|
samples=render_samples or int(_sys.get(f"blender_{render_engine or _sys.get('blender_engine','cycles')}_samples", 128)),
|
|
smooth_angle=int(_sys.get("blender_smooth_angle", 30)),
|
|
cycles_device=cycles_device_val,
|
|
transparent_bg=transparent_bg,
|
|
bg_color=bg_color,
|
|
turntable_axis=turntable_axis,
|
|
part_colors=part_colors or None,
|
|
template_path=template.blend_file_path if template else None,
|
|
target_collection=template.target_collection if template else "Product",
|
|
material_library_path=material_library if use_materials else None,
|
|
material_map=material_map,
|
|
part_names_ordered=part_names_ordered,
|
|
lighting_only=bool(template.lighting_only) if template else False,
|
|
shadow_catcher=bool(template.shadow_catcher_enabled) if template else False,
|
|
rotation_x=rotation_x,
|
|
rotation_y=rotation_y,
|
|
rotation_z=rotation_z,
|
|
camera_orbit=bool(template.camera_orbit) if template else True,
|
|
usd_path=usd_render_path,
|
|
focal_length_mm=focal_length_mm,
|
|
sensor_width_mm=sensor_width_mm,
|
|
material_override=override_mat,
|
|
)
|
|
success = True
|
|
render_log = {
|
|
"renderer": "blender",
|
|
"type": "turntable",
|
|
"format": "mp4",
|
|
"engine": render_engine or _sys.get("blender_engine", "cycles"),
|
|
"engine_used": service_data.get("engine_used", "cycles"),
|
|
"samples": render_samples,
|
|
"cycles_device": cycles_device_val,
|
|
"width": render_width or 1920,
|
|
"height": render_height or 1920,
|
|
"frame_count": service_data.get("frame_count", frame_count),
|
|
"fps": fps,
|
|
"total_duration_s": service_data.get("total_duration_s"),
|
|
"stl_duration_s": service_data.get("stl_duration_s"),
|
|
"render_duration_s": service_data.get("render_duration_s"),
|
|
"ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"),
|
|
"stl_size_bytes": service_data.get("stl_size_bytes"),
|
|
"output_size_bytes": service_data.get("output_size_bytes"),
|
|
"log_lines": service_data.get("log_lines", []),
|
|
}
|
|
if template:
|
|
render_log["template"] = template.blend_file_path
|
|
pl.step_done("blender_turntable")
|
|
except Exception as exc:
|
|
success = False
|
|
render_log = {"renderer": "blender", "type": "turntable", "error": str(exc)[:500]}
|
|
pl.step_error("blender_turntable", str(exc), exc)
|
|
logger.error("Turntable render failed for %s: %s", order_line_id, exc)
|
|
else:
|
|
# ── Still image path ────────────────────────────────────────
|
|
_render_path_label = "USD → Blender" if usd_render_path else "STEP → GLB → Blender"
|
|
emit(order_line_id, f"Calling renderer ({_render_path_label}) {render_width or 'default'}x{render_height or 'default'}{' [transparent]' if transparent_bg else ''}{f' engine={render_engine}' if render_engine else ''}{f' samples={render_samples}' if render_samples else ''}{tmpl_info}")
|
|
pl.step_start("blender_still", {"width": render_width, "height": render_height})
|
|
from app.services.step_processor import render_to_file
|
|
|
|
success, render_log = render_to_file(
|
|
step_path=cad_file.stored_path,
|
|
output_path=output_path,
|
|
part_colors=part_colors,
|
|
width=render_width,
|
|
height=render_height,
|
|
transparent_bg=transparent_bg,
|
|
engine=render_engine,
|
|
samples=render_samples,
|
|
template_path=template.blend_file_path if template else None,
|
|
target_collection=template.target_collection if template else "Product",
|
|
material_library_path=material_library if use_materials else None,
|
|
material_map=material_map,
|
|
part_names_ordered=part_names_ordered,
|
|
lighting_only=bool(template.lighting_only) if template else False,
|
|
shadow_catcher=bool(template.shadow_catcher_enabled) if template else False,
|
|
cycles_device=line.output_type.cycles_device if line.output_type else None,
|
|
rotation_x=rotation_x,
|
|
rotation_y=rotation_y,
|
|
rotation_z=rotation_z,
|
|
focal_length_mm=focal_length_mm,
|
|
sensor_width_mm=sensor_width_mm,
|
|
material_override=override_mat,
|
|
job_id=order_line_id,
|
|
order_line_id=order_line_id,
|
|
noise_threshold=noise_threshold,
|
|
denoiser=denoiser,
|
|
denoising_input_passes=denoising_input_passes,
|
|
denoising_prefilter=denoising_prefilter,
|
|
denoising_quality=denoising_quality,
|
|
denoising_use_gpu=denoising_use_gpu,
|
|
usd_path=usd_render_path,
|
|
)
|
|
if success:
|
|
pl.step_done("blender_still")
|
|
else:
|
|
pl.step_error("blender_still", "render_to_file returned False")
|
|
|
|
new_status = "completed" if success else "failed"
|
|
render_end = datetime.utcnow()
|
|
elapsed = (render_end - render_start).total_seconds()
|
|
try:
|
|
persist_order_line_output(
|
|
session,
|
|
line,
|
|
success=success,
|
|
output_path=output_path,
|
|
render_log=render_log if isinstance(render_log, dict) else None,
|
|
render_completed_at=render_end,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to persist render output for order_line %s", order_line_id)
|
|
raise
|
|
|
|
if success:
|
|
emit(order_line_id, f"Render completed in {elapsed:.1f}s", "success")
|
|
else:
|
|
emit(order_line_id, f"Render failed after {elapsed:.1f}s", "error")
|
|
|
|
# Broadcast WebSocket event for live UI updates
|
|
try:
|
|
from app.core.websocket import publish_event_sync
|
|
_tenant_id = str(line.product.cad_file.tenant_id) if (
|
|
line.product and line.product.cad_file and line.product.cad_file.tenant_id
|
|
) else None
|
|
if _tenant_id:
|
|
publish_event_sync(_tenant_id, {
|
|
"type": "render_complete" if success else "render_failed",
|
|
"order_line_id": order_line_id,
|
|
"order_id": str(line.order_id),
|
|
"status": new_status,
|
|
})
|
|
except Exception:
|
|
logger.debug("WebSocket publish skipped (non-fatal)")
|
|
|
|
# Emit per-render activity event (channel=activity, not shown in bell dropdown)
|
|
try:
|
|
from app.models.order import Order as OrderModel
|
|
order_row = session.execute(
|
|
select(OrderModel.created_by, OrderModel.order_number)
|
|
.where(OrderModel.id == line.order_id)
|
|
).one_or_none()
|
|
if order_row:
|
|
from app.services.notification_service import emit_notification_sync, CHANNEL_ACTIVITY
|
|
details: dict = {
|
|
"order_number": order_row[1],
|
|
"product_name": product_name,
|
|
"output_type": ot_name,
|
|
}
|
|
if not success and isinstance(render_log, dict):
|
|
err = render_log.get("error") or render_log.get("stderr", "")
|
|
if err:
|
|
details["error"] = str(err)[:300]
|
|
emit_notification_sync(
|
|
actor_user_id=None,
|
|
target_user_id=str(order_row[0]),
|
|
action="render.completed" if success else "render.failed",
|
|
entity_type="order",
|
|
entity_id=str(line.order_id),
|
|
details=details,
|
|
channel=CHANNEL_ACTIVITY,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to emit render activity event")
|
|
|
|
# Check if all lines for this order are done → auto-advance
|
|
order_id_str = str(line.order_id)
|
|
|
|
engine.dispose()
|
|
|
|
from app.services.order_status_service import check_order_completion
|
|
check_order_completion(order_id_str)
|
|
|
|
pl.step_done("render_order_line_task")
|
|
|
|
except Exception as exc:
|
|
logger.error(f"render_order_line_task failed for {order_line_id}: {exc}")
|
|
# If retries exhausted, mark as failed so the line doesn't stay stuck
|
|
if self.request.retries >= self.max_retries:
|
|
logger.error(f"Max retries reached for {order_line_id}, marking as failed")
|
|
try:
|
|
from sqlalchemy import create_engine, update as sql_update2
|
|
from sqlalchemy.orm import Session as SyncSession
|
|
from app.config import settings as app_settings
|
|
from app.models.order_line import OrderLine as OL2
|
|
sync_url2 = app_settings.database_url.replace("+asyncpg", "")
|
|
eng2 = create_engine(sync_url2)
|
|
with SyncSession(eng2) as s2:
|
|
set_tenant_context_sync(s2, _tenant_id)
|
|
from datetime import datetime as dt2
|
|
s2.execute(
|
|
sql_update2(OL2).where(OL2.id == order_line_id)
|
|
.values(
|
|
render_status="failed",
|
|
render_completed_at=dt2.utcnow(),
|
|
render_log={"error": str(exc)[:500]},
|
|
)
|
|
)
|
|
s2.commit()
|
|
eng2.dispose()
|
|
from app.services.order_status_service import check_order_completion
|
|
# Try to get order_id from DB
|
|
eng3 = create_engine(sync_url2)
|
|
with SyncSession(eng3) as s3:
|
|
set_tenant_context_sync(s3, _tenant_id)
|
|
from sqlalchemy import select as sel
|
|
row = s3.execute(sel(OL2.order_id).where(OL2.id == order_line_id)).scalar_one_or_none()
|
|
if row:
|
|
check_order_completion(str(row))
|
|
eng3.dispose()
|
|
# Notify the order creator about the failure
|
|
try:
|
|
from sqlalchemy import select as sel2
|
|
from app.models.order import Order as OrderModel2
|
|
eng4 = create_engine(sync_url2)
|
|
with SyncSession(eng4) as s4:
|
|
set_tenant_context_sync(s4, _tenant_id)
|
|
order_row2 = s4.execute(
|
|
sel2(OrderModel2.created_by, OrderModel2.order_number)
|
|
.join(OL2, OL2.order_id == OrderModel2.id)
|
|
.where(OL2.id == order_line_id)
|
|
).one_or_none()
|
|
eng4.dispose()
|
|
if order_row2:
|
|
from app.services.notification_service import emit_notification_sync, CHANNEL_ACTIVITY
|
|
emit_notification_sync(
|
|
actor_user_id=None,
|
|
target_user_id=str(order_row2[0]),
|
|
action="render.failed",
|
|
entity_type="order",
|
|
entity_id=None,
|
|
details={
|
|
"order_number": order_row2[1],
|
|
"product_name": "unknown",
|
|
"output_type": "unknown",
|
|
"error": str(exc)[:300],
|
|
},
|
|
channel=CHANNEL_ACTIVITY,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to emit render failure activity event")
|
|
except Exception:
|
|
logger.exception(f"Failed to mark {order_line_id} as failed in DB")
|
|
raise
|
|
raise self.retry(exc=exc, countdown=60)
|