428 lines
22 KiB
Python
428 lines
22 KiB
Python
"""Order-line render tasks.
|
|
|
|
Covers:
|
|
- dispatch_order_line_render — thin router that queues render_order_line_task
|
|
- render_order_line_task — full still/turntable render pipeline for one order line
|
|
"""
|
|
import logging
|
|
from datetime import datetime
|
|
|
|
from app.tasks.celery_app import celery_app
|
|
from app.core.render_paths import ensure_group_writable_dir
|
|
from app.core.task_logs import log_task_event
|
|
from app.core.pipeline_logger import PipelineLogger
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@celery_app.task(name="app.tasks.step_tasks.dispatch_order_line_render", queue="step_processing")
|
|
def dispatch_order_line_render(order_line_id: str):
|
|
"""Route an order-line render to render_order_line_task."""
|
|
# Pre-check: skip if line is already cancelled or order is rejected
|
|
from sqlalchemy import create_engine, select
|
|
from sqlalchemy.orm import Session
|
|
from app.config import settings as app_settings
|
|
from app.models.order_line import OrderLine
|
|
from app.domains.orders.models import Order, OrderStatus
|
|
|
|
sync_url = app_settings.database_url.replace("+asyncpg", "")
|
|
engine = create_engine(sync_url)
|
|
with Session(engine) as session:
|
|
line = session.execute(
|
|
select(OrderLine).where(OrderLine.id == order_line_id)
|
|
).scalar_one_or_none()
|
|
if line and line.render_status == "cancelled":
|
|
logger.info(f"OrderLine {order_line_id} cancelled — not dispatching")
|
|
return
|
|
if line:
|
|
order = session.execute(
|
|
select(Order).where(Order.id == line.order_id)
|
|
).scalar_one_or_none()
|
|
if order and order.status in (OrderStatus.rejected, OrderStatus.completed):
|
|
logger.info(f"OrderLine {order_line_id}: order {order.status.value} — not dispatching")
|
|
return
|
|
|
|
# All renders go to asset_pipeline (single-GPU default).
|
|
# For multi-GPU setups: enable render-worker-light in docker-compose
|
|
# and change target_queue logic below to route small stills to
|
|
# asset_pipeline_light for concurrent rendering.
|
|
pass
|
|
|
|
target_queue = "asset_pipeline"
|
|
logger.info(f"Dispatching render for order line: {order_line_id} -> queue={target_queue}")
|
|
render_order_line_task.apply_async(args=[order_line_id], queue=target_queue)
|
|
|
|
|
|
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3)
|
|
def render_order_line_task(self, order_line_id: str):
|
|
"""Render a specific output type for an order line.
|
|
|
|
Loads OrderLine → Product → CadFile → OutputType.render_settings.
|
|
Merges with system render settings. Stores result at order_line.result_path.
|
|
"""
|
|
pl = PipelineLogger(task_id=self.request.id, order_line_id=order_line_id)
|
|
pl.step_start("render_order_line_task", {"order_line_id": order_line_id})
|
|
logger.info(f"Rendering order line: {order_line_id}")
|
|
|
|
# Resolve and log tenant context at task start (required for RLS)
|
|
from app.core.tenant_context import resolve_tenant_id_for_order_line, set_tenant_context_sync
|
|
_tenant_id = resolve_tenant_id_for_order_line(order_line_id)
|
|
|
|
from app.services.render_log import emit
|
|
|
|
emit(order_line_id, "Celery render task started")
|
|
try:
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session
|
|
from app.config import settings as app_settings
|
|
from app.domains.rendering.workflow_runtime_services import (
|
|
build_order_line_render_invocation,
|
|
emit_order_line_render_notifications,
|
|
persist_order_line_output,
|
|
prepare_order_line_render_context,
|
|
resolve_order_line_template_context,
|
|
resolve_render_position_context,
|
|
)
|
|
|
|
# Use sync session for Celery (no async event loop)
|
|
sync_url = app_settings.database_url.replace("+asyncpg", "")
|
|
engine = create_engine(sync_url)
|
|
|
|
with Session(engine) as session:
|
|
set_tenant_context_sync(session, _tenant_id)
|
|
from pathlib import Path as _Path
|
|
|
|
setup = prepare_order_line_render_context(
|
|
session,
|
|
order_line_id,
|
|
emit=emit,
|
|
)
|
|
if not setup.is_ready:
|
|
return
|
|
|
|
line = setup.order_line
|
|
cad_file = setup.cad_file
|
|
part_colors = setup.part_colors
|
|
render_start = setup.render_start
|
|
|
|
template_context = resolve_order_line_template_context(
|
|
session,
|
|
setup,
|
|
emit=emit,
|
|
)
|
|
template = template_context.template
|
|
material_map = template_context.material_map
|
|
|
|
cad_name = cad_file.original_name if cad_file else "?"
|
|
position_context = resolve_render_position_context(session, line, emit=emit)
|
|
rotation_x = position_context.rotation_x
|
|
rotation_y = position_context.rotation_y
|
|
rotation_z = position_context.rotation_z
|
|
focal_length_mm = position_context.focal_length_mm
|
|
sensor_width_mm = position_context.sensor_width_mm
|
|
|
|
render_invocation = build_order_line_render_invocation(
|
|
setup,
|
|
template_context=template_context,
|
|
position_context=position_context,
|
|
)
|
|
emit(order_line_id, f"Starting render for {cad_name} ({len(part_colors)} coloured parts)")
|
|
if getattr(line, "render_overrides", None):
|
|
emit(order_line_id, f"Render overrides active: {line.render_overrides}")
|
|
if (
|
|
line.output_type
|
|
and line.output_type.render_settings
|
|
and render_invocation.samples is not None
|
|
and line.output_type.render_settings.get("samples")
|
|
and render_invocation.width is not None
|
|
and render_invocation.height is not None
|
|
):
|
|
base_samples = int(line.output_type.render_settings["samples"])
|
|
if render_invocation.samples < base_samples:
|
|
emit(
|
|
order_line_id,
|
|
f"Auto-scaled samples {base_samples} -> {render_invocation.samples} "
|
|
f"for {render_invocation.width}x{render_invocation.height}",
|
|
)
|
|
|
|
is_animation = render_invocation.is_animation
|
|
is_cinematic = render_invocation.is_cinematic
|
|
product_name = render_invocation.product_name
|
|
ot_name = render_invocation.output_type_name
|
|
output_path = render_invocation.output_path
|
|
ensure_group_writable_dir(_Path(output_path).parent)
|
|
render_width = render_invocation.width
|
|
render_height = render_invocation.height
|
|
render_engine = render_invocation.engine
|
|
render_samples = render_invocation.samples
|
|
frame_count = render_invocation.frame_count
|
|
fps = render_invocation.fps
|
|
bg_color = render_invocation.bg_color
|
|
turntable_axis = render_invocation.turntable_axis
|
|
noise_threshold = render_invocation.noise_threshold
|
|
denoiser = render_invocation.denoiser
|
|
denoising_input_passes = render_invocation.denoising_input_passes
|
|
denoising_prefilter = render_invocation.denoising_prefilter
|
|
denoising_quality = render_invocation.denoising_quality
|
|
denoising_use_gpu = render_invocation.denoising_use_gpu
|
|
transparent_bg = render_invocation.transparent_bg
|
|
cycles_device_val = render_invocation.cycles_device
|
|
part_names_ordered = render_invocation.part_names_ordered
|
|
material_override = render_invocation.material_override
|
|
target_collection = render_invocation.target_collection
|
|
template_path = render_invocation.template_path
|
|
material_library_path = render_invocation.material_library_path
|
|
camera_orbit = render_invocation.camera_orbit
|
|
lighting_only = render_invocation.lighting_only
|
|
shadow_catcher = render_invocation.shadow_catcher
|
|
usd_path = render_invocation.usd_path
|
|
step_path = _Path(cad_file.stored_path)
|
|
tmpl_info = f" template={template.name}" if template else ""
|
|
|
|
if is_cinematic:
|
|
# ── Cinematic highlight animation path ──────────────────────
|
|
# Use frame_count/fps from output_type.render_settings (already extracted above)
|
|
_cine_fps = fps # extracted from render_settings, default 25
|
|
_cine_frames = frame_count # extracted from render_settings, default 24
|
|
emit(order_line_id, f"Starting cinematic render: {_cine_frames} frames @ {_cine_fps}fps, {render_width or 1920}x{render_height or 1080}{tmpl_info}")
|
|
pl.step_start("blender_cinematic", {"frame_count": _cine_frames, "fps": _cine_fps})
|
|
from app.services.render_blender import is_blender_available, render_cinematic_to_file
|
|
if not is_blender_available():
|
|
raise RuntimeError("Blender not available on this worker")
|
|
|
|
from app.services.step_processor import _get_all_settings
|
|
_sys = _get_all_settings()
|
|
try:
|
|
cinematic_kwargs = render_invocation.as_cinematic_renderer_kwargs(
|
|
step_path=step_path,
|
|
output_path=_Path(output_path),
|
|
default_width=1920,
|
|
default_height=1080,
|
|
default_engine=_sys.get("blender_engine", "cycles"),
|
|
default_samples=int(
|
|
_sys.get(
|
|
f"blender_{render_engine or _sys.get('blender_engine', 'cycles')}_samples",
|
|
128,
|
|
)
|
|
),
|
|
smooth_angle=int(_sys.get("blender_smooth_angle", 30)),
|
|
log_callback=lambda line: emit(order_line_id, line),
|
|
)
|
|
service_data = render_cinematic_to_file(**cinematic_kwargs)
|
|
success = True
|
|
render_log = {
|
|
"renderer": "blender",
|
|
"type": "cinematic",
|
|
"format": "mp4",
|
|
"engine": render_engine or _sys.get("blender_engine", "cycles"),
|
|
"engine_used": service_data.get("engine_used", "cycles"),
|
|
"samples": render_samples,
|
|
"cycles_device": cycles_device_val,
|
|
"width": render_width or 1920,
|
|
"height": render_height or 1080,
|
|
"frame_count": service_data.get("frame_count", _cine_frames),
|
|
"fps": _cine_fps,
|
|
"total_duration_s": service_data.get("total_duration_s"),
|
|
"stl_duration_s": service_data.get("stl_duration_s"),
|
|
"render_duration_s": service_data.get("render_duration_s"),
|
|
"ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"),
|
|
"stl_size_bytes": service_data.get("stl_size_bytes"),
|
|
"output_size_bytes": service_data.get("output_size_bytes"),
|
|
"log_lines": service_data.get("log_lines", []),
|
|
}
|
|
if template:
|
|
render_log["template"] = template.blend_file_path
|
|
pl.step_done("blender_cinematic")
|
|
except Exception as exc:
|
|
success = False
|
|
render_log = {"renderer": "blender", "type": "cinematic", "error": str(exc)[:500]}
|
|
pl.step_error("blender_cinematic", str(exc), exc)
|
|
logger.error("Cinematic render failed for %s: %s", order_line_id, exc)
|
|
elif is_animation:
|
|
# ── Turntable animation path ────────────────────────────────
|
|
emit(order_line_id, f"Starting turntable render: {frame_count} frames @ {fps}fps, {render_width or 1920}x{render_height or 1920}{tmpl_info}")
|
|
pl.step_start("blender_turntable", {"frame_count": frame_count, "fps": fps})
|
|
from app.services.render_blender import is_blender_available, render_turntable_to_file
|
|
if not is_blender_available():
|
|
raise RuntimeError("Blender not available on this worker")
|
|
|
|
from app.services.step_processor import _get_all_settings
|
|
_sys = _get_all_settings()
|
|
try:
|
|
turntable_kwargs = render_invocation.as_turntable_renderer_kwargs(
|
|
step_path=step_path,
|
|
output_path=_Path(output_path),
|
|
default_width=1920,
|
|
default_height=1920,
|
|
default_engine=_sys.get("blender_engine", "cycles"),
|
|
default_samples=int(
|
|
_sys.get(
|
|
f"blender_{render_engine or _sys.get('blender_engine', 'cycles')}_samples",
|
|
128,
|
|
)
|
|
),
|
|
smooth_angle=int(_sys.get("blender_smooth_angle", 30)),
|
|
)
|
|
service_data = render_turntable_to_file(**turntable_kwargs)
|
|
success = True
|
|
render_log = {
|
|
"renderer": "blender",
|
|
"type": "turntable",
|
|
"format": "mp4",
|
|
"engine": render_engine or _sys.get("blender_engine", "cycles"),
|
|
"engine_used": service_data.get("engine_used", "cycles"),
|
|
"samples": render_samples,
|
|
"cycles_device": cycles_device_val,
|
|
"width": render_width or 1920,
|
|
"height": render_height or 1920,
|
|
"frame_count": service_data.get("frame_count", frame_count),
|
|
"fps": fps,
|
|
"total_duration_s": service_data.get("total_duration_s"),
|
|
"stl_duration_s": service_data.get("stl_duration_s"),
|
|
"render_duration_s": service_data.get("render_duration_s"),
|
|
"ffmpeg_duration_s": service_data.get("ffmpeg_duration_s"),
|
|
"stl_size_bytes": service_data.get("stl_size_bytes"),
|
|
"output_size_bytes": service_data.get("output_size_bytes"),
|
|
"log_lines": service_data.get("log_lines", []),
|
|
}
|
|
if template:
|
|
render_log["template"] = template.blend_file_path
|
|
pl.step_done("blender_turntable")
|
|
except Exception as exc:
|
|
success = False
|
|
render_log = {"renderer": "blender", "type": "turntable", "error": str(exc)[:500]}
|
|
pl.step_error("blender_turntable", str(exc), exc)
|
|
logger.error("Turntable render failed for %s: %s", order_line_id, exc)
|
|
else:
|
|
# ── Still image path ────────────────────────────────────────
|
|
_render_path_label = "USD → Blender" if usd_path else "STEP → GLB → Blender"
|
|
emit(order_line_id, f"Calling renderer ({_render_path_label}) {render_width or 'default'}x{render_height or 'default'}{' [transparent]' if transparent_bg else ''}{f' engine={render_engine}' if render_engine else ''}{f' samples={render_samples}' if render_samples else ''}{tmpl_info}")
|
|
pl.step_start("blender_still", {"width": render_width, "height": render_height})
|
|
from app.services.step_processor import render_to_file
|
|
|
|
success, render_log = render_to_file(
|
|
**render_invocation.as_still_renderer_kwargs(
|
|
step_path=cad_file.stored_path,
|
|
output_path=output_path,
|
|
job_id=order_line_id,
|
|
order_line_id=order_line_id,
|
|
)
|
|
)
|
|
if success:
|
|
pl.step_done("blender_still")
|
|
else:
|
|
pl.step_error("blender_still", "render_to_file returned False")
|
|
|
|
render_end = datetime.utcnow()
|
|
elapsed = (render_end - render_start).total_seconds()
|
|
try:
|
|
persist_order_line_output(
|
|
session,
|
|
line,
|
|
success=success,
|
|
output_path=output_path,
|
|
render_log=render_log if isinstance(render_log, dict) else None,
|
|
render_completed_at=render_end,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to persist render output for order_line %s", order_line_id)
|
|
raise
|
|
|
|
if success:
|
|
emit(order_line_id, f"Render completed in {elapsed:.1f}s", "success")
|
|
else:
|
|
emit(order_line_id, f"Render failed after {elapsed:.1f}s", "error")
|
|
|
|
emit_order_line_render_notifications(
|
|
success=success,
|
|
order_line_id=order_line_id,
|
|
tenant_id=str(line.product.cad_file.tenant_id) if (
|
|
line.product and line.product.cad_file and line.product.cad_file.tenant_id
|
|
) else None,
|
|
product_name=product_name,
|
|
output_type_name=ot_name,
|
|
render_log=render_log if isinstance(render_log, dict) else None,
|
|
session=session,
|
|
line=line,
|
|
)
|
|
|
|
# Check if all lines for this order are done → auto-advance
|
|
order_id_str = str(line.order_id)
|
|
|
|
engine.dispose()
|
|
|
|
from app.services.order_status_service import check_order_completion
|
|
check_order_completion(order_id_str)
|
|
|
|
pl.step_done("render_order_line_task")
|
|
|
|
except Exception as exc:
|
|
logger.error(f"render_order_line_task failed for {order_line_id}: {exc}")
|
|
# If retries exhausted, mark as failed so the line doesn't stay stuck
|
|
if self.request.retries >= self.max_retries:
|
|
logger.error(f"Max retries reached for {order_line_id}, marking as failed")
|
|
try:
|
|
from sqlalchemy import create_engine, update as sql_update2
|
|
from sqlalchemy.orm import Session as SyncSession
|
|
from app.config import settings as app_settings
|
|
from app.models.order_line import OrderLine as OL2
|
|
sync_url2 = app_settings.database_url.replace("+asyncpg", "")
|
|
eng2 = create_engine(sync_url2)
|
|
with SyncSession(eng2) as s2:
|
|
set_tenant_context_sync(s2, _tenant_id)
|
|
from datetime import datetime as dt2
|
|
s2.execute(
|
|
sql_update2(OL2).where(OL2.id == order_line_id)
|
|
.values(
|
|
render_status="failed",
|
|
render_completed_at=dt2.utcnow(),
|
|
render_log={"error": str(exc)[:500]},
|
|
)
|
|
)
|
|
s2.commit()
|
|
eng2.dispose()
|
|
from app.services.order_status_service import check_order_completion
|
|
# Try to get order_id from DB
|
|
eng3 = create_engine(sync_url2)
|
|
with SyncSession(eng3) as s3:
|
|
set_tenant_context_sync(s3, _tenant_id)
|
|
from sqlalchemy import select as sel
|
|
row = s3.execute(sel(OL2.order_id).where(OL2.id == order_line_id)).scalar_one_or_none()
|
|
if row:
|
|
check_order_completion(str(row))
|
|
eng3.dispose()
|
|
# Notify the order creator about the failure
|
|
try:
|
|
from sqlalchemy import select as sel2
|
|
from app.models.order import Order as OrderModel2
|
|
from app.domains.rendering.workflow_runtime_services import (
|
|
emit_order_line_render_notifications,
|
|
)
|
|
eng4 = create_engine(sync_url2)
|
|
with SyncSession(eng4) as s4:
|
|
set_tenant_context_sync(s4, _tenant_id)
|
|
order_row2 = s4.execute(
|
|
sel2(OrderModel2.created_by, OrderModel2.order_number)
|
|
.join(OL2, OL2.order_id == OrderModel2.id)
|
|
.where(OL2.id == order_line_id)
|
|
).one_or_none()
|
|
eng4.dispose()
|
|
if order_row2:
|
|
emit_order_line_render_notifications(
|
|
success=False,
|
|
order_line_id=order_line_id,
|
|
order_number=order_row2[1],
|
|
order_creator_id=str(order_row2[0]),
|
|
product_name="unknown",
|
|
output_type_name="unknown",
|
|
render_log={"error": str(exc)},
|
|
emit_websocket=False,
|
|
activity_entity_id=None,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to emit render failure activity event")
|
|
except Exception:
|
|
logger.exception(f"Failed to mark {order_line_id} as failed in DB")
|
|
raise
|
|
raise self.retry(exc=exc, countdown=60)
|