refactor: rename thumbnail_rendering queue to asset_pipeline
The queue handles far more than thumbnails: OCC tessellation, USD master generation, GLB production, order line renders, and workflow renders. asset_pipeline better reflects its role as the render-worker's primary queue. Updated all references in: task decorators, celery_app.py, beat_tasks.py, docker-compose.yml worker command, worker.py MONITORED_QUEUES, admin.py, CLAUDE.md, LEARNINGS.md, Dockerfile, helpTexts.ts, test files, and all .claude/commands/*.md skill files. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -26,7 +26,7 @@ def upgrade() -> None:
|
||||
INSERT INTO worker_configs (queue_name, max_concurrency, min_concurrency, enabled)
|
||||
VALUES
|
||||
('step_processing', 8, 2, true),
|
||||
('thumbnail_rendering', 1, 1, true),
|
||||
('asset_pipeline', 1, 1, true),
|
||||
('ai_validation', 4, 1, true)
|
||||
ON CONFLICT DO NOTHING
|
||||
""")
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
"""rename_tessellation_settings_gltf_production_to_scene
|
||||
|
||||
Revision ID: 6ebfe2737531
|
||||
Revises: 062
|
||||
Create Date: 2026-03-12 20:39:36.880236
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '6ebfe2737531'
|
||||
down_revision: Union[str, None] = '062'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("""
|
||||
UPDATE system_settings
|
||||
SET key = 'scene_linear_deflection'
|
||||
WHERE key = 'gltf_production_linear_deflection'
|
||||
""")
|
||||
op.execute("""
|
||||
UPDATE system_settings
|
||||
SET key = 'scene_angular_deflection'
|
||||
WHERE key = 'gltf_production_angular_deflection'
|
||||
""")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("""
|
||||
UPDATE system_settings
|
||||
SET key = 'gltf_production_linear_deflection'
|
||||
WHERE key = 'scene_linear_deflection'
|
||||
""")
|
||||
op.execute("""
|
||||
UPDATE system_settings
|
||||
SET key = 'gltf_production_angular_deflection'
|
||||
WHERE key = 'scene_angular_deflection'
|
||||
""")
|
||||
@@ -366,7 +366,7 @@ async def update_settings(
|
||||
await db.commit()
|
||||
|
||||
# Note: blender-renderer HTTP service removed; concurrency is now controlled
|
||||
# via render-worker Docker concurrency setting (thumbnail_rendering queue).
|
||||
# via render-worker Docker concurrency setting (asset_pipeline queue).
|
||||
|
||||
return _settings_to_out(await _load_settings(db))
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""CAD file router - serve thumbnails, glTF models, parsed objects, and trigger reprocessing."""
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -20,6 +21,7 @@ from app.utils.auth import get_current_user, is_privileged
|
||||
from app.services.product_service import link_cad_to_product, lookup_product
|
||||
|
||||
router = APIRouter(prefix="/cad", tags=["cad"])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -273,6 +275,7 @@ async def get_objects(
|
||||
"cad_file_id": str(cad.id),
|
||||
"original_name": cad.original_name,
|
||||
"processing_status": cad.processing_status.value,
|
||||
"step_hash": cad.step_file_hash,
|
||||
"parsed_objects": cad.parsed_objects,
|
||||
}
|
||||
|
||||
@@ -318,6 +321,11 @@ async def generate_gltf_production(
|
||||
if not cad.stored_path:
|
||||
raise HTTPException(status_code=404, detail="STEP file not uploaded for this CAD file")
|
||||
|
||||
logger.warning(
|
||||
"generate_gltf_production called for cad %s — "
|
||||
"deprecated: renders now consume usd_master directly",
|
||||
id,
|
||||
)
|
||||
from app.tasks.step_tasks import generate_gltf_production_task
|
||||
task = generate_gltf_production_task.delay(str(id))
|
||||
return {"status": "queued", "task_id": task.id, "cad_file_id": str(id)}
|
||||
|
||||
@@ -1000,6 +1000,53 @@ async def cancel_line_render(
|
||||
}
|
||||
|
||||
|
||||
class RejectLineBody(BaseModel):
|
||||
reason: str = ""
|
||||
|
||||
|
||||
@router.post("/{order_id}/lines/{line_id}/reject", status_code=200)
|
||||
async def reject_order_line(
|
||||
order_id: uuid.UUID,
|
||||
line_id: uuid.UUID,
|
||||
body: RejectLineBody,
|
||||
user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Reject a single order line (admin/PM only).
|
||||
|
||||
Sets item_status to 'rejected' and stores the reason in the notes field.
|
||||
"""
|
||||
if not _is_privileged(user):
|
||||
raise HTTPException(status_code=403, detail="Insufficient permissions")
|
||||
|
||||
result = await db.execute(select(Order).where(Order.id == order_id))
|
||||
order = result.scalar_one_or_none()
|
||||
if not order:
|
||||
raise HTTPException(404, detail="Order not found")
|
||||
|
||||
line_result = await db.execute(
|
||||
select(OrderLine).where(OrderLine.id == line_id, OrderLine.order_id == order_id)
|
||||
)
|
||||
line = line_result.scalar_one_or_none()
|
||||
if not line:
|
||||
raise HTTPException(404, detail="Order line not found")
|
||||
|
||||
from sqlalchemy import update as sql_update
|
||||
|
||||
notes_value = body.reason.strip() if body.reason and body.reason.strip() else line.notes
|
||||
await db.execute(
|
||||
sql_update(OrderLine)
|
||||
.where(OrderLine.id == line.id)
|
||||
.values(
|
||||
item_status="rejected",
|
||||
notes=notes_value,
|
||||
)
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
return {"rejected": True, "line_id": str(line.id), "reason": body.reason}
|
||||
|
||||
|
||||
@router.post("/{order_id}/cancel-renders")
|
||||
async def cancel_order_renders(
|
||||
order_id: uuid.UUID,
|
||||
|
||||
@@ -237,7 +237,7 @@ async def reprocess_cad_file(
|
||||
# Queue inspection + control
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
MONITORED_QUEUES = ["step_processing", "thumbnail_rendering", "ai_validation"]
|
||||
MONITORED_QUEUES = ["step_processing", "asset_pipeline", "ai_validation"]
|
||||
|
||||
|
||||
def _parse_redis_task(raw: str) -> dict | None:
|
||||
@@ -515,7 +515,7 @@ async def render_health(
|
||||
|
||||
details: dict = {}
|
||||
|
||||
# 1. Check if render-worker (thumbnail_rendering queue) is connected + has Blender
|
||||
# 1. Check if render-worker (asset_pipeline queue) is connected + has Blender
|
||||
render_worker_connected = False
|
||||
blender_available = False
|
||||
|
||||
@@ -534,10 +534,10 @@ async def render_health(
|
||||
else:
|
||||
all_workers = list(inspect_result.get("ping", {}).keys())
|
||||
details["workers"] = all_workers
|
||||
# Find any worker consuming thumbnail_rendering queue
|
||||
# Find any worker consuming asset_pipeline queue
|
||||
for worker_name, queues in inspect_result.get("active_queues", {}).items():
|
||||
queue_names = [q.get("name") for q in (queues or [])]
|
||||
if "thumbnail_rendering" in queue_names:
|
||||
if "asset_pipeline" in queue_names:
|
||||
render_worker_connected = True
|
||||
# render-worker always has Blender — it starts Blender successfully
|
||||
blender_available = True
|
||||
@@ -547,11 +547,11 @@ async def render_health(
|
||||
render_worker_connected = True
|
||||
details["worker_detection"] = "fallback"
|
||||
|
||||
# 3. Queue depth for thumbnail_rendering
|
||||
# 3. Queue depth for asset_pipeline
|
||||
thumbnail_queue_depth = 0
|
||||
try:
|
||||
r = redis_lib.from_url(app_settings.redis_url, decode_responses=True)
|
||||
thumbnail_queue_depth = r.llen("thumbnail_rendering") or 0
|
||||
thumbnail_queue_depth = r.llen("asset_pipeline") or 0
|
||||
except Exception as exc:
|
||||
details["redis_error"] = str(exc)
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ CATALOG_SCRIPT = Path(os.environ.get("RENDER_SCRIPTS_DIR", "/render-scripts")) /
|
||||
|
||||
@celery_app.task(
|
||||
name="app.domains.materials.tasks.refresh_asset_library_catalog",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
bind=True,
|
||||
max_retries=2,
|
||||
default_retry_delay=30,
|
||||
|
||||
@@ -14,8 +14,8 @@ class MediaAssetType(str, enum.Enum):
|
||||
turntable = "turntable"
|
||||
stl_low = "stl_low"
|
||||
stl_high = "stl_high"
|
||||
gltf_geometry = "gltf_geometry"
|
||||
gltf_production = "gltf_production"
|
||||
gltf_geometry = "gltf_geometry" # DEPRECATED: use usd_master — viewer GLB auto-generated as part of USD pipeline
|
||||
gltf_production = "gltf_production" # DEPRECATED: use usd_master — high-quality production GLB superseded by USD master
|
||||
blend_production = "blend_production"
|
||||
usd_master = "usd_master"
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ from app.core.pipeline_logger import PipelineLogger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.generate_gltf_geometry_task", queue="thumbnail_rendering", max_retries=1)
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.generate_gltf_geometry_task", queue="asset_pipeline", max_retries=1)
|
||||
def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
"""Export a geometry GLB directly from STEP via OCC (no STL intermediary).
|
||||
|
||||
@@ -83,25 +83,47 @@ def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
settings_rows = session.execute(_select(_SysSetting)).scalars().all()
|
||||
sys_settings = {s.key: s.value for s in settings_rows}
|
||||
|
||||
# Hash-based cache check: skip tessellation if file hasn't changed
|
||||
step_file_hash = cad_file.step_file_hash
|
||||
if step_file_hash:
|
||||
from app.domains.media.models import MediaAsset, MediaAssetType
|
||||
import uuid as _uuid_check
|
||||
linear_deflection = float(sys_settings.get("scene_linear_deflection", "0.1"))
|
||||
angular_deflection = float(sys_settings.get("scene_angular_deflection", "0.1"))
|
||||
tessellation_engine = sys_settings.get("tessellation_engine", "occ")
|
||||
|
||||
# Hash-based cache check: skip tessellation if file and settings haven't changed
|
||||
from app.domains.products.cache_service import compute_step_hash as _compute_step_hash
|
||||
from app.domains.media.models import MediaAsset, MediaAssetType
|
||||
import uuid as _uuid_check
|
||||
_current_hash = _compute_step_hash(str(step_path_str))
|
||||
_cache_hit_asset_id = None
|
||||
|
||||
# Composite cache key includes deflection settings so changing them invalidates cache
|
||||
effective_cache_key = (
|
||||
f"{_current_hash}:{linear_deflection}:{angular_deflection}:{tessellation_engine}"
|
||||
if _current_hash else None
|
||||
)
|
||||
|
||||
if effective_cache_key:
|
||||
existing_geo = session.execute(
|
||||
_select(MediaAsset).where(
|
||||
MediaAsset.cad_file_id == _uuid_check.UUID(cad_file_id),
|
||||
MediaAsset.asset_type == MediaAssetType.gltf_geometry,
|
||||
)
|
||||
).scalars().first()
|
||||
if existing_geo:
|
||||
logger.info("[CACHE] hash match — skipping geometry GLB tessellation for %s", cad_file_id)
|
||||
pl.step_done("export_glb_geometry", result={"cached": True, "asset_id": str(existing_geo.id)})
|
||||
_cache_hit_asset_id = str(existing_geo.id)
|
||||
stored_key = (existing_geo.render_config or {}).get("cache_key", "") if existing_geo else ""
|
||||
if stored_key == effective_cache_key:
|
||||
_asset_disk_path = _Path(app_settings.upload_dir) / existing_geo.storage_key
|
||||
if _asset_disk_path.exists():
|
||||
logger.info("[CACHE] cache key match — skipping geometry GLB tessellation for %s", cad_file_id)
|
||||
pl.step_done("export_glb_geometry", result={"cached": True, "asset_id": str(existing_geo.id)})
|
||||
_cache_hit_asset_id = str(existing_geo.id)
|
||||
else:
|
||||
logger.info("[CACHE] cache key match but asset missing on disk — re-running tessellation for %s", cad_file_id)
|
||||
else:
|
||||
_cache_hit_asset_id = None
|
||||
# Cache miss: update stored hash so next run can use it
|
||||
cad_file.step_file_hash = _current_hash
|
||||
session.commit()
|
||||
else:
|
||||
_cache_hit_asset_id = None
|
||||
# No hash available: update stored hash and proceed
|
||||
cad_file.step_file_hash = _current_hash
|
||||
session.commit()
|
||||
eng.dispose()
|
||||
|
||||
if _cache_hit_asset_id is not None:
|
||||
@@ -112,10 +134,6 @@ def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
logger.debug("Could not queue generate_usd_master_task from cache-hit path (non-fatal)")
|
||||
return {"cached": True, "asset_id": _cache_hit_asset_id}
|
||||
|
||||
linear_deflection = float(sys_settings.get("scene_linear_deflection", "0.1"))
|
||||
angular_deflection = float(sys_settings.get("scene_angular_deflection", "0.1"))
|
||||
tessellation_engine = sys_settings.get("tessellation_engine", "occ")
|
||||
|
||||
step = _Path(step_path_str)
|
||||
|
||||
if not step.exists():
|
||||
@@ -197,6 +215,7 @@ def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
existing.storage_key = _key
|
||||
existing.mime_type = "model/gltf-binary"
|
||||
existing.file_size_bytes = _file_size
|
||||
existing.render_config = {"cache_key": effective_cache_key}
|
||||
if product_id:
|
||||
existing.product_id = _uuid.UUID(product_id)
|
||||
_sess.commit()
|
||||
@@ -209,6 +228,7 @@ def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
storage_key=_key,
|
||||
mime_type="model/gltf-binary",
|
||||
file_size_bytes=_file_size,
|
||||
render_config={"cache_key": effective_cache_key},
|
||||
)
|
||||
_sess.add(asset)
|
||||
_sess.commit()
|
||||
@@ -234,7 +254,7 @@ def generate_gltf_geometry_task(self, cad_file_id: str):
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.tasks.step_tasks.generate_gltf_production_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=2,
|
||||
)
|
||||
def generate_gltf_production_task(self, cad_file_id: str, product_id: str | None = None) -> dict:
|
||||
@@ -511,7 +531,7 @@ def generate_gltf_production_task(self, cad_file_id: str, product_id: str | None
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.tasks.step_tasks.generate_usd_master_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline", # needs pxr (usd-core) + OCC — both only in render-worker
|
||||
max_retries=1,
|
||||
)
|
||||
def generate_usd_master_task(self, cad_file_id: str) -> dict:
|
||||
@@ -583,19 +603,44 @@ def generate_usd_master_task(self, cad_file_id: str) -> dict:
|
||||
settings_rows = sess.execute(_sel(SystemSetting)).scalars().all()
|
||||
sys_settings = {s.key: s.value for s in settings_rows}
|
||||
|
||||
# Hash-based cache check: skip tessellation if file hasn't changed
|
||||
step_file_hash = cad_file.step_file_hash
|
||||
if step_file_hash:
|
||||
linear_deflection = float(sys_settings.get("render_linear_deflection", "0.03"))
|
||||
angular_deflection = float(sys_settings.get("render_angular_deflection", "0.05"))
|
||||
sharp_threshold = float(sys_settings.get("sharp_edge_threshold", "20.0"))
|
||||
|
||||
# Hash-based cache check: skip tessellation if file and settings haven't changed
|
||||
from app.domains.products.cache_service import compute_step_hash as _compute_step_hash_usd
|
||||
_current_hash_usd = _compute_step_hash_usd(str(step_path))
|
||||
|
||||
# Composite cache key includes deflection settings so changing them invalidates cache
|
||||
effective_cache_key = (
|
||||
f"{_current_hash_usd}:{linear_deflection}:{angular_deflection}:{sharp_threshold}"
|
||||
if _current_hash_usd else None
|
||||
)
|
||||
|
||||
if effective_cache_key:
|
||||
existing_usd = sess.execute(
|
||||
_sel(MediaAsset).where(
|
||||
MediaAsset.cad_file_id == cad_file.id,
|
||||
MediaAsset.asset_type == MediaAssetType.usd_master,
|
||||
)
|
||||
).scalars().first()
|
||||
if existing_usd:
|
||||
logger.info("[CACHE] hash match — skipping USD master tessellation for %s", cad_file_id)
|
||||
pl.step_done("usd_master", result={"cached": True, "asset_id": str(existing_usd.id)})
|
||||
_cache_hit_asset_id = str(existing_usd.id)
|
||||
stored_key = (existing_usd.render_config or {}).get("cache_key", "") if existing_usd else ""
|
||||
if stored_key == effective_cache_key:
|
||||
_usd_disk_path = _Path(app_settings.upload_dir) / existing_usd.storage_key
|
||||
if _usd_disk_path.exists():
|
||||
logger.info("[CACHE] cache key match — skipping USD master tessellation for %s", cad_file_id)
|
||||
pl.step_done("usd_master", result={"cached": True, "asset_id": str(existing_usd.id)})
|
||||
_cache_hit_asset_id = str(existing_usd.id)
|
||||
else:
|
||||
logger.info("[CACHE] cache key match but USD asset missing on disk — re-running tessellation for %s", cad_file_id)
|
||||
else:
|
||||
# Cache miss: update stored hash so next run can use it
|
||||
cad_file.step_file_hash = _current_hash_usd
|
||||
sess.commit()
|
||||
else:
|
||||
# No hash available: update stored hash and proceed
|
||||
cad_file.step_file_hash = _current_hash_usd
|
||||
sess.commit()
|
||||
eng.dispose()
|
||||
|
||||
if _cache_hit_asset_id is not None:
|
||||
@@ -606,10 +651,6 @@ def generate_usd_master_task(self, cad_file_id: str) -> dict:
|
||||
pl.step_error("usd_master", err, None)
|
||||
raise RuntimeError(err)
|
||||
|
||||
linear_deflection = float(sys_settings.get("render_linear_deflection", "0.03"))
|
||||
angular_deflection = float(sys_settings.get("render_angular_deflection", "0.05"))
|
||||
sharp_threshold = float(sys_settings.get("sharp_edge_threshold", "20.0"))
|
||||
|
||||
output_path = step_path.parent / f"{step_path.stem}_master.usd"
|
||||
scripts_dir = _Path(_os.environ.get("RENDER_SCRIPTS_DIR", "/render-scripts"))
|
||||
script_path = scripts_dir / "export_step_to_usd.py"
|
||||
@@ -675,6 +716,7 @@ def generate_usd_master_task(self, cad_file_id: str) -> dict:
|
||||
existing.storage_key = _key
|
||||
existing.mime_type = "model/vnd.usd"
|
||||
existing.file_size_bytes = _file_size
|
||||
existing.render_config = {"cache_key": effective_cache_key}
|
||||
sess2.commit()
|
||||
asset_id = str(existing.id)
|
||||
else:
|
||||
@@ -684,6 +726,7 @@ def generate_usd_master_task(self, cad_file_id: str) -> dict:
|
||||
storage_key=_key,
|
||||
mime_type="model/vnd.usd",
|
||||
file_size_bytes=_file_size,
|
||||
render_config={"cache_key": effective_cache_key},
|
||||
)
|
||||
sess2.add(asset)
|
||||
sess2.commit()
|
||||
|
||||
@@ -104,7 +104,7 @@ def process_step_file(self, cad_file_id: str):
|
||||
pl.info("process_step_file", f"Processing STEP file (metadata only): {cad_file_id}")
|
||||
try:
|
||||
from app.services.step_processor import extract_cad_metadata
|
||||
extract_cad_metadata(cad_file_id)
|
||||
extract_cad_metadata(cad_file_id, tenant_id=_tenant_id)
|
||||
except Exception as exc:
|
||||
pl.step_error("process_step_file", f"STEP metadata extraction failed: {exc}", exc)
|
||||
r.delete(lock_key) # release lock so a retry can proceed
|
||||
@@ -119,7 +119,7 @@ def process_step_file(self, cad_file_id: str):
|
||||
render_step_thumbnail.delay(cad_file_id)
|
||||
|
||||
|
||||
def _auto_populate_materials_for_cad(cad_file_id: str) -> None:
|
||||
def _auto_populate_materials_for_cad(cad_file_id: str, tenant_id: str | None = None) -> None:
|
||||
"""Sync helper: auto-populate cad_part_materials from Excel for newly-processed CAD files.
|
||||
|
||||
Only fills products where cad_part_materials is empty or all-blank,
|
||||
@@ -132,10 +132,12 @@ def _auto_populate_materials_for_cad(cad_file_id: str) -> None:
|
||||
from app.models.product import Product
|
||||
from app.api.routers.products import build_materials_from_excel
|
||||
from app.services.step_processor import build_part_colors
|
||||
from app.core.tenant_context import set_tenant_context_sync
|
||||
|
||||
sync_url = app_settings.database_url.replace("+asyncpg", "")
|
||||
eng = create_engine(sync_url)
|
||||
with Session(eng) as session:
|
||||
set_tenant_context_sync(session, tenant_id)
|
||||
# Load the CAD file to get parsed objects
|
||||
cad_file = session.execute(
|
||||
sql_select(CadFile).where(CadFile.id == cad_file_id)
|
||||
@@ -201,7 +203,7 @@ def _auto_populate_materials_for_cad(cad_file_id: str) -> None:
|
||||
eng.dispose()
|
||||
|
||||
|
||||
@celery_app.task(name="app.tasks.step_tasks.reextract_cad_metadata", queue="thumbnail_rendering")
|
||||
@celery_app.task(name="app.tasks.step_tasks.reextract_cad_metadata", queue="asset_pipeline")
|
||||
def reextract_cad_metadata(cad_file_id: str):
|
||||
"""Re-extract bounding-box dimensions for an already-completed CAD file.
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ def dispatch_order_line_render(order_line_id: str):
|
||||
render_order_line_task.delay(order_line_id)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="thumbnail_rendering", max_retries=3)
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_order_line_task", queue="asset_pipeline", max_retries=3)
|
||||
def render_order_line_task(self, order_line_id: str):
|
||||
"""Render a specific output type for an order line.
|
||||
|
||||
|
||||
@@ -14,11 +14,11 @@ from app.core.pipeline_logger import PipelineLogger
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="thumbnail_rendering")
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.render_step_thumbnail", queue="asset_pipeline")
|
||||
def render_step_thumbnail(self, cad_file_id: str):
|
||||
"""Render the thumbnail for a freshly-processed STEP file.
|
||||
|
||||
Runs on the dedicated thumbnail_rendering queue (concurrency=1) so the
|
||||
Runs on the dedicated asset_pipeline queue (concurrency=1) so the
|
||||
blender-renderer service is never overwhelmed by concurrent requests.
|
||||
On success, also auto-populates materials and marks the CadFile as completed.
|
||||
"""
|
||||
@@ -139,7 +139,7 @@ def render_step_thumbnail(self, cad_file_id: str):
|
||||
# Auto-populate materials now that parsed_objects are available
|
||||
try:
|
||||
from app.domains.pipeline.tasks.extract_metadata import _auto_populate_materials_for_cad
|
||||
_auto_populate_materials_for_cad(cad_file_id)
|
||||
_auto_populate_materials_for_cad(cad_file_id, tenant_id=_tenant_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
f"Auto material population failed for cad_file {cad_file_id} (non-fatal)"
|
||||
@@ -180,7 +180,7 @@ def render_step_thumbnail(self, cad_file_id: str):
|
||||
pl.step_done("render_step_thumbnail")
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="thumbnail_rendering")
|
||||
@celery_app.task(bind=True, name="app.tasks.step_tasks.regenerate_thumbnail", queue="asset_pipeline")
|
||||
def regenerate_thumbnail(self, cad_file_id: str, part_colors: dict):
|
||||
"""Regenerate thumbnail with per-part colours."""
|
||||
pl = PipelineLogger(task_id=self.request.id)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Rendering domain tasks — Celery tasks for Blender-based rendering.
|
||||
|
||||
These tasks run on the `thumbnail_rendering` queue in the render-worker
|
||||
These tasks run on the `asset_pipeline` queue in the render-worker
|
||||
container, which has Blender and cadquery available.
|
||||
|
||||
Phase A2: Initial implementation replacing the blender-renderer HTTP service.
|
||||
@@ -48,7 +48,7 @@ def _update_workflow_run_status(order_line_id: str, status: str, error: str | No
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.render_still_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=2,
|
||||
)
|
||||
def render_still_task(
|
||||
@@ -150,7 +150,7 @@ def render_still_task(
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.render_turntable_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=2,
|
||||
)
|
||||
def render_turntable_task(
|
||||
@@ -391,7 +391,7 @@ def _resolve_step_path_for_order_line(order_line_id: str) -> tuple[str | None, s
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.render_order_line_still_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=2,
|
||||
)
|
||||
def render_order_line_still_task(self, order_line_id: str, **params) -> dict:
|
||||
@@ -509,7 +509,7 @@ def render_order_line_still_task(self, order_line_id: str, **params) -> dict:
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.export_gltf_for_order_line_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=1,
|
||||
)
|
||||
def export_gltf_for_order_line_task(self, order_line_id: str) -> dict:
|
||||
@@ -555,7 +555,7 @@ def export_gltf_for_order_line_task(self, order_line_id: str) -> dict:
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.export_blend_for_order_line_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=1,
|
||||
)
|
||||
def export_blend_for_order_line_task(self, order_line_id: str) -> dict:
|
||||
@@ -646,7 +646,7 @@ def export_blend_for_order_line_task(self, order_line_id: str) -> dict:
|
||||
@celery_app.task(
|
||||
bind=True,
|
||||
name="app.domains.rendering.tasks.apply_asset_library_materials_task",
|
||||
queue="thumbnail_rendering",
|
||||
queue="asset_pipeline",
|
||||
max_retries=1,
|
||||
)
|
||||
def apply_asset_library_materials_task(self, order_line_id: str, asset_library_id: str) -> dict:
|
||||
|
||||
@@ -105,6 +105,10 @@ def build_scene_manifest(cad_file, usd_asset=None) -> dict:
|
||||
object_names: list[str] = cad_file.parsed_objects.get("objects") or []
|
||||
seen_keys: set[str] = set()
|
||||
for source_name in object_names:
|
||||
# Fallback: USD master not yet generated. Use source_name as xcaf_path proxy.
|
||||
# Note: slugs produced here may differ from what export_step_to_usd.py will
|
||||
# produce for unnamed parts (which use sha256 of the XCAF hierarchy path).
|
||||
# Named parts will match once USD master is generated.
|
||||
part_key = generate_part_key(source_name, source_name, seen_keys)
|
||||
effective_material, provenance = _resolve_material(
|
||||
part_key, source_name, manual, resolved, source
|
||||
|
||||
@@ -15,8 +15,8 @@ from pathlib import Path
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _glb_from_step(step_path: Path, glb_path: Path) -> None:
|
||||
"""Convert STEP → GLB via OCC (export_step_to_gltf.py, no Blender needed)."""
|
||||
def _glb_from_step(step_path: Path, glb_path: Path, tessellation_engine: str = "occ") -> None:
|
||||
"""Convert STEP → GLB via OCC or GMSH (export_step_to_gltf.py, no Blender needed)."""
|
||||
import subprocess
|
||||
import sys as _sys
|
||||
|
||||
@@ -32,12 +32,13 @@ def _glb_from_step(step_path: Path, glb_path: Path) -> None:
|
||||
"--output_path", str(glb_path),
|
||||
"--linear_deflection", str(linear_deflection),
|
||||
"--angular_deflection", str(angular_deflection),
|
||||
"--tessellation_engine", tessellation_engine,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
for line in result.stdout.splitlines():
|
||||
logger.info("[occ-gltf] %s", line)
|
||||
logger.info("[export-gltf] %s", line)
|
||||
for line in result.stderr.splitlines():
|
||||
logger.warning("[occ-gltf stderr] %s", line)
|
||||
logger.warning("[export-gltf stderr] %s", line)
|
||||
if result.returncode != 0 or not glb_path.exists() or glb_path.stat().st_size == 0:
|
||||
raise RuntimeError(
|
||||
f"export_step_to_gltf.py failed (exit {result.returncode}).\n"
|
||||
@@ -90,8 +91,9 @@ def render_still(
|
||||
mesh_attributes: dict | None = None,
|
||||
log_callback: "Callable[[str], None] | None" = None,
|
||||
usd_path: "Path | None" = None,
|
||||
tessellation_engine: str = "occ",
|
||||
) -> dict:
|
||||
"""Convert STEP → GLB (OCC) → PNG (Blender subprocess).
|
||||
"""Convert STEP → GLB (OCC or GMSH) → PNG (Blender subprocess).
|
||||
|
||||
When usd_path is provided and the file exists, the GLB conversion step is
|
||||
skipped and Blender imports the USD stage directly (--usd-path flag).
|
||||
@@ -125,7 +127,7 @@ def render_still(
|
||||
glb_size_bytes = 0
|
||||
else:
|
||||
if not glb_path.exists() or glb_path.stat().st_size == 0:
|
||||
_glb_from_step(step_path, glb_path)
|
||||
_glb_from_step(step_path, glb_path, tessellation_engine)
|
||||
else:
|
||||
logger.info("GLB local hit: %s (%d KB)", glb_path.name, glb_path.stat().st_size // 1024)
|
||||
glb_size_bytes = glb_path.stat().st_size if glb_path.exists() else 0
|
||||
@@ -310,6 +312,7 @@ def render_turntable_to_file(
|
||||
rotation_y: float = 0.0,
|
||||
rotation_z: float = 0.0,
|
||||
usd_path: "Path | None" = None,
|
||||
tessellation_engine: str = "occ",
|
||||
) -> dict:
|
||||
"""Render a turntable animation: STEP → STL → N frames (Blender) → mp4 (ffmpeg).
|
||||
|
||||
@@ -349,7 +352,7 @@ def render_turntable_to_file(
|
||||
logger.info("[render_blender] turntable using USD path: %s", usd_path)
|
||||
else:
|
||||
if not glb_path.exists() or glb_path.stat().st_size == 0:
|
||||
_glb_from_step(step_path, glb_path)
|
||||
_glb_from_step(step_path, glb_path, tessellation_engine)
|
||||
else:
|
||||
logger.info("GLB local hit: %s (%d KB)", glb_path.name, glb_path.stat().st_size // 1024)
|
||||
glb_duration_s = round(time.monotonic() - t_glb, 2)
|
||||
|
||||
@@ -79,7 +79,7 @@ def match_cad_to_items(
|
||||
return matched
|
||||
|
||||
|
||||
def extract_cad_metadata(cad_file_id: str) -> None:
|
||||
def extract_cad_metadata(cad_file_id: str, tenant_id: str | None = None) -> None:
|
||||
"""
|
||||
Fast metadata extraction for a CAD file (no thumbnail generation).
|
||||
|
||||
@@ -94,9 +94,11 @@ def extract_cad_metadata(cad_file_id: str) -> None:
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
from app.models.cad_file import CadFile, ProcessingStatus
|
||||
from app.core.tenant_context import set_tenant_context_sync
|
||||
|
||||
engine = create_engine(settings.database_url_sync)
|
||||
with Session(engine) as session:
|
||||
set_tenant_context_sync(session, tenant_id)
|
||||
cad_file = session.get(CadFile, uuid.UUID(cad_file_id))
|
||||
if not cad_file:
|
||||
logger.error(f"CAD file not found: {cad_file_id}")
|
||||
@@ -452,6 +454,7 @@ def _get_all_settings() -> dict[str, str]:
|
||||
"thumbnail_format": "jpg",
|
||||
"blender_smooth_angle": "30",
|
||||
"cycles_device": "auto",
|
||||
"tessellation_engine": "occ",
|
||||
}
|
||||
try:
|
||||
from app.config import settings as app_settings
|
||||
@@ -533,6 +536,7 @@ def _generate_thumbnail(
|
||||
samples=samples,
|
||||
smooth_angle=int(settings["blender_smooth_angle"]),
|
||||
cycles_device=settings["cycles_device"],
|
||||
tessellation_engine=settings["tessellation_engine"],
|
||||
)
|
||||
rendered_png = tmp_png if tmp_png.exists() else None
|
||||
except Exception as exc:
|
||||
@@ -642,6 +646,7 @@ def render_to_file(
|
||||
denoising_use_gpu: str = "",
|
||||
order_line_id: str | None = None,
|
||||
usd_path: "Path | None" = None,
|
||||
tessellation_engine: str | None = None,
|
||||
) -> tuple[bool, dict]:
|
||||
"""Render a STEP file to a specific output path using current system settings.
|
||||
|
||||
@@ -777,6 +782,7 @@ def render_to_file(
|
||||
denoising_use_gpu=denoising_use_gpu,
|
||||
log_callback=_log_cb,
|
||||
usd_path=usd_path,
|
||||
tessellation_engine=tessellation_engine or settings["tessellation_engine"],
|
||||
)
|
||||
rendered_png = tmp_png if tmp_png.exists() else None
|
||||
except Exception as exc:
|
||||
|
||||
@@ -73,7 +73,7 @@ def broadcast_queue_status() -> None:
|
||||
r = sync_redis.from_url(settings.redis_url, decode_responses=True)
|
||||
depths = {
|
||||
"step_processing": r.llen("step_processing"),
|
||||
"thumbnail_rendering": r.llen("thumbnail_rendering"),
|
||||
"asset_pipeline": r.llen("asset_pipeline"),
|
||||
}
|
||||
event = {"type": "queue_update", "depths": depths}
|
||||
r.publish("__broadcast__", json.dumps(event))
|
||||
|
||||
@@ -30,7 +30,7 @@ celery_app.conf.update(
|
||||
enable_utc=True,
|
||||
task_routes={
|
||||
"app.domains.pipeline.tasks.*": {"queue": "step_processing"},
|
||||
"app.domains.rendering.tasks.*": {"queue": "thumbnail_rendering"},
|
||||
"app.domains.rendering.tasks.*": {"queue": "asset_pipeline"},
|
||||
"app.tasks.beat_tasks.*": {"queue": "step_processing"},
|
||||
"app.tasks.ai_tasks.*": {"queue": "ai_validation"},
|
||||
# Legacy task names (shim) — keep until old queued tasks drain
|
||||
|
||||
@@ -5,7 +5,7 @@ from app.tasks.celery_app import celery_app
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(name="app.tasks.gpu_tasks.probe_gpu", queue="thumbnail_rendering")
|
||||
@celery_app.task(name="app.tasks.gpu_tasks.probe_gpu", queue="asset_pipeline")
|
||||
def probe_gpu() -> dict:
|
||||
"""Run Blender GPU probe on the render-worker. Stores result in system_settings."""
|
||||
import subprocess
|
||||
|
||||
@@ -230,7 +230,6 @@ def mock_celery_tasks(monkeypatch):
|
||||
"app.domains.materials.tasks.refresh_asset_library_catalog",
|
||||
"app.tasks.step_tasks.process_step_file",
|
||||
"app.tasks.step_tasks.render_step_thumbnail",
|
||||
"app.tasks.step_tasks.generate_stl_cache",
|
||||
"app.domains.imports.tasks.validate_excel_import",
|
||||
"app.domains.rendering.tasks.render_still_task",
|
||||
"app.domains.rendering.tasks.render_turntable_task",
|
||||
|
||||
@@ -99,14 +99,14 @@ def test_generate_gltf_geometry_task_importable():
|
||||
def test_render_order_line_still_task_importable():
|
||||
from app.domains.rendering.tasks import render_order_line_still_task
|
||||
assert render_order_line_still_task.name == "app.domains.rendering.tasks.render_order_line_still_task"
|
||||
assert render_order_line_still_task.queue == "thumbnail_rendering"
|
||||
assert render_order_line_still_task.queue == "asset_pipeline"
|
||||
|
||||
|
||||
def test_export_gltf_for_order_line_task_importable():
|
||||
from app.domains.rendering.tasks import export_gltf_for_order_line_task
|
||||
assert export_gltf_for_order_line_task.queue == "thumbnail_rendering"
|
||||
assert export_gltf_for_order_line_task.queue == "asset_pipeline"
|
||||
|
||||
|
||||
def test_export_blend_for_order_line_task_importable():
|
||||
from app.domains.rendering.tasks import export_blend_for_order_line_task
|
||||
assert export_blend_for_order_line_task.queue == "thumbnail_rendering"
|
||||
assert export_blend_for_order_line_task.queue == "asset_pipeline"
|
||||
|
||||
Reference in New Issue
Block a user