1457 lines
55 KiB
Python
1457 lines
55 KiB
Python
#!/usr/bin/env python3
|
|
"""Serial live parity runner for real Blender parity-sensitive output types.
|
|
|
|
Creates reversible shadow-workflow probes for active Blender output types,
|
|
dispatches real renders against the live stack, waits for workflow comparisons,
|
|
and prints a compact JSON summary per output type. Still images use the backend
|
|
comparison endpoint directly; turntable videos additionally perform a manual
|
|
frame-sampling comparison so template-backed Legacy and Graph outputs can be
|
|
validated with real Blender renders and identical authoritative settings.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
from PIL import Image, ImageChops, ImageStat
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
HARNESS_PATH = ROOT / "scripts" / "test_render_pipeline.py"
|
|
|
|
|
|
def _load_harness():
|
|
spec = importlib.util.spec_from_file_location("live_render_harness", HARNESS_PATH)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError(f"Could not load harness from {HARNESS_PATH}")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
SUPPORTED_ARTIFACT_KINDS = {"still_image", "thumbnail_image", "turntable_video"}
|
|
|
|
|
|
def _is_real_blender_output_type(output_type: dict, *, include_generated: bool) -> bool:
|
|
if not output_type.get("is_active", True):
|
|
return False
|
|
if output_type.get("renderer") != "blender":
|
|
return False
|
|
if output_type.get("artifact_kind") not in SUPPORTED_ARTIFACT_KINDS:
|
|
return False
|
|
if include_generated:
|
|
return True
|
|
name = str(output_type.get("name") or "")
|
|
return not name.startswith("[")
|
|
|
|
|
|
def _normalize_thumbnail_render_params(output_type: dict) -> dict:
|
|
normalized = harness_like_normalize(dict(output_type.get("render_settings") or {}))
|
|
if "transparent_bg" not in normalized and output_type.get("transparent_bg") is not None:
|
|
normalized["transparent_bg"] = bool(output_type.get("transparent_bg"))
|
|
return {
|
|
key: value
|
|
for key, value in normalized.items()
|
|
if key in {"render_engine", "samples", "width", "height", "transparent_bg"}
|
|
}
|
|
|
|
|
|
def harness_like_normalize(params: dict | None = None) -> dict:
|
|
normalized = dict(params or {})
|
|
resolution = normalized.pop("resolution", None)
|
|
if isinstance(resolution, (list, tuple)) and len(resolution) == 2:
|
|
normalized.setdefault("width", int(resolution[0]))
|
|
normalized.setdefault("height", int(resolution[1]))
|
|
if "engine" in normalized and "render_engine" not in normalized:
|
|
normalized["render_engine"] = normalized.pop("engine")
|
|
return normalized
|
|
|
|
|
|
def _build_graph_thumbnail_config(*, execution_mode: str, output_type: dict) -> dict:
|
|
render_params = _normalize_thumbnail_render_params(output_type)
|
|
renderer = str(output_type.get("renderer") or "blender").lower().strip()
|
|
render_step = "blender_render" if renderer == "blender" else "threejs_render"
|
|
render_node_id = "thumbnail_render"
|
|
|
|
return {
|
|
"version": 1,
|
|
"ui": {
|
|
"preset": "custom",
|
|
"execution_mode": execution_mode,
|
|
"family": "cad_file",
|
|
"blueprint": "cad_intake",
|
|
},
|
|
"nodes": [
|
|
{
|
|
"id": "resolve_step",
|
|
"step": "resolve_step_path",
|
|
"params": {},
|
|
"ui": {"label": "Resolve STEP Path", "position": {"x": 0, "y": 180}},
|
|
},
|
|
{
|
|
"id": "extract_objects",
|
|
"step": "occ_object_extract",
|
|
"params": {},
|
|
"ui": {"label": "Extract STEP Objects", "position": {"x": 220, "y": 180}},
|
|
},
|
|
{
|
|
"id": "export_glb",
|
|
"step": "occ_glb_export",
|
|
"params": {},
|
|
"ui": {"label": "Export GLB", "position": {"x": 440, "y": 180}},
|
|
},
|
|
{
|
|
"id": render_node_id,
|
|
"step": render_step,
|
|
"params": render_params,
|
|
"ui": {"label": "Render Thumbnail", "position": {"x": 680, "y": 180}},
|
|
},
|
|
{
|
|
"id": "save_thumbnail",
|
|
"step": "thumbnail_save",
|
|
"params": {},
|
|
"ui": {"label": "Save Thumbnail", "position": {"x": 920, "y": 180}},
|
|
},
|
|
],
|
|
"edges": [
|
|
{"from": "resolve_step", "to": "extract_objects"},
|
|
{"from": "extract_objects", "to": "export_glb"},
|
|
{"from": "export_glb", "to": render_node_id},
|
|
{"from": render_node_id, "to": "save_thumbnail"},
|
|
],
|
|
}
|
|
|
|
|
|
def _build_graph_turntable_config(*, execution_mode: str) -> dict:
|
|
return {
|
|
"version": 1,
|
|
"ui": {
|
|
"preset": "turntable",
|
|
"execution_mode": execution_mode,
|
|
"family": "order_line",
|
|
},
|
|
"nodes": [
|
|
{
|
|
"id": "setup",
|
|
"step": "order_line_setup",
|
|
"params": {},
|
|
"ui": {"label": "Order Line Setup", "position": {"x": 0, "y": 100}},
|
|
},
|
|
{
|
|
"id": "template",
|
|
"step": "resolve_template",
|
|
"params": {},
|
|
"ui": {"label": "Resolve Template", "position": {"x": 220, "y": 100}},
|
|
},
|
|
{
|
|
"id": "populate_materials",
|
|
"step": "auto_populate_materials",
|
|
"params": {},
|
|
"ui": {"label": "Auto Populate Materials", "position": {"x": 220, "y": 260}},
|
|
},
|
|
{
|
|
"id": "bbox",
|
|
"step": "glb_bbox",
|
|
"params": {},
|
|
"ui": {"label": "Compute Bounding Box", "position": {"x": 220, "y": -20}},
|
|
},
|
|
{
|
|
"id": "resolve_materials",
|
|
"step": "material_map_resolve",
|
|
"params": {},
|
|
"ui": {"label": "Resolve Material Map", "position": {"x": 440, "y": 160}},
|
|
},
|
|
{
|
|
"id": "turntable",
|
|
"step": "blender_turntable",
|
|
"params": {
|
|
"use_custom_render_settings": False,
|
|
},
|
|
"ui": {"type": "renderFramesNode", "label": "Turntable Render", "position": {"x": 440, "y": 100}},
|
|
},
|
|
{
|
|
"id": "output",
|
|
"step": "output_save",
|
|
"params": {},
|
|
"ui": {"type": "outputNode", "label": "Save Output", "position": {"x": 660, "y": 100}},
|
|
},
|
|
],
|
|
"edges": [
|
|
{"from": "setup", "to": "template"},
|
|
{"from": "setup", "to": "populate_materials"},
|
|
{"from": "setup", "to": "bbox"},
|
|
{"from": "template", "to": "resolve_materials"},
|
|
{"from": "populate_materials", "to": "resolve_materials"},
|
|
{"from": "resolve_materials", "to": "turntable"},
|
|
{"from": "bbox", "to": "turntable"},
|
|
{"from": "template", "to": "turntable"},
|
|
{"from": "turntable", "to": "output"},
|
|
],
|
|
}
|
|
|
|
|
|
def _ensure_shadow_probe_workflow(harness, client, *, output_type: dict) -> dict:
|
|
workflow_name = f"[Parity Matrix] {output_type['name']}"
|
|
workflows = harness.get_workflows(client)
|
|
workflow = harness.find_named(workflows, workflow_name)
|
|
artifact_kind = output_type.get("artifact_kind")
|
|
if artifact_kind == "thumbnail_image":
|
|
workflow_config = _build_graph_thumbnail_config(
|
|
execution_mode="shadow",
|
|
output_type=output_type,
|
|
)
|
|
elif artifact_kind == "turntable_video":
|
|
workflow_config = _build_graph_turntable_config(execution_mode="shadow")
|
|
else:
|
|
workflow_config = harness.build_graph_still_config(
|
|
execution_mode="shadow",
|
|
use_custom_render_settings=False,
|
|
)
|
|
workflow_payload = {
|
|
"name": workflow_name,
|
|
"output_type_id": output_type["id"],
|
|
"config": workflow_config,
|
|
"is_active": True,
|
|
}
|
|
if workflow is None:
|
|
resp = client.post("/workflows", json=workflow_payload)
|
|
if resp.status_code not in (200, 201):
|
|
raise RuntimeError(
|
|
f"Shadow probe workflow create failed for {output_type['name']}: "
|
|
f"{resp.status_code} {resp.text[:400]}"
|
|
)
|
|
workflow = resp.json()
|
|
else:
|
|
resp = client.put(
|
|
f"/workflows/{workflow['id']}",
|
|
json={
|
|
"name": workflow_payload["name"],
|
|
"config": workflow_payload["config"],
|
|
"is_active": workflow_payload["is_active"],
|
|
},
|
|
)
|
|
if resp.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Shadow probe workflow update failed for {output_type['name']}: "
|
|
f"{resp.status_code} {resp.text[:400]}"
|
|
)
|
|
workflow = resp.json()
|
|
return workflow
|
|
|
|
|
|
def _wait_for_workflow_run_id(
|
|
harness,
|
|
client,
|
|
*,
|
|
workflow_id: str,
|
|
workflow_run_id: str,
|
|
timeout_seconds: int,
|
|
) -> dict | None:
|
|
deadline = time.time() + timeout_seconds
|
|
terminal_statuses = {"completed", "failed", "cancelled"}
|
|
while time.time() < deadline:
|
|
resp = client.get(f"/workflows/{workflow_id}/runs")
|
|
if resp.status_code == 200:
|
|
for run in resp.json():
|
|
if str(run.get("id")) != workflow_run_id:
|
|
continue
|
|
if run.get("status") in terminal_statuses:
|
|
return run
|
|
time.sleep(2)
|
|
return None
|
|
|
|
|
|
def _download_bytes(client, path: str) -> tuple[bytes, str | None]:
|
|
response = client.session.get(f"{client.host}{path}", timeout=60)
|
|
response.raise_for_status()
|
|
return response.content, response.headers.get("content-type")
|
|
|
|
|
|
def _build_byte_artifact(payload: bytes, *, path: str | None, mime_type: str | None) -> dict:
|
|
image_width = None
|
|
image_height = None
|
|
if payload:
|
|
try:
|
|
with Image.open(BytesIO(payload)) as image:
|
|
image_width, image_height = image.size
|
|
except Exception:
|
|
image_width = None
|
|
image_height = None
|
|
|
|
guessed_mime_type = mime_type or mimetypes.guess_type(path or "")[0]
|
|
return {
|
|
"path": path,
|
|
"storage_key": None,
|
|
"exists": bool(payload),
|
|
"file_size_bytes": len(payload) if payload else None,
|
|
"sha256": hashlib.sha256(payload).hexdigest() if payload else None,
|
|
"mime_type": guessed_mime_type,
|
|
"image_width": image_width,
|
|
"image_height": image_height,
|
|
}
|
|
|
|
|
|
def _compute_image_delta(authoritative: bytes, observer: bytes) -> tuple[bool | None, float | None]:
|
|
if not authoritative or not observer:
|
|
return None, None
|
|
|
|
try:
|
|
with Image.open(BytesIO(authoritative)) as authoritative_image, Image.open(BytesIO(observer)) as observer_image:
|
|
authoritative_rgba = authoritative_image.convert("RGBA")
|
|
observer_rgba = observer_image.convert("RGBA")
|
|
if authoritative_rgba.size != observer_rgba.size:
|
|
return False, None
|
|
diff = ImageChops.difference(authoritative_rgba, observer_rgba)
|
|
mean_channels = ImageStat.Stat(diff).mean
|
|
return True, sum(mean_channels) / (len(mean_channels) * 255.0)
|
|
except Exception:
|
|
return None, None
|
|
|
|
|
|
def _build_manual_comparison(
|
|
authoritative_bytes: bytes,
|
|
observer_bytes: bytes,
|
|
*,
|
|
authoritative_path: str,
|
|
authoritative_mime_type: str | None,
|
|
observer_path: str,
|
|
observer_mime_type: str | None,
|
|
) -> dict:
|
|
authoritative_output = _build_byte_artifact(
|
|
authoritative_bytes,
|
|
path=authoritative_path,
|
|
mime_type=authoritative_mime_type,
|
|
)
|
|
observer_output = _build_byte_artifact(
|
|
observer_bytes,
|
|
path=observer_path,
|
|
mime_type=observer_mime_type,
|
|
)
|
|
exact_match = (
|
|
authoritative_output["sha256"] == observer_output["sha256"]
|
|
if authoritative_output["exists"] and observer_output["exists"]
|
|
else None
|
|
)
|
|
dimensions_match, mean_pixel_delta = _compute_image_delta(authoritative_bytes, observer_bytes)
|
|
if exact_match is True:
|
|
dimensions_match = True
|
|
mean_pixel_delta = 0.0
|
|
|
|
status = "ready"
|
|
if not authoritative_output["exists"]:
|
|
status = "missing_authoritative"
|
|
elif not observer_output["exists"]:
|
|
status = "missing_observer"
|
|
elif dimensions_match is False:
|
|
status = "dimension_mismatch"
|
|
elif mean_pixel_delta is None and exact_match is not True:
|
|
status = "non_image_or_uncomparable"
|
|
|
|
summary = (
|
|
"Manual thumbnail parity comparison completed."
|
|
if status == "ready"
|
|
else f"Manual thumbnail parity comparison status={status}."
|
|
)
|
|
|
|
return {
|
|
"workflow_run_id": None,
|
|
"workflow_def_id": None,
|
|
"order_line_id": None,
|
|
"execution_mode": "shadow",
|
|
"status": status,
|
|
"summary": summary,
|
|
"authoritative_output": authoritative_output,
|
|
"observer_output": observer_output,
|
|
"exact_match": exact_match,
|
|
"dimensions_match": dimensions_match,
|
|
"mean_pixel_delta": mean_pixel_delta,
|
|
}
|
|
|
|
|
|
def _parse_fraction(value: str | None) -> float | None:
|
|
if not value or value in {"0/0", "N/A"}:
|
|
return None
|
|
if "/" in value:
|
|
left, right = value.split("/", 1)
|
|
try:
|
|
numerator = float(left)
|
|
denominator = float(right)
|
|
except ValueError:
|
|
return None
|
|
if denominator == 0:
|
|
return None
|
|
return numerator / denominator
|
|
try:
|
|
return float(value)
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _ffprobe_video(video_path: Path) -> dict:
|
|
command = [
|
|
"ffprobe",
|
|
"-v",
|
|
"error",
|
|
"-select_streams",
|
|
"v:0",
|
|
"-show_entries",
|
|
"stream=codec_name,width,height,avg_frame_rate,nb_frames,duration:format=duration",
|
|
"-of",
|
|
"json",
|
|
str(video_path),
|
|
]
|
|
result = subprocess.run(command, capture_output=True, text=True, check=True)
|
|
payload = json.loads(result.stdout or "{}")
|
|
stream = (payload.get("streams") or [{}])[0]
|
|
format_info = payload.get("format") or {}
|
|
|
|
frame_count = stream.get("nb_frames")
|
|
try:
|
|
frame_count = int(frame_count) if frame_count not in (None, "", "N/A") else None
|
|
except ValueError:
|
|
frame_count = None
|
|
|
|
duration = stream.get("duration") or format_info.get("duration")
|
|
try:
|
|
duration_value = float(duration) if duration not in (None, "", "N/A") else None
|
|
except ValueError:
|
|
duration_value = None
|
|
|
|
width = stream.get("width")
|
|
height = stream.get("height")
|
|
try:
|
|
width = int(width) if width is not None else None
|
|
except ValueError:
|
|
width = None
|
|
try:
|
|
height = int(height) if height is not None else None
|
|
except ValueError:
|
|
height = None
|
|
|
|
return {
|
|
"codec_name": stream.get("codec_name"),
|
|
"width": width,
|
|
"height": height,
|
|
"avg_frame_rate_raw": stream.get("avg_frame_rate"),
|
|
"avg_frame_rate": _parse_fraction(stream.get("avg_frame_rate")),
|
|
"frame_count": frame_count,
|
|
"duration_s": duration_value,
|
|
}
|
|
|
|
|
|
def _extract_video_frame(video_path: Path, *, frame_index: int, target_path: Path) -> None:
|
|
command = [
|
|
"ffmpeg",
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-i",
|
|
str(video_path),
|
|
"-vf",
|
|
f"select=eq(n\\,{frame_index})",
|
|
"-vsync",
|
|
"vfr",
|
|
"-frames:v",
|
|
"1",
|
|
str(target_path),
|
|
]
|
|
subprocess.run(command, capture_output=True, text=True, check=True)
|
|
|
|
|
|
def _sample_frame_indexes(frame_count: int | None) -> list[int]:
|
|
if frame_count is None or frame_count <= 1:
|
|
return [0]
|
|
last_index = max(frame_count - 1, 0)
|
|
indexes = {0, last_index}
|
|
indexes.add(last_index // 2)
|
|
if last_index >= 4:
|
|
indexes.add(last_index // 4)
|
|
indexes.add((3 * last_index) // 4)
|
|
return sorted(indexes)
|
|
|
|
|
|
def _mean_or_none(values: list[float | None]) -> float | None:
|
|
filtered = [value for value in values if value is not None]
|
|
if not filtered:
|
|
return None
|
|
return sum(filtered) / len(filtered)
|
|
|
|
|
|
def _build_manual_video_comparison(
|
|
authoritative_bytes: bytes,
|
|
observer_bytes: bytes,
|
|
*,
|
|
authoritative_path: str,
|
|
authoritative_mime_type: str | None,
|
|
observer_path: str,
|
|
observer_mime_type: str | None,
|
|
) -> dict:
|
|
authoritative_output = _build_byte_artifact(
|
|
authoritative_bytes,
|
|
path=authoritative_path,
|
|
mime_type=authoritative_mime_type,
|
|
)
|
|
observer_output = _build_byte_artifact(
|
|
observer_bytes,
|
|
path=observer_path,
|
|
mime_type=observer_mime_type,
|
|
)
|
|
exact_match = (
|
|
authoritative_output["sha256"] == observer_output["sha256"]
|
|
if authoritative_output["exists"] and observer_output["exists"]
|
|
else None
|
|
)
|
|
if exact_match is True:
|
|
return {
|
|
"workflow_run_id": None,
|
|
"workflow_def_id": None,
|
|
"order_line_id": None,
|
|
"execution_mode": "shadow",
|
|
"status": "ready",
|
|
"summary": "Manual turntable parity comparison completed (byte-identical MP4).",
|
|
"authoritative_output": authoritative_output,
|
|
"observer_output": observer_output,
|
|
"exact_match": True,
|
|
"dimensions_match": True,
|
|
"mean_pixel_delta": 0.0,
|
|
"video_metadata": {},
|
|
"frame_samples": [],
|
|
}
|
|
|
|
with tempfile.TemporaryDirectory(prefix="hartomat-turntable-parity-") as temp_dir_raw:
|
|
temp_dir = Path(temp_dir_raw)
|
|
authoritative_video_path = temp_dir / "authoritative.mp4"
|
|
observer_video_path = temp_dir / "observer.mp4"
|
|
authoritative_video_path.write_bytes(authoritative_bytes)
|
|
observer_video_path.write_bytes(observer_bytes)
|
|
|
|
authoritative_video = _ffprobe_video(authoritative_video_path)
|
|
observer_video = _ffprobe_video(observer_video_path)
|
|
dimensions_match = (
|
|
authoritative_video.get("width") == observer_video.get("width")
|
|
and authoritative_video.get("height") == observer_video.get("height")
|
|
and authoritative_video.get("width") is not None
|
|
and authoritative_video.get("height") is not None
|
|
)
|
|
frame_count_match = authoritative_video.get("frame_count") == observer_video.get("frame_count")
|
|
|
|
sample_count_source = authoritative_video.get("frame_count")
|
|
if sample_count_source is None:
|
|
sample_count_source = observer_video.get("frame_count")
|
|
frame_indexes = _sample_frame_indexes(sample_count_source)
|
|
|
|
frame_samples: list[dict] = []
|
|
sample_deltas: list[float | None] = []
|
|
sample_dimensions: list[bool | None] = []
|
|
for frame_index in frame_indexes:
|
|
authoritative_frame_path = temp_dir / f"authoritative-{frame_index:04d}.png"
|
|
observer_frame_path = temp_dir / f"observer-{frame_index:04d}.png"
|
|
_extract_video_frame(authoritative_video_path, frame_index=frame_index, target_path=authoritative_frame_path)
|
|
_extract_video_frame(observer_video_path, frame_index=frame_index, target_path=observer_frame_path)
|
|
authoritative_frame = authoritative_frame_path.read_bytes()
|
|
observer_frame = observer_frame_path.read_bytes()
|
|
frame_dimensions_match, frame_delta = _compute_image_delta(authoritative_frame, observer_frame)
|
|
frame_exact_match = hashlib.sha256(authoritative_frame).hexdigest() == hashlib.sha256(observer_frame).hexdigest()
|
|
if frame_exact_match:
|
|
frame_dimensions_match = True
|
|
frame_delta = 0.0
|
|
sample_deltas.append(frame_delta)
|
|
sample_dimensions.append(frame_dimensions_match)
|
|
frame_samples.append(
|
|
{
|
|
"frame_index": frame_index,
|
|
"exact_match": frame_exact_match,
|
|
"dimensions_match": frame_dimensions_match,
|
|
"mean_pixel_delta": frame_delta,
|
|
}
|
|
)
|
|
|
|
mean_pixel_delta = _mean_or_none(sample_deltas)
|
|
sampled_dimensions_match = all(value is not False for value in sample_dimensions) if sample_dimensions else None
|
|
|
|
status = "ready"
|
|
if not authoritative_output["exists"]:
|
|
status = "missing_authoritative"
|
|
elif not observer_output["exists"]:
|
|
status = "missing_observer"
|
|
elif dimensions_match is False or sampled_dimensions_match is False:
|
|
status = "dimension_mismatch"
|
|
elif frame_count_match is False:
|
|
status = "frame_count_mismatch"
|
|
elif mean_pixel_delta is None:
|
|
status = "non_image_or_uncomparable"
|
|
|
|
summary = (
|
|
"Manual turntable parity comparison completed."
|
|
if status == "ready"
|
|
else f"Manual turntable parity comparison status={status}."
|
|
)
|
|
|
|
return {
|
|
"workflow_run_id": None,
|
|
"workflow_def_id": None,
|
|
"order_line_id": None,
|
|
"execution_mode": "shadow",
|
|
"status": status,
|
|
"summary": summary,
|
|
"authoritative_output": authoritative_output,
|
|
"observer_output": observer_output,
|
|
"exact_match": exact_match,
|
|
"dimensions_match": dimensions_match if sampled_dimensions_match is not False else False,
|
|
"mean_pixel_delta": mean_pixel_delta,
|
|
"video_metadata": {
|
|
"authoritative": authoritative_video,
|
|
"observer": observer_video,
|
|
"frame_count_match": frame_count_match,
|
|
},
|
|
"frame_samples": frame_samples,
|
|
}
|
|
|
|
|
|
def _wait_for_turntable_asset(
|
|
harness,
|
|
client,
|
|
*,
|
|
order_line_id: str,
|
|
workflow_run_id: str,
|
|
timeout_seconds: int,
|
|
) -> tuple[dict, dict]:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
assets = harness.list_media_assets(
|
|
client,
|
|
order_line_id=order_line_id,
|
|
asset_type="turntable",
|
|
)
|
|
observer_asset = None
|
|
authoritative_asset = None
|
|
for asset in assets:
|
|
if str(asset.get("workflow_run_id")) == workflow_run_id:
|
|
observer_asset = asset
|
|
elif authoritative_asset is None and asset.get("download_url"):
|
|
authoritative_asset = asset
|
|
if authoritative_asset is not None and observer_asset is not None:
|
|
return authoritative_asset, observer_asset
|
|
time.sleep(2)
|
|
raise RuntimeError(
|
|
f"Timed out waiting for turntable assets for order_line={order_line_id} workflow_run={workflow_run_id}"
|
|
)
|
|
|
|
|
|
def _unsupported_output_type_reason(output_type: dict, templates: list[dict]) -> str | None:
|
|
artifact_kind = output_type.get("artifact_kind")
|
|
if artifact_kind == "turntable_video" and (output_type.get("render_settings") or {}).get("cinematic"):
|
|
return "Cinematic legacy path has no graph-equivalent node today; turntable parity is not comparable."
|
|
if artifact_kind == "blend_asset":
|
|
return "No real legacy blend-export parity path exists today."
|
|
if artifact_kind == "turntable_video" and not templates:
|
|
return "Turntable parity requires a real render template link."
|
|
return None
|
|
|
|
|
|
def _find_thumbnail_asset_for_run(
|
|
client,
|
|
*,
|
|
cad_file_id: str,
|
|
workflow_run_id: str,
|
|
timeout_seconds: int,
|
|
) -> dict | None:
|
|
deadline = time.time() + timeout_seconds
|
|
while time.time() < deadline:
|
|
resp = client.get(
|
|
"/media",
|
|
params={
|
|
"cad_file_id": cad_file_id,
|
|
"asset_type": "thumbnail",
|
|
"limit": 50,
|
|
},
|
|
)
|
|
if resp.status_code == 200:
|
|
for asset in resp.json():
|
|
if str(asset.get("workflow_run_id")) == workflow_run_id:
|
|
return asset
|
|
time.sleep(2)
|
|
return None
|
|
|
|
|
|
def _run_single_output_type(harness, client, *, cad_file_id: str, output_type: dict) -> dict:
|
|
templates = harness.render_template_candidates_for_output_type(
|
|
harness.get_render_templates(client),
|
|
output_type["id"],
|
|
)
|
|
template_backed = bool(templates)
|
|
snapshot = harness.build_output_type_workflow_snapshot(output_type)
|
|
workflow = None
|
|
product_id = harness.get_or_create_test_product(client, cad_file_id)
|
|
if not product_id:
|
|
raise RuntimeError(f"Could not resolve product for CAD file {cad_file_id}")
|
|
|
|
try:
|
|
if template_backed:
|
|
resources = harness.ensure_template_parity_shadow_resources(
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
bound_output_type = resources["output_type"]
|
|
workflow = resources["workflow"]
|
|
else:
|
|
workflow = _ensure_shadow_probe_workflow(
|
|
harness,
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
resp = client.patch(
|
|
f"/output-types/{output_type['id']}",
|
|
json=harness.build_output_type_workflow_link_payload(
|
|
workflow_definition_id=workflow["id"],
|
|
execution_mode="shadow",
|
|
),
|
|
)
|
|
if resp.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Output type shadow link failed for {output_type['name']}: "
|
|
f"{resp.status_code} {resp.text[:400]}"
|
|
)
|
|
bound_output_type = resp.json()
|
|
|
|
order = harness.create_test_order(
|
|
client,
|
|
product_id=product_id,
|
|
output_type_ids=[bound_output_type["id"]],
|
|
test_label=f"Still Parity Matrix [{bound_output_type['name']}]",
|
|
)
|
|
if order is None:
|
|
raise RuntimeError(f"Order creation failed for {bound_output_type['name']}")
|
|
|
|
lines = order.get("lines", [])
|
|
if len(lines) != 1:
|
|
raise RuntimeError(
|
|
f"Expected exactly one order line for {bound_output_type['name']}, got {len(lines)}"
|
|
)
|
|
line_id = lines[0]["id"]
|
|
|
|
resp_preflight = client.get(
|
|
f"/workflows/{workflow['id']}/preflight",
|
|
params={"context_id": line_id},
|
|
)
|
|
if resp_preflight.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Workflow preflight failed for {bound_output_type['name']}: "
|
|
f"{resp_preflight.status_code} {resp_preflight.text[:400]}"
|
|
)
|
|
|
|
preflight = resp_preflight.json()
|
|
if not preflight.get("graph_dispatch_allowed"):
|
|
raise RuntimeError(
|
|
f"Workflow preflight blocked dispatch for {bound_output_type['name']}: "
|
|
f"{preflight.get('summary')}"
|
|
)
|
|
|
|
success = harness._submit_and_wait(
|
|
client,
|
|
order,
|
|
[bound_output_type["id"]],
|
|
use_graph_dispatch=False,
|
|
)
|
|
if not success:
|
|
raise RuntimeError(f"Render dispatch did not complete successfully for {bound_output_type['name']}")
|
|
|
|
workflow_run = harness.wait_for_workflow_run(
|
|
client,
|
|
workflow_id=workflow["id"],
|
|
line_id=line_id,
|
|
timeout_seconds=harness.WORKFLOW_RUN_TIMEOUT_SECONDS,
|
|
terminal_only=True,
|
|
)
|
|
if workflow_run is None:
|
|
raise RuntimeError(f"Workflow run not found for {bound_output_type['name']}")
|
|
|
|
comparison = harness.wait_for_workflow_comparison(
|
|
client,
|
|
workflow_run_id=workflow_run["id"],
|
|
timeout_seconds=harness.WORKFLOW_COMPARISON_TIMEOUT_SECONDS,
|
|
)
|
|
if comparison is None:
|
|
raise RuntimeError(f"Workflow comparison did not stabilize for {bound_output_type['name']}")
|
|
|
|
rollout_gate = harness.evaluate_rollout_gate_from_comparison(comparison)
|
|
template_node = harness._node_result_by_name(workflow_run, "template")
|
|
render_node = harness._node_result_by_name(workflow_run, "render")
|
|
|
|
return {
|
|
"output_type": {
|
|
"id": bound_output_type["id"],
|
|
"name": bound_output_type["name"],
|
|
"format": bound_output_type.get("output_format"),
|
|
"transparent_bg": bound_output_type.get("transparent_bg"),
|
|
"artifact_kind": bound_output_type.get("artifact_kind"),
|
|
"render_settings": bound_output_type.get("render_settings"),
|
|
"invocation_overrides": bound_output_type.get("invocation_overrides"),
|
|
},
|
|
"template_backed": template_backed,
|
|
"templates": [
|
|
{
|
|
"name": template.get("name"),
|
|
"blend_file_path": template.get("blend_file_path"),
|
|
"lighting_only": template.get("lighting_only"),
|
|
"shadow_catcher_enabled": template.get("shadow_catcher_enabled"),
|
|
"target_collection": template.get("target_collection"),
|
|
}
|
|
for template in templates
|
|
],
|
|
"workflow": {
|
|
"id": workflow.get("id"),
|
|
"name": workflow.get("name"),
|
|
},
|
|
"preflight": {
|
|
"execution_mode": preflight.get("execution_mode"),
|
|
"context_kind": preflight.get("context_kind"),
|
|
"graph_dispatch_allowed": preflight.get("graph_dispatch_allowed"),
|
|
},
|
|
"order_line_id": line_id,
|
|
"workflow_run": {
|
|
"id": workflow_run.get("id"),
|
|
"status": workflow_run.get("status"),
|
|
"execution_mode": workflow_run.get("execution_mode"),
|
|
},
|
|
"template_resolution": template_node.get("output") if template_node else None,
|
|
"render_output": render_node.get("output") if render_node else None,
|
|
"comparison": comparison,
|
|
"rollout_gate": rollout_gate,
|
|
}
|
|
finally:
|
|
harness.restore_output_type_workflow_snapshot(
|
|
client,
|
|
output_type_id=output_type["id"],
|
|
snapshot=snapshot,
|
|
)
|
|
|
|
|
|
def _run_single_turntable_output_type_with_product(harness, client, *, product_id: str, output_type: dict) -> dict:
|
|
templates = harness.render_template_candidates_for_output_type(
|
|
harness.get_render_templates(client),
|
|
output_type["id"],
|
|
)
|
|
unsupported_reason = _unsupported_output_type_reason(output_type, templates)
|
|
if unsupported_reason is not None:
|
|
return {
|
|
"output_type": {
|
|
"id": output_type["id"],
|
|
"name": output_type["name"],
|
|
"format": output_type.get("output_format"),
|
|
"transparent_bg": output_type.get("transparent_bg"),
|
|
"artifact_kind": output_type.get("artifact_kind"),
|
|
"render_settings": output_type.get("render_settings"),
|
|
"invocation_overrides": output_type.get("invocation_overrides"),
|
|
},
|
|
"template_backed": bool(templates),
|
|
"templates": [
|
|
{
|
|
"name": template.get("name"),
|
|
"blend_file_path": template.get("blend_file_path"),
|
|
"lighting_only": template.get("lighting_only"),
|
|
"shadow_catcher_enabled": template.get("shadow_catcher_enabled"),
|
|
"target_collection": template.get("target_collection"),
|
|
}
|
|
for template in templates
|
|
],
|
|
"workflow": None,
|
|
"preflight": None,
|
|
"workflow_run": None,
|
|
"comparison": {
|
|
"status": "unsupported_parity_path",
|
|
"summary": unsupported_reason,
|
|
"authoritative_output": {"exists": False},
|
|
"observer_output": {"exists": False},
|
|
"exact_match": None,
|
|
"dimensions_match": None,
|
|
"mean_pixel_delta": None,
|
|
},
|
|
"rollout_gate": {
|
|
"verdict": "fail",
|
|
"ready": False,
|
|
"reasons": [unsupported_reason],
|
|
},
|
|
"skipped": True,
|
|
}
|
|
|
|
snapshot = harness.build_output_type_workflow_snapshot(output_type)
|
|
workflow = None
|
|
try:
|
|
workflow = _ensure_shadow_probe_workflow(
|
|
harness,
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
resp = client.patch(
|
|
f"/output-types/{output_type['id']}",
|
|
json=harness.build_output_type_workflow_link_payload(
|
|
workflow_definition_id=workflow["id"],
|
|
execution_mode="shadow",
|
|
),
|
|
)
|
|
if resp.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Output type shadow link failed for {output_type['name']}: "
|
|
f"{resp.status_code} {resp.text[:400]}"
|
|
)
|
|
bound_output_type = resp.json()
|
|
|
|
order = harness.create_test_order(
|
|
client,
|
|
product_id=product_id,
|
|
output_type_ids=[bound_output_type["id"]],
|
|
test_label=f"Turntable Parity Matrix [{bound_output_type['name']}]",
|
|
)
|
|
if order is None:
|
|
raise RuntimeError(f"Order creation failed for {bound_output_type['name']}")
|
|
|
|
lines = order.get("lines", [])
|
|
if len(lines) != 1:
|
|
raise RuntimeError(
|
|
f"Expected exactly one order line for {bound_output_type['name']}, got {len(lines)}"
|
|
)
|
|
line_id = lines[0]["id"]
|
|
|
|
resp_preflight = client.get(
|
|
f"/workflows/{workflow['id']}/preflight",
|
|
params={"context_id": line_id},
|
|
)
|
|
if resp_preflight.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Workflow preflight failed for {bound_output_type['name']}: "
|
|
f"{resp_preflight.status_code} {resp_preflight.text[:400]}"
|
|
)
|
|
|
|
preflight = resp_preflight.json()
|
|
if not preflight.get("graph_dispatch_allowed"):
|
|
raise RuntimeError(
|
|
f"Workflow preflight blocked dispatch for {bound_output_type['name']}: "
|
|
f"{preflight.get('summary')}"
|
|
)
|
|
|
|
success = harness._submit_and_wait(
|
|
client,
|
|
order,
|
|
[bound_output_type["id"]],
|
|
use_graph_dispatch=False,
|
|
)
|
|
if not success:
|
|
raise RuntimeError(f"Render dispatch did not complete successfully for {bound_output_type['name']}")
|
|
|
|
workflow_run = harness.wait_for_workflow_run(
|
|
client,
|
|
workflow_id=workflow["id"],
|
|
line_id=line_id,
|
|
timeout_seconds=harness.WORKFLOW_RUN_TIMEOUT_SECONDS,
|
|
terminal_only=True,
|
|
)
|
|
if workflow_run is None:
|
|
raise RuntimeError(f"Workflow run not found for {bound_output_type['name']}")
|
|
|
|
comparison_seed = harness.wait_for_workflow_comparison(
|
|
client,
|
|
workflow_run_id=workflow_run["id"],
|
|
timeout_seconds=harness.WORKFLOW_COMPARISON_TIMEOUT_SECONDS,
|
|
)
|
|
if comparison_seed is None:
|
|
raise RuntimeError(f"Workflow comparison did not stabilize for {bound_output_type['name']}")
|
|
|
|
authoritative_asset, observer_asset = _wait_for_turntable_asset(
|
|
harness,
|
|
client,
|
|
order_line_id=line_id,
|
|
workflow_run_id=str(workflow_run["id"]),
|
|
timeout_seconds=harness.WORKFLOW_COMPARISON_TIMEOUT_SECONDS,
|
|
)
|
|
authoritative_download_url = authoritative_asset.get("download_url")
|
|
observer_download_url = observer_asset.get("download_url")
|
|
if not authoritative_download_url or not observer_download_url:
|
|
raise RuntimeError("Turntable media asset download URL missing")
|
|
|
|
authoritative_bytes, authoritative_mime_type = _download_bytes(client, authoritative_download_url)
|
|
observer_bytes, observer_mime_type = _download_bytes(client, observer_download_url)
|
|
comparison = _build_manual_video_comparison(
|
|
authoritative_bytes,
|
|
observer_bytes,
|
|
authoritative_path=authoritative_download_url,
|
|
authoritative_mime_type=authoritative_mime_type,
|
|
observer_path=observer_download_url,
|
|
observer_mime_type=observer_mime_type,
|
|
)
|
|
comparison["workflow_run_id"] = workflow_run["id"]
|
|
comparison["workflow_def_id"] = workflow["id"]
|
|
comparison["order_line_id"] = line_id
|
|
comparison["comparison_seed"] = comparison_seed
|
|
rollout_gate = harness.evaluate_rollout_gate_from_comparison(comparison)
|
|
template_node = harness._node_result_by_name(workflow_run, "template")
|
|
render_node = harness._node_result_by_name(workflow_run, "turntable")
|
|
|
|
return {
|
|
"output_type": {
|
|
"id": bound_output_type["id"],
|
|
"name": bound_output_type["name"],
|
|
"format": bound_output_type.get("output_format"),
|
|
"transparent_bg": bound_output_type.get("transparent_bg"),
|
|
"artifact_kind": bound_output_type.get("artifact_kind"),
|
|
"render_settings": bound_output_type.get("render_settings"),
|
|
"invocation_overrides": bound_output_type.get("invocation_overrides"),
|
|
},
|
|
"template_backed": bool(templates),
|
|
"templates": [
|
|
{
|
|
"name": template.get("name"),
|
|
"blend_file_path": template.get("blend_file_path"),
|
|
"lighting_only": template.get("lighting_only"),
|
|
"shadow_catcher_enabled": template.get("shadow_catcher_enabled"),
|
|
"target_collection": template.get("target_collection"),
|
|
"camera_orbit": template.get("camera_orbit"),
|
|
}
|
|
for template in templates
|
|
],
|
|
"workflow": {
|
|
"id": workflow.get("id"),
|
|
"name": workflow.get("name"),
|
|
},
|
|
"preflight": {
|
|
"execution_mode": preflight.get("execution_mode"),
|
|
"context_kind": preflight.get("context_kind"),
|
|
"graph_dispatch_allowed": preflight.get("graph_dispatch_allowed"),
|
|
},
|
|
"order_line_id": line_id,
|
|
"workflow_run": {
|
|
"id": workflow_run.get("id"),
|
|
"status": workflow_run.get("status"),
|
|
"execution_mode": workflow_run.get("execution_mode"),
|
|
},
|
|
"template_resolution": template_node.get("output") if template_node else None,
|
|
"render_output": render_node.get("output") if render_node else None,
|
|
"comparison": comparison,
|
|
"rollout_gate": rollout_gate,
|
|
}
|
|
finally:
|
|
harness.restore_output_type_workflow_snapshot(
|
|
client,
|
|
output_type_id=output_type["id"],
|
|
snapshot=snapshot,
|
|
)
|
|
|
|
|
|
def _run_single_thumbnail_output_type(harness, client, *, cad_file_id: str, output_type: dict) -> dict:
|
|
workflow = _ensure_shadow_probe_workflow(
|
|
harness,
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
|
|
resp_preflight = client.get(
|
|
f"/workflows/{workflow['id']}/preflight",
|
|
params={"context_id": cad_file_id},
|
|
)
|
|
if resp_preflight.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Workflow preflight failed for {output_type['name']}: "
|
|
f"{resp_preflight.status_code} {resp_preflight.text[:400]}"
|
|
)
|
|
|
|
preflight = resp_preflight.json()
|
|
if not preflight.get("graph_dispatch_allowed"):
|
|
raise RuntimeError(
|
|
f"Workflow preflight blocked dispatch for {output_type['name']}: "
|
|
f"{preflight.get('summary')}"
|
|
)
|
|
|
|
resp_dispatch = client.post(
|
|
f"/workflows/{workflow['id']}/dispatch",
|
|
params={"context_id": cad_file_id},
|
|
)
|
|
if resp_dispatch.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Workflow dispatch failed for {output_type['name']}: "
|
|
f"{resp_dispatch.status_code} {resp_dispatch.text[:400]}"
|
|
)
|
|
|
|
dispatch_payload = resp_dispatch.json()
|
|
workflow_run_id = str(dispatch_payload["workflow_run"]["id"])
|
|
workflow_run = _wait_for_workflow_run_id(
|
|
harness,
|
|
client,
|
|
workflow_id=str(workflow["id"]),
|
|
workflow_run_id=workflow_run_id,
|
|
timeout_seconds=harness.WORKFLOW_RUN_TIMEOUT_SECONDS,
|
|
)
|
|
if workflow_run is None:
|
|
raise RuntimeError(f"Workflow run did not reach terminal status for {output_type['name']}")
|
|
|
|
asset = _find_thumbnail_asset_for_run(
|
|
client,
|
|
cad_file_id=cad_file_id,
|
|
workflow_run_id=workflow_run_id,
|
|
timeout_seconds=harness.WORKFLOW_COMPARISON_TIMEOUT_SECONDS,
|
|
)
|
|
if asset is None:
|
|
raise RuntimeError(f"Shadow thumbnail asset not found for workflow run {workflow_run_id}")
|
|
|
|
authoritative_bytes, authoritative_mime_type = _download_bytes(
|
|
client,
|
|
f"/api/cad/{cad_file_id}/thumbnail",
|
|
)
|
|
observer_path = asset.get("download_url")
|
|
if not observer_path:
|
|
raise RuntimeError(f"Thumbnail asset {asset.get('id')} is missing download_url")
|
|
observer_bytes, observer_mime_type = _download_bytes(client, observer_path)
|
|
|
|
comparison = _build_manual_comparison(
|
|
authoritative_bytes,
|
|
observer_bytes,
|
|
authoritative_path=f"/api/cad/{cad_file_id}/thumbnail",
|
|
authoritative_mime_type=authoritative_mime_type,
|
|
observer_path=observer_path,
|
|
observer_mime_type=observer_mime_type,
|
|
)
|
|
comparison["workflow_run_id"] = workflow_run_id
|
|
comparison["workflow_def_id"] = workflow.get("id")
|
|
rollout_gate = harness.evaluate_rollout_gate_from_comparison(comparison)
|
|
|
|
return {
|
|
"output_type": {
|
|
"id": output_type["id"],
|
|
"name": output_type["name"],
|
|
"format": output_type.get("output_format"),
|
|
"transparent_bg": output_type.get("transparent_bg"),
|
|
"render_settings": output_type.get("render_settings"),
|
|
"invocation_overrides": output_type.get("invocation_overrides"),
|
|
"artifact_kind": output_type.get("artifact_kind"),
|
|
},
|
|
"template_backed": False,
|
|
"templates": [],
|
|
"workflow": {
|
|
"id": workflow.get("id"),
|
|
"name": workflow.get("name"),
|
|
},
|
|
"preflight": {
|
|
"execution_mode": preflight.get("execution_mode"),
|
|
"context_kind": preflight.get("context_kind"),
|
|
"graph_dispatch_allowed": preflight.get("graph_dispatch_allowed"),
|
|
},
|
|
"cad_file_id": cad_file_id,
|
|
"thumbnail_asset_id": asset.get("id"),
|
|
"workflow_run": {
|
|
"id": workflow_run.get("id"),
|
|
"status": workflow_run.get("status"),
|
|
"execution_mode": workflow_run.get("execution_mode"),
|
|
},
|
|
"comparison": comparison,
|
|
"rollout_gate": rollout_gate,
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
harness = _load_harness()
|
|
|
|
parser = argparse.ArgumentParser(description="Compare live legacy vs shadow renders for real Blender parity output types")
|
|
parser.add_argument("--host", default=os.environ.get("TEST_HOST", "http://localhost:8888"))
|
|
parser.add_argument("--email", default=os.environ.get("TEST_EMAIL", "admin@hartomat.com"))
|
|
parser.add_argument("--password", default=os.environ.get("TEST_PASSWORD", "Admin1234!"))
|
|
parser.add_argument("--step", default=str(harness.SAMPLE_STEP))
|
|
parser.add_argument("--product-id", default=None, help="Existing product id to use instead of creating/reusing one from --step")
|
|
parser.add_argument("--cad-file-id", default=None, help="CAD file id for --product-id; inferred from /api/products when omitted")
|
|
parser.add_argument("--include-generated", action="store_true")
|
|
parser.add_argument(
|
|
"--artifact-kind",
|
|
action="append",
|
|
default=[],
|
|
help="Restrict to artifact kinds like still_image, thumbnail_image, turntable_video; repeatable",
|
|
)
|
|
parser.add_argument("--output", default=None, help="Optional path for JSON report")
|
|
parser.add_argument("--only", action="append", default=[], help="Only run the named output type; repeatable")
|
|
args = parser.parse_args()
|
|
|
|
client = harness.APIClient(args.host, args.email, args.password)
|
|
health_ok = harness.test_health(client)
|
|
if not health_ok:
|
|
raise RuntimeError("Render stack health check failed")
|
|
|
|
if args.product_id:
|
|
product_resp = client.get(f"/products/{args.product_id}")
|
|
if product_resp.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Could not load product {args.product_id}: {product_resp.status_code} {product_resp.text[:400]}"
|
|
)
|
|
product = product_resp.json()
|
|
cad_file_id = args.cad_file_id or product.get("cad_file_id")
|
|
if not cad_file_id:
|
|
raise RuntimeError(f"Product {args.product_id} has no cad_file_id")
|
|
fixed_product_id = args.product_id
|
|
print(f"Using existing product {fixed_product_id} with cad_file_id {cad_file_id}", flush=True)
|
|
else:
|
|
cad_file_id = harness.test_step_upload(client, Path(args.step))
|
|
if not cad_file_id:
|
|
raise RuntimeError("STEP upload / CAD processing failed")
|
|
fixed_product_id = None
|
|
|
|
output_types = [
|
|
ot
|
|
for ot in harness.get_output_types(client, include_inactive=True)
|
|
if _is_real_blender_output_type(ot, include_generated=args.include_generated)
|
|
]
|
|
if args.artifact_kind:
|
|
wanted_artifact_kinds = set(args.artifact_kind)
|
|
output_types = [ot for ot in output_types if ot.get("artifact_kind") in wanted_artifact_kinds]
|
|
if args.only:
|
|
wanted = set(args.only)
|
|
output_types = [ot for ot in output_types if ot.get("name") in wanted]
|
|
|
|
output_types.sort(key=lambda item: item.get("name") or "")
|
|
if not output_types:
|
|
raise RuntimeError("No eligible still-image or thumbnail output types found")
|
|
|
|
results: list[dict] = []
|
|
for output_type in output_types:
|
|
print(f"\n=== {output_type['name']} ===", flush=True)
|
|
artifact_kind = output_type.get("artifact_kind")
|
|
if artifact_kind == "thumbnail_image":
|
|
result = _run_single_thumbnail_output_type(
|
|
harness,
|
|
client,
|
|
cad_file_id=cad_file_id,
|
|
output_type=output_type,
|
|
)
|
|
elif artifact_kind == "turntable_video":
|
|
if fixed_product_id is None:
|
|
product_id = harness.get_or_create_test_product(client, cad_file_id)
|
|
if not product_id:
|
|
raise RuntimeError(f"Could not resolve product for CAD file {cad_file_id}")
|
|
else:
|
|
product_id = fixed_product_id
|
|
result = _run_single_turntable_output_type_with_product(
|
|
harness,
|
|
client,
|
|
product_id=product_id,
|
|
output_type=output_type,
|
|
)
|
|
else:
|
|
if fixed_product_id is None:
|
|
result = _run_single_output_type(
|
|
harness,
|
|
client,
|
|
cad_file_id=cad_file_id,
|
|
output_type=output_type,
|
|
)
|
|
else:
|
|
result = _run_single_output_type_with_product(
|
|
harness,
|
|
client,
|
|
product_id=fixed_product_id,
|
|
output_type=output_type,
|
|
)
|
|
results.append(result)
|
|
summary = {
|
|
"name": result["output_type"]["name"],
|
|
"artifact_kind": result["output_type"].get("artifact_kind"),
|
|
"template_backed": result["template_backed"],
|
|
"skipped": bool(result.get("skipped")),
|
|
"exact_match": result["comparison"].get("exact_match"),
|
|
"status": result["comparison"].get("status"),
|
|
"mean_pixel_delta": result["comparison"].get("mean_pixel_delta"),
|
|
"rollout_verdict": result["rollout_gate"].get("verdict"),
|
|
"workflow_run_id": result["workflow_run"]["id"] if result.get("workflow_run") else None,
|
|
"order_line_id": result.get("order_line_id"),
|
|
"cad_file_id": result.get("cad_file_id"),
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
|
|
|
report = {
|
|
"host": args.host,
|
|
"cad_file_id": cad_file_id,
|
|
"product_id": fixed_product_id,
|
|
"results": results,
|
|
}
|
|
|
|
if args.output:
|
|
output_path = Path(args.output)
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
output_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
|
|
print(f"\nWrote report to {output_path}", flush=True)
|
|
|
|
overall = {
|
|
"total": len(results),
|
|
"exact_match": sum(1 for item in results if item["comparison"].get("exact_match") is True),
|
|
"pass": sum(1 for item in results if item["rollout_gate"].get("verdict") == "pass"),
|
|
"warn": sum(1 for item in results if item["rollout_gate"].get("verdict") == "warn"),
|
|
"fail": sum(1 for item in results if item["rollout_gate"].get("verdict") == "fail"),
|
|
"skipped": sum(1 for item in results if item.get("skipped")),
|
|
}
|
|
print("\n=== Overall ===", flush=True)
|
|
print(json.dumps(overall, ensure_ascii=False), flush=True)
|
|
return 0
|
|
|
|
|
|
def _run_single_output_type_with_product(harness, client, *, product_id: str, output_type: dict) -> dict:
|
|
templates = harness.render_template_candidates_for_output_type(
|
|
harness.get_render_templates(client),
|
|
output_type["id"],
|
|
)
|
|
template_backed = bool(templates)
|
|
snapshot = harness.build_output_type_workflow_snapshot(output_type)
|
|
workflow = None
|
|
|
|
try:
|
|
if template_backed:
|
|
resources = harness.ensure_template_parity_shadow_resources(
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
bound_output_type = resources["output_type"]
|
|
workflow = resources["workflow"]
|
|
else:
|
|
workflow = _ensure_shadow_probe_workflow(
|
|
harness,
|
|
client,
|
|
output_type=output_type,
|
|
)
|
|
resp = client.patch(
|
|
f"/output-types/{output_type['id']}",
|
|
json=harness.build_output_type_workflow_link_payload(
|
|
workflow_definition_id=workflow["id"],
|
|
execution_mode="shadow",
|
|
),
|
|
)
|
|
if resp.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Output type shadow link failed for {output_type['name']}: "
|
|
f"{resp.status_code} {resp.text[:400]}"
|
|
)
|
|
bound_output_type = resp.json()
|
|
|
|
order = harness.create_test_order(
|
|
client,
|
|
product_id=product_id,
|
|
output_type_ids=[bound_output_type["id"]],
|
|
test_label=f"Still Parity Matrix [{bound_output_type['name']}]",
|
|
)
|
|
if order is None:
|
|
raise RuntimeError(f"Order creation failed for {bound_output_type['name']}")
|
|
|
|
lines = order.get("lines", [])
|
|
if len(lines) != 1:
|
|
raise RuntimeError(
|
|
f"Expected exactly one order line for {bound_output_type['name']}, got {len(lines)}"
|
|
)
|
|
line_id = lines[0]["id"]
|
|
|
|
resp_preflight = client.get(
|
|
f"/workflows/{workflow['id']}/preflight",
|
|
params={"context_id": line_id},
|
|
)
|
|
if resp_preflight.status_code != 200:
|
|
raise RuntimeError(
|
|
f"Workflow preflight failed for {bound_output_type['name']}: "
|
|
f"{resp_preflight.status_code} {resp_preflight.text[:400]}"
|
|
)
|
|
|
|
preflight = resp_preflight.json()
|
|
if not preflight.get("graph_dispatch_allowed"):
|
|
raise RuntimeError(
|
|
f"Workflow preflight blocked dispatch for {bound_output_type['name']}: "
|
|
f"{preflight.get('summary')}"
|
|
)
|
|
|
|
success = harness._submit_and_wait(
|
|
client,
|
|
order,
|
|
[bound_output_type["id"]],
|
|
use_graph_dispatch=False,
|
|
)
|
|
if not success:
|
|
raise RuntimeError(f"Render dispatch did not complete successfully for {bound_output_type['name']}")
|
|
|
|
workflow_run = harness.wait_for_workflow_run(
|
|
client,
|
|
workflow_id=workflow["id"],
|
|
line_id=line_id,
|
|
timeout_seconds=harness.WORKFLOW_RUN_TIMEOUT_SECONDS,
|
|
terminal_only=True,
|
|
)
|
|
if workflow_run is None:
|
|
raise RuntimeError(f"Workflow run not found for {bound_output_type['name']}")
|
|
|
|
comparison = harness.wait_for_workflow_comparison(
|
|
client,
|
|
workflow_run_id=workflow_run["id"],
|
|
timeout_seconds=harness.WORKFLOW_COMPARISON_TIMEOUT_SECONDS,
|
|
)
|
|
if comparison is None:
|
|
raise RuntimeError(f"Workflow comparison did not stabilize for {bound_output_type['name']}")
|
|
|
|
rollout_gate = harness.evaluate_rollout_gate_from_comparison(comparison)
|
|
template_node = harness._node_result_by_name(workflow_run, "template")
|
|
render_node = harness._node_result_by_name(workflow_run, "render")
|
|
|
|
return {
|
|
"output_type": {
|
|
"id": bound_output_type["id"],
|
|
"name": bound_output_type["name"],
|
|
"format": bound_output_type.get("output_format"),
|
|
"transparent_bg": bound_output_type.get("transparent_bg"),
|
|
"artifact_kind": bound_output_type.get("artifact_kind"),
|
|
"render_settings": bound_output_type.get("render_settings"),
|
|
"invocation_overrides": bound_output_type.get("invocation_overrides"),
|
|
},
|
|
"template_backed": template_backed,
|
|
"templates": [
|
|
{
|
|
"name": template.get("name"),
|
|
"blend_file_path": template.get("blend_file_path"),
|
|
"lighting_only": template.get("lighting_only"),
|
|
"shadow_catcher_enabled": template.get("shadow_catcher_enabled"),
|
|
"target_collection": template.get("target_collection"),
|
|
}
|
|
for template in templates
|
|
],
|
|
"workflow": {
|
|
"id": workflow.get("id"),
|
|
"name": workflow.get("name"),
|
|
},
|
|
"preflight": {
|
|
"execution_mode": preflight.get("execution_mode"),
|
|
"context_kind": preflight.get("context_kind"),
|
|
"graph_dispatch_allowed": preflight.get("graph_dispatch_allowed"),
|
|
},
|
|
"order_line_id": line_id,
|
|
"workflow_run": {
|
|
"id": workflow_run.get("id"),
|
|
"status": workflow_run.get("status"),
|
|
"execution_mode": workflow_run.get("execution_mode"),
|
|
},
|
|
"template_resolution": template_node.get("output") if template_node else None,
|
|
"render_output": render_node.get("output") if render_node else None,
|
|
"comparison": comparison,
|
|
"rollout_gate": rollout_gate,
|
|
}
|
|
finally:
|
|
harness.restore_output_type_workflow_snapshot(
|
|
client,
|
|
output_type_id=output_type["id"],
|
|
snapshot=snapshot,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except KeyboardInterrupt:
|
|
raise SystemExit(130)
|