feat: add graph workflow fallback and retry metadata

2026-04-07 10:56:45 +02:00
parent c17b7d2e8f
commit f9d4da52b9
9 changed files with 473 additions and 39 deletions
@@ -220,6 +220,129 @@ async def test_dispatch_render_with_workflow_falls_back_when_workflow_runtime_pr
    assert runs == []


+@pytest.mark.asyncio
+async def test_dispatch_render_with_workflow_graph_mode_dispatches_supported_custom_workflow(
+    db,
+    admin_user,
+    monkeypatch,
+    tmp_path,
+):
+    _use_test_database(monkeypatch)
+    order_line = await _seed_renderable_order_line(db, admin_user, tmp_path)
+    workflow_definition = WorkflowDefinition(
+        name=f"Graph Workflow {uuid.uuid4().hex[:8]}",
+        output_type_id=order_line.output_type_id,
+        config={
+            "version": 1,
+            "ui": {"preset": "custom", "execution_mode": "graph"},
+            "nodes": [
+                {"id": "setup", "step": "order_line_setup", "params": {}},
+                {"id": "template", "step": "resolve_template", "params": {}},
+                {"id": "render", "step": "blender_still", "params": {"width": 1024, "height": 768}},
+            ],
+            "edges": [
+                {"from": "setup", "to": "template"},
+                {"from": "template", "to": "render"},
+            ],
+        },
+        is_active=True,
+    )
+    db.add(workflow_definition)
+    await db.flush()
+    output_type = await db.get(OutputType, order_line.output_type_id)
+    assert output_type is not None
+    output_type.workflow_definition_id = workflow_definition.id
+    await db.commit()
+
+    monkeypatch.setattr(
+        "app.tasks.celery_app.celery_app.send_task",
+        lambda task_name, args, kwargs: type("Result", (), {"id": "graph-task-1"})(),
+    )
+
+    result = dispatch_render_with_workflow(str(order_line.id))
+
+    await db.rollback()
+
+    run_result = await db.execute(
+        select(WorkflowRun)
+        .where(WorkflowRun.id == uuid.UUID(result["workflow_run_id"]))
+        .options(selectinload(WorkflowRun.node_results))
+    )
+    run = run_result.scalar_one()
+    node_results = {node_result.node_name: node_result for node_result in run.node_results}
+
+    assert result["backend"] == "workflow_graph"
+    assert result["execution_mode"] == "graph"
+    assert result["task_ids"] == ["graph-task-1"]
+    assert run.status == "pending"
+    assert node_results["setup"].status == "completed"
+    assert node_results["template"].status == "completed"
+    assert node_results["render"].status == "queued"
+
+
+@pytest.mark.asyncio
+async def test_dispatch_render_with_workflow_graph_mode_falls_back_to_legacy_on_graph_failure(
+    db,
+    admin_user,
+    monkeypatch,
+    tmp_path,
+):
+    _use_test_database(monkeypatch)
+    order_line = await _seed_renderable_order_line(db, admin_user, tmp_path)
+    workflow_definition = WorkflowDefinition(
+        name=f"Graph Workflow {uuid.uuid4().hex[:8]}",
+        output_type_id=order_line.output_type_id,
+        config={
+            "version": 1,
+            "ui": {"preset": "custom", "execution_mode": "graph"},
+            "nodes": [
+                {
+                    "id": "setup",
+                    "step": "order_line_setup",
+                    "params": {"failure_policy": {"fallback_to_legacy": True}},
+                },
+                {"id": "render", "step": "blender_still", "params": {"width": 1024, "height": 768}},
+            ],
+            "edges": [
+                {"from": "setup", "to": "render"},
+            ],
+        },
+        is_active=True,
+    )
+    db.add(workflow_definition)
+    await db.flush()
+    output_type = await db.get(OutputType, order_line.output_type_id)
+    assert output_type is not None
+    output_type.workflow_definition_id = workflow_definition.id
+    await db.commit()
+
+    monkeypatch.setattr(
+        "app.domains.rendering.workflow_graph_runtime.execute_graph_workflow",
+        lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("graph dispatch exploded")),
+    )
+    monkeypatch.setattr(
+        "app.domains.rendering.dispatch_service._legacy_dispatch",
+        lambda order_line_id: {"backend": "legacy", "order_line_id": order_line_id},
+    )
+
+    result = dispatch_render_with_workflow(str(order_line.id))
+
+    await db.rollback()
+
+    runs = (
+        await db.execute(
+            select(WorkflowRun).options(selectinload(WorkflowRun.node_results)).order_by(WorkflowRun.created_at.desc())
+        )
+    ).scalars().all()
+    run = runs[0]
+
+    assert result["backend"] == "legacy"
+    assert result["fallback_from"] == "workflow_graph"
+    assert result["workflow_run_id"] == str(run.id)
+    assert run.status == "failed"
+    assert run.error_message == "graph dispatch exploded"
+
+
@pytest.mark.asyncio
 async def test_workflow_dispatch_endpoint_returns_workflow_run_with_node_results(
    client,
@@ -16,8 +16,9 @@ from app.domains.orders.models import Order, OrderLine, OrderStatus
 from app.domains.products.models import CadFile, Product
 from app.domains.rendering.models import OutputType, RenderTemplate, WorkflowRun
 from app.domains.rendering.workflow_executor import prepare_workflow_context
-from app.domains.rendering.workflow_graph_runtime import execute_graph_workflow
+from app.domains.rendering.workflow_graph_runtime import WorkflowGraphRuntimeError, execute_graph_workflow
 from app.domains.rendering.workflow_run_service import create_workflow_run
+from app.domains.rendering.workflow_runtime_services import OrderLineRenderSetupResult

 import app.models  # noqa: F401

@@ -239,3 +240,116 @@ def test_execute_graph_workflow_persists_bridge_outputs_and_queues_render_task(
            {"InnerRing": "Steel", "OuterRing": "Rubber"},
        )
    ]
+
+
+def test_execute_graph_workflow_retries_bridge_node_and_persists_attempt_metadata(
+    sync_session,
+    monkeypatch,
+):
+    attempts = {"count": 0}
+
+    def _flaky_prepare(_session, _context_id):
+        attempts["count"] += 1
+        if attempts["count"] == 1:
+            raise RuntimeError("temporary setup failure")
+        return OrderLineRenderSetupResult(status="skip", reason="line_cancelled")
+
+    monkeypatch.setattr(
+        "app.domains.rendering.workflow_graph_runtime.prepare_order_line_render_context",
+        _flaky_prepare,
+    )
+
+    workflow_context = prepare_workflow_context(
+        {
+            "version": 1,
+            "nodes": [
+                {
+                    "id": "setup",
+                    "step": "order_line_setup",
+                    "params": {"retry_policy": {"max_attempts": 2}},
+                },
+            ],
+            "edges": [],
+        },
+        context_id=str(uuid.uuid4()),
+        execution_mode="graph",
+    )
+    run = create_workflow_run(
+        sync_session,
+        workflow_def_id=None,
+        order_line_id=None,
+        workflow_context=workflow_context,
+    )
+
+    dispatch_result = execute_graph_workflow(sync_session, workflow_context)
+    sync_session.commit()
+
+    refreshed_run = sync_session.execute(
+        select(WorkflowRun)
+        .where(WorkflowRun.id == run.id)
+        .options(selectinload(WorkflowRun.node_results))
+    ).scalar_one()
+    setup_result = next(node for node in refreshed_run.node_results if node.node_name == "setup")
+
+    assert dispatch_result.task_ids == []
+    assert refreshed_run.status == "completed"
+    assert setup_result.status == "skipped"
+    assert setup_result.output["attempt_count"] == 2
+    assert setup_result.output["max_attempts"] == 2
+    assert setup_result.output["retry_state"] == "recovered"
+    assert setup_result.output["last_error"] == "temporary setup failure"
+    assert setup_result.output["retry_policy"]["max_attempts"] == 2
+
+
+def test_execute_graph_workflow_marks_failed_node_with_retry_exhausted_metadata(
+    sync_session,
+    monkeypatch,
+):
+    monkeypatch.setattr(
+        "app.domains.rendering.workflow_graph_runtime.prepare_order_line_render_context",
+        lambda _session, _context_id: (_ for _ in ()).throw(RuntimeError("permanent setup failure")),
+    )
+
+    workflow_context = prepare_workflow_context(
+        {
+            "version": 1,
+            "nodes": [
+                {
+                    "id": "setup",
+                    "step": "order_line_setup",
+                    "params": {
+                        "retry_policy": {"max_attempts": 2},
+                        "failure_policy": {"fallback_to_legacy": True},
+                    },
+                },
+            ],
+            "edges": [],
+        },
+        context_id=str(uuid.uuid4()),
+        execution_mode="graph",
+    )
+    run = create_workflow_run(
+        sync_session,
+        workflow_def_id=None,
+        order_line_id=None,
+        workflow_context=workflow_context,
+    )
+
+    with pytest.raises(WorkflowGraphRuntimeError, match="permanent setup failure"):
+        execute_graph_workflow(sync_session, workflow_context)
+
+    sync_session.commit()
+
+    refreshed_run = sync_session.execute(
+        select(WorkflowRun)
+        .where(WorkflowRun.id == run.id)
+        .options(selectinload(WorkflowRun.node_results))
+    ).scalar_one()
+    setup_result = next(node for node in refreshed_run.node_results if node.node_name == "setup")
+
+    assert setup_result.status == "failed"
+    assert setup_result.output["attempt_count"] == 2
+    assert setup_result.output["max_attempts"] == 2
+    assert setup_result.output["retry_exhausted"] is True
+    assert setup_result.output["last_error"] == "permanent setup failure"
+    assert setup_result.output["failure_policy"]["fallback_to_legacy"] is True