Skip to main content
POST
/
v1
/
workflows
/
evals
from retab import Retab

client = Retab()

# Manual inputs: hand-write the handle inputs the block should run with.
workflow_eval = client.workflows.evals.create(
    workflow_id="wf_abc123xyz",
    target={"type": "block", "block_id": "block_extract_invoice"},
    source={
        "type": "manual",
        "handle_inputs": {
            "input-document-0": {
                "type": "file",
                "document": {
                    "id": "file_invoice_q1",
                    "filename": "invoice.pdf",
                    "mime_type": "application/pdf",
                },
            },
        },
    },
    assertion={
        "target": {"output_handle_id": "output-json-0", "path": "total"},
        "condition": {"kind": "equals", "expected": 1234.56},
    },
    name="Q1 invoice total",
)

# Run-step inputs: replay the inputs the block ACTUALLY received in a run.
workflow_eval = client.workflows.evals.create(
    workflow_id="wf_abc123xyz",
    target={"type": "block", "block_id": "block_extract_invoice"},
    source={
        "type": "run_step",
        "run_id": "wfrun_def456",
        "step_id": "block_extract_invoice",
    },
    assertion={
        "target": {"output_handle_id": "output-json-0", "path": "vendor.name"},
        "condition": {"kind": "contains", "expected": "Acme"},
    },
)

print(f"Eval created: {workflow_eval.id}")

{
  "id": "wfnodeeval_hsLEQiM61ez9Piv147MWk",
  "workflow_id": "wf_abc123xyz",
  "target": {
    "type": "block",
    "block_id": "block_extract_invoice"
  },
  "source": {
    "type": "manual",
    "handle_inputs": {
      "input-document-0": {
        "type": "file",
        "document": {
          "id": "file_invoice_q1",
          "filename": "invoice.pdf",
          "mime_type": "application/pdf"
        }
      }
    }
  },
  "name": "Q1 invoice total",
  "assertion": {
    "id": "assert_xyz",
    "target": { "output_handle_id": "output-json-0", "path": "total" },
    "condition": { "kind": "equals", "expected": 1234.56 },
    "label": null
  },
  "assertion_schema_dep": {
    "schema_path": "total",
    "subtree_hash": "7d79dd764ab6548d",
    "depends_on_root": false
  },
  "assertion_drift_status": null,
  "schema_drift": "unknown",
  "schema_drift_detail": null,
  "validation_status": "valid",
  "validation_issues": [],
  "latest_run_summary": null,
  "latest_passing_run_summary": null,
  "latest_failing_run_summary": null,
  "created_at": "2026-05-01T14:30:00Z",
  "updated_at": "2026-05-01T14:30:00Z"
}
Create a new workflow eval against a single block in the workflow. A workflow eval freezes a set of inputs (either captured from a previous run or provided manually) and asserts something about the block’s output the next time it runs. The create route is flat: send workflow_id in the request body. The request body has three parts:
  • target — the block the eval runs against.
  • source — where the inputs come from. manual carries an explicit handle_inputs map; run_step references a previous workflow run plus the optional step inside it whose inputs to capture.
  • assertion — required. One assertion per eval against one declared output handle (see Workflow Evals for the assertion shape and the available operators).
from retab import Retab

client = Retab()

# Manual inputs: hand-write the handle inputs the block should run with.
workflow_eval = client.workflows.evals.create(
    workflow_id="wf_abc123xyz",
    target={"type": "block", "block_id": "block_extract_invoice"},
    source={
        "type": "manual",
        "handle_inputs": {
            "input-document-0": {
                "type": "file",
                "document": {
                    "id": "file_invoice_q1",
                    "filename": "invoice.pdf",
                    "mime_type": "application/pdf",
                },
            },
        },
    },
    assertion={
        "target": {"output_handle_id": "output-json-0", "path": "total"},
        "condition": {"kind": "equals", "expected": 1234.56},
    },
    name="Q1 invoice total",
)

# Run-step inputs: replay the inputs the block ACTUALLY received in a run.
workflow_eval = client.workflows.evals.create(
    workflow_id="wf_abc123xyz",
    target={"type": "block", "block_id": "block_extract_invoice"},
    source={
        "type": "run_step",
        "run_id": "wfrun_def456",
        "step_id": "block_extract_invoice",
    },
    assertion={
        "target": {"output_handle_id": "output-json-0", "path": "vendor.name"},
        "condition": {"kind": "contains", "expected": "Acme"},
    },
)

print(f"Eval created: {workflow_eval.id}")

{
  "id": "wfnodeeval_hsLEQiM61ez9Piv147MWk",
  "workflow_id": "wf_abc123xyz",
  "target": {
    "type": "block",
    "block_id": "block_extract_invoice"
  },
  "source": {
    "type": "manual",
    "handle_inputs": {
      "input-document-0": {
        "type": "file",
        "document": {
          "id": "file_invoice_q1",
          "filename": "invoice.pdf",
          "mime_type": "application/pdf"
        }
      }
    }
  },
  "name": "Q1 invoice total",
  "assertion": {
    "id": "assert_xyz",
    "target": { "output_handle_id": "output-json-0", "path": "total" },
    "condition": { "kind": "equals", "expected": 1234.56 },
    "label": null
  },
  "assertion_schema_dep": {
    "schema_path": "total",
    "subtree_hash": "7d79dd764ab6548d",
    "depends_on_root": false
  },
  "assertion_drift_status": null,
  "schema_drift": "unknown",
  "schema_drift_detail": null,
  "validation_status": "valid",
  "validation_issues": [],
  "latest_run_summary": null,
  "latest_passing_run_summary": null,
  "latest_failing_run_summary": null,
  "created_at": "2026-05-01T14:30:00Z",
  "updated_at": "2026-05-01T14:30:00Z"
}

Authorizations

​
Authorization
string
header
required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json

Body to create a workflow eval: the target block, an input source, and an assertion to evaluate its output.

​
workflow_id
string
required
​
target
WorkflowEvalBlockTarget · object
required

Public workflow-eval target.

The storage layer remains block-scoped today, but the API shape names the tested entity explicitly so workflow-level targets can be added later.

​
source
ManualWorkflowEvalSource · object
required
​
assertion
AssertionSpec · object
required

Block-eval assertion against one declared output handle.

target is the only supported shape: an output handle id and an optional relative path inside that handle's payload.

​
name
string | null

Response

Successful Response

A saved workflow eval: a target block, an input source, and the assertion evaluated against its output.

​
id
string
required
​
workflow_id
string
required
​
target
WorkflowEvalBlockTarget · object
required

Public workflow-eval target.

The storage layer remains block-scoped today, but the API shape names the tested entity explicitly so workflow-level targets can be added later.

​
source
ManualWorkflowEvalSource · object
required
​
name
string | null
​
assertion
AssertionSpec · object

Block-eval assertion against one declared output handle.

target is the only supported shape: an output handle id and an optional relative path inside that handle's payload.

​
assertion_schema_dep
AssertionSchemaDep · object

Single-rule schema dependency for Level 2 drift detection.

​
assertion_drift_status
enum<string> | null
Available options:
valid,
drifted,
broken
​
schema_drift
enum<string>
default:unknown
Available options:
none,
partial,
drifted,
unknown
​
schema_drift_detail
string | null
​
freshness
ArtifactFreshness · object
​
drift
ArtifactDrift · object
​
validation_status
string
default:valid
​
validation_issues
any[]
​
latest_run_summary
LatestBlockEvalRunSummary · object

Summary of the most recent block-eval run.

Execution status and verdict outcome are exposed as separate fields. The summary is written on terminal-state transitions, so in practice status is one of completed | error | cancelled and outcome is populated when status == "completed".

​
latest_passing_run_summary
LatestBlockEvalRunSummary · object

Summary of the most recent block-eval run.

Execution status and verdict outcome are exposed as separate fields. The summary is written on terminal-state transitions, so in practice status is one of completed | error | cancelled and outcome is populated when status == "completed".

​
latest_failing_run_summary
LatestBlockEvalRunSummary · object

Summary of the most recent block-eval run.

Execution status and verdict outcome are exposed as separate fields. The summary is written on terminal-state transitions, so in practice status is one of completed | error | cancelled and outcome is populated when status == "completed".

​
created_at
string<date-time>

When the workflow eval was created

​
updated_at
string<date-time>

When the workflow eval was last updated