> ## Documentation Index
> Fetch the complete documentation index at: https://docs.retab.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Create Workflow Eval

> Create a workflow eval.

Pins an expected outcome for one block in a workflow. Provide the
`workflow_id`, the `target` block, an `assertion` describing the expected
output, and a `source` of eval inputs (explicit handle inputs or a capture
from a prior run/step). Returns the created eval with status 201.

Create a new workflow eval against a single block in the workflow. A workflow eval
freezes a set of inputs (either captured from a previous run or provided
manually) and asserts something about the block's output the next time it runs.

The create route is flat: send `workflow_id` in the request body.

The request body has three parts:

* **`target`** — the block the eval runs against.
* **`source`** — where the inputs come from. `manual` carries an explicit
  `handle_inputs` map; `run_step` references a previous workflow run plus the
  optional step inside it whose inputs to capture.
* **`assertion`** — required. One assertion per eval against one declared
  output handle (see [Workflow Evals](/workflows/evals-api) for the assertion
  shape and the available operators).

<RequestExample>
  ```python Python theme={null}
  from retab import Retab

  client = Retab()

  # Manual inputs: hand-write the handle inputs the block should run with.
  workflow_eval = client.workflows.evals.create(
      workflow_id="wf_abc123xyz",
      target={"type": "block", "block_id": "block_extract_invoice"},
      source={
          "type": "manual",
          "handle_inputs": {
              "input-document-0": {
                  "type": "file",
                  "document": {
                      "id": "file_invoice_q1",
                      "filename": "invoice.pdf",
                      "mime_type": "application/pdf",
                  },
              },
          },
      },
      assertion={
          "target": {"output_handle_id": "output-json-0", "path": "total"},
          "condition": {"kind": "equals", "expected": 1234.56},
      },
      name="Q1 invoice total",
  )

  # Run-step inputs: replay the inputs the block ACTUALLY received in a run.
  workflow_eval = client.workflows.evals.create(
      workflow_id="wf_abc123xyz",
      target={"type": "block", "block_id": "block_extract_invoice"},
      source={
          "type": "run_step",
          "run_id": "wfrun_def456",
          "step_id": "block_extract_invoice",
      },
      assertion={
          "target": {"output_handle_id": "output-json-0", "path": "vendor.name"},
          "condition": {"kind": "contains", "expected": "Acme"},
      },
  )

  print(f"Eval created: {workflow_eval.id}")
  ```

  ```typescript TypeScript theme={null}
  import { Retab } from "@retab/node";

  const client = new Retab({ apiKey: process.env.RETAB_API_KEY });

  const workflowEval = await client.workflows.evals.create("wf_abc123xyz", { type: "block", blockId: "block_extract_invoice" }, {
      type: "manual",
      handleInputs: {
        "input-document-0": {
          type: "file",
          document: {
            id: "file_invoice_q1",
            filename: "invoice.pdf",
            mimeType: "application/pdf",
          },
        },
      },
    }, {
      target: { outputHandleId: "output-json-0", path: "total" },
      condition: { kind: "equals", expected: 1234.56 },
    }, "Q1 invoice total");

  console.log(`Eval created: ${workflowEval.id}`);
  ```

  ```go Go theme={null}
  package main

  import (
  	"context"
  	"fmt"
  	"log"

  	retab "github.com/retab-dev/retab/clients/go"
  )

  func ptr[T any](v T) *T { return &v }

  func main() {
  	ctx := context.Background()

  	client, err := retab.NewClient("")
  	if err != nil {
  		log.Fatal(err)
  	}

  	// Manual inputs: hand-write the handle inputs the block should run with.
  	workflowEval, err := client.Workflows.Evals.Create(ctx, &retab.WorkflowEvalsCreateParams{
  		WorkflowID: "wf_abc123xyz",
  		Target: retab.WorkflowEvalBlockTarget{
  			Type:    ptr("block"),
  			BlockID: "block_extract_invoice",
  		},
  		Source: retab.WorkflowEvalSourceFromManualWorkflowEvalSource(retab.ManualWorkflowEvalSource{
  			Type: ptr("manual"),
  		}),
  		Assertion: retab.AssertionSpec{
  			Target:    retab.OutputTarget{OutputHandleID: "output-json-0", Path: ptr("total")},
  			Condition: retab.ConditionFromExistCondition(retab.ExistCondition{Kind: ptr("exists")}),
  		},
  		Name: ptr("Q1 invoice total"),
  	})
  	if err != nil {
  		log.Fatal(err)
  	}

  	// Run-step inputs: replay the inputs the block actually received in a run.
  	runStepEval, err := client.Workflows.Evals.Create(ctx, &retab.WorkflowEvalsCreateParams{
  		WorkflowID: "wf_abc123xyz",
  		Target: retab.WorkflowEvalBlockTarget{
  			Type:    ptr("block"),
  			BlockID: "block_extract_invoice",
  		},
  		Source: retab.WorkflowEvalSourceFromManualWorkflowEvalSource(retab.ManualWorkflowEvalSource{
  			Type: ptr("manual"),
  		}),
  		Assertion: retab.AssertionSpec{
  			Target:    retab.OutputTarget{OutputHandleID: "output-json-0", Path: ptr("vendor.name")},
  			Condition: retab.ConditionFromExistCondition(retab.ExistCondition{Kind: ptr("exists")}),
  		},
  	})
  	if err != nil {
  		log.Fatal(err)
  	}

  	fmt.Printf("Eval created: %s\n", workflowEval.ID)
  	_ = runStepEval
  }
  ```

  ```ruby Ruby theme={null}
  require 'retab'

  client = Retab::Client.new(api_key: ENV['RETAB_API_KEY'])

  # Manual inputs: hand-write the handle inputs the block should run with.
  workflow_eval = client.workflows.evals.create(
    workflow_id: 'wf_abc123xyz',
    target: { type: 'block', block_id: 'block_extract_invoice' },
    source: {
      type: 'manual',
      handle_inputs: {
        'input-document-0' => {
          type: 'file',
          document: {
            id: 'file_invoice_q1',
            filename: 'invoice.pdf',
            mime_type: 'application/pdf',
          },
        },
      },
    },
    assertion: {
      target: { output_handle_id: 'output-json-0', path: 'total' },
      condition: { kind: 'equals', expected: 1234.56 },
    },
    name: 'Q1 invoice total',
  )

  # Run-step inputs: replay the inputs the block ACTUALLY received in a run.
  workflow_eval = client.workflows.evals.create(
    workflow_id: 'wf_abc123xyz',
    target: { type: 'block', block_id: 'block_extract_invoice' },
    source: {
      type: 'run_step',
      run_id: 'wfrun_def456',
      step_id: 'block_extract_invoice',
    },
    assertion: {
      target: { output_handle_id: 'output-json-0', path: 'vendor.name' },
      condition: { kind: 'contains', expected: 'Acme' },
    },
  )

  puts "Eval created: #{workflow_eval.id}"
  ```

  ```rust Rust theme={null}
  use retab::models::{
      AssertionSpec, ContainCondition, CreateWorkflowEvalRequest, OutputTarget,
      RunStepWorkflowEvalSource, WorkflowEvalBlockTarget,
  };
  use retab::resources::workflow_evals::CreateParams;
  use retab::Retab;

  #[tokio::main]
  async fn main() -> Result<(), Box<dyn std::error::Error>> {
      let client = Retab::new(std::env::var("RETAB_API_KEY")?);

      let mut source = RunStepWorkflowEvalSource::new("wfrun_def456");
      source.step_id = Some("block_extract_invoice".into());

      let mut target = OutputTarget::new("output-json-0");
      target.path = Some("vendor.name".into());

      let condition = ContainCondition {
          kind: None,
          expected: serde_json::json!("Acme"),
      };

      let body = CreateWorkflowEvalRequest::new(
          "wf_abc123xyz",
          WorkflowEvalBlockTarget::new("block_extract_invoice"),
          source.into(),
          AssertionSpec::new(target, condition.into()),
      );

      let workflow_eval = client
          .workflows()
          .evals()
          .create(CreateParams::new(body))
          .await?;
      println!("Eval created: {}", workflow_eval.id);
      Ok(())
  }
  ```

  ```php PHP theme={null}
  <?php
  require 'vendor/autoload.php';

  use Retab\Client;
  use Retab\Resource\AssertionSpec;
  use Retab\Resource\ExistCondition;
  use Retab\Resource\JsonHandleInput;
  use Retab\Resource\ManualWorkflowEvalSource;
  use Retab\Resource\OutputTarget;
  use Retab\Resource\WorkflowEvalBlockTarget;

  $client = new Client(apiKey: getenv('RETAB_API_KEY'));

  $result = $client->workflows()->evals()->create(
      workflowId: 'wf_abc123',
      target: new WorkflowEvalBlockTarget(blockId: 'block_extract_invoice'),
      source: new ManualWorkflowEvalSource(handleInputs: [
          'input-json-0' => new JsonHandleInput(data: ['invoice_number' => 'INV-001']),
      ]),
      assertion: new AssertionSpec(
          target: new OutputTarget(outputHandleId: 'output-json-0', path: 'invoice_number'),
          condition: new ExistCondition(),
      ),
      name: 'Invoice number exists',
  );
  print_r($result);
  ```

  ```csharp C# theme={null}
  using Retab;
  using RetabClient = Retab.Retab;

  var apiKey = Environment.GetEnvironmentVariable("RETAB_API_KEY")!;
  var client = new RetabClient(apiKey);

  var result = await client.Workflows.Evals.CreateAsync(new WorkflowEvalsCreateOptions());
  Console.WriteLine(result);
  ```

  ```java Java theme={null}
  import com.retab.RetabClient;
  import com.retab.workflowevals.WorkflowEvalsApi;

  public final class Example {
    public static void main(String[] args) throws Exception {
      RetabClient client = new RetabClient(System.getenv("RETAB_API_KEY"));

      var result = new WorkflowEvalsApi(client).create("wf_abc123", null, null, "Invoice Processing", null);
      System.out.println(result);
    }
  }
  ```

  ```curl cURL theme={null}
  curl -X 'POST' \
    'https://api.retab.com/v1/workflows/evals' \
    -H 'Content-Type: application/json' \
    -H 'Authorization: Bearer <your-api-key>' \
    -d '{
      "workflow_id": "wf_abc123xyz",
      "target": { "type": "block", "block_id": "block_extract_invoice" },
      "source": {
        "type": "manual",
        "handle_inputs": {
          "input-document-0": {
            "type": "file",
            "document": {
              "id": "file_invoice_q1",
              "filename": "invoice.pdf",
              "mime_type": "application/pdf"
            }
          }
        }
      },
      "assertion": {
        "target": { "output_handle_id": "output-json-0", "path": "total" },
        "condition": { "kind": "equals", "expected": 1234.56 }
      },
      "name": "Q1 invoice total"
    }'
  ```
</RequestExample>

<ResponseExample>
  ```json 201 theme={null}
  {
    "id": "wfnodeeval_hsLEQiM61ez9Piv147MWk",
    "workflow_id": "wf_abc123xyz",
    "target": {
      "type": "block",
      "block_id": "block_extract_invoice"
    },
    "source": {
      "type": "manual",
      "handle_inputs": {
        "input-document-0": {
          "type": "file",
          "document": {
            "id": "file_invoice_q1",
            "filename": "invoice.pdf",
            "mime_type": "application/pdf"
          }
        }
      }
    },
    "name": "Q1 invoice total",
    "assertion": {
      "id": "assert_xyz",
      "target": { "output_handle_id": "output-json-0", "path": "total" },
      "condition": { "kind": "equals", "expected": 1234.56 },
      "label": null
    },
    "assertion_schema_dep": {
      "schema_path": "total",
      "subtree_hash": "7d79dd764ab6548d",
      "depends_on_root": false
    },
    "assertion_drift_status": null,
    "schema_drift": "unknown",
    "schema_drift_detail": null,
    "validation_status": "valid",
    "validation_issues": [],
    "latest_run_summary": null,
    "latest_passing_run_summary": null,
    "latest_failing_run_summary": null,
    "created_at": "2026-05-01T14:30:00Z",
    "updated_at": "2026-05-01T14:30:00Z"
  }
  ```

  ```json 400 theme={null}
  {
    "detail": "assertion is required for workflow evals."
  }
  ```

  ```json 404 theme={null}
  {
    "detail": "Workflow eval not found: <block_id>"
  }
  ```
</ResponseExample>


## OpenAPI

````yaml POST /v1/workflows/evals
openapi: 3.1.0
info:
  title: FastAPI
  version: 0.1.0
servers:
  - url: https://api.retab.com
security: []
paths:
  /v1/workflows/evals:
    post:
      tags:
        - Workflows
        - Workflow Evals
      summary: Create Workflow Eval
      description: >-
        Create a workflow eval.


        Pins an expected outcome for one block in a workflow. Provide the

        `workflow_id`, the `target` block, an `assertion` describing the
        expected

        output, and a `source` of eval inputs (explicit handle inputs or a
        capture

        from a prior run/step). Returns the created eval with status 201.
      operationId: create_workflow_eval
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateWorkflowEvalRequest'
        required: true
      responses:
        '201':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/WorkflowEval'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - HTTPBearer: []
components:
  schemas:
    CreateWorkflowEvalRequest:
      properties:
        workflow_id:
          type: string
          title: Workflow Id
        target:
          $ref: '#/components/schemas/WorkflowEvalBlockTarget'
        source:
          oneOf:
            - $ref: '#/components/schemas/ManualWorkflowEvalSource'
            - $ref: '#/components/schemas/RunStepWorkflowEvalSource'
          title: Source
          discriminator:
            propertyName: type
            mapping:
              manual:
                $ref: '#/components/schemas/ManualWorkflowEvalSource'
              run_step:
                $ref: '#/components/schemas/RunStepWorkflowEvalSource'
        name:
          anyOf:
            - type: string
            - type: 'null'
          title: Name
        assertion:
          $ref: '#/components/schemas/AssertionSpec'
      additionalProperties: false
      type: object
      required:
        - assertion
        - source
        - target
        - workflow_id
      title: CreateWorkflowEvalRequest
      description: >-
        Body to create a workflow eval: the target block, an input `source`, and
        an `assertion` to evaluate its output.
    WorkflowEval:
      properties:
        id:
          type: string
          title: Id
        workflow_id:
          type: string
          title: Workflow Id
        target:
          $ref: '#/components/schemas/WorkflowEvalBlockTarget'
        source:
          oneOf:
            - $ref: '#/components/schemas/ManualWorkflowEvalSource'
            - $ref: '#/components/schemas/RunStepWorkflowEvalSource'
          title: Source
          discriminator:
            propertyName: type
            mapping:
              manual:
                $ref: '#/components/schemas/ManualWorkflowEvalSource'
              run_step:
                $ref: '#/components/schemas/RunStepWorkflowEvalSource'
        name:
          anyOf:
            - type: string
            - type: 'null'
          title: Name
        assertion:
          anyOf:
            - $ref: '#/components/schemas/AssertionSpec'
            - type: 'null'
        assertion_schema_dep:
          anyOf:
            - $ref: '#/components/schemas/AssertionSchemaDep'
            - type: 'null'
        assertion_drift_status:
          anyOf:
            - type: string
              enum:
                - valid
                - drifted
                - broken
            - type: 'null'
          title: Assertion Drift Status
        schema_drift:
          type: string
          enum:
            - none
            - partial
            - drifted
            - unknown
          title: Schema Drift
          default: unknown
        schema_drift_detail:
          anyOf:
            - type: string
            - type: 'null'
          title: Schema Drift Detail
        freshness:
          $ref: '#/components/schemas/ArtifactFreshness'
        drift:
          $ref: '#/components/schemas/ArtifactDrift'
        validation_status:
          type: string
          title: Validation Status
          default: valid
        validation_issues:
          items: {}
          type: array
          title: Validation Issues
          default: []
        latest_run_summary:
          anyOf:
            - $ref: '#/components/schemas/LatestBlockEvalRunSummary'
            - type: 'null'
        latest_passing_run_summary:
          anyOf:
            - $ref: '#/components/schemas/LatestBlockEvalRunSummary'
            - type: 'null'
        latest_failing_run_summary:
          anyOf:
            - $ref: '#/components/schemas/LatestBlockEvalRunSummary'
            - type: 'null'
        created_at:
          type: string
          format: date-time
          title: Created At
          description: When the workflow eval was created
        updated_at:
          type: string
          format: date-time
          title: Updated At
          description: When the workflow eval was last updated
      type: object
      required:
        - id
        - source
        - target
        - workflow_id
      title: WorkflowEval
      description: >-
        A saved workflow eval: a target block, an input `source`, and the
        `assertion` evaluated against its output.
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
          default: []
      type: object
      title: HTTPValidationError
    WorkflowEvalBlockTarget:
      properties:
        type:
          type: string
          const: block
          title: Type
          default: block
        block_id:
          type: string
          title: Block Id
      type: object
      required:
        - block_id
      title: WorkflowEvalBlockTarget
      description: >-
        Public workflow-eval target.


        The storage layer remains block-scoped today, but the API shape names
        the

        tested entity explicitly so workflow-level targets can be added later.
    ManualWorkflowEvalSource:
      properties:
        type:
          type: string
          const: manual
          title: Type
          default: manual
        handle_inputs:
          additionalProperties:
            oneOf:
              - $ref: '#/components/schemas/EvalJsonHandleInput'
              - $ref: '#/components/schemas/EvalFileHandleInput'
            discriminator:
              propertyName: type
              mapping:
                file:
                  $ref: '#/components/schemas/EvalFileHandleInput'
                json:
                  $ref: '#/components/schemas/EvalJsonHandleInput'
          type: object
          title: Handle Inputs
          default: {}
      type: object
      title: ManualWorkflowEvalSource
    RunStepWorkflowEvalSource:
      properties:
        type:
          type: string
          const: run_step
          title: Type
          default: run_step
        run_id:
          type: string
          title: Run Id
        step_id:
          anyOf:
            - type: string
            - type: 'null'
          title: Step Id
      type: object
      required:
        - run_id
      title: RunStepWorkflowEvalSource
    AssertionSpec:
      properties:
        id:
          anyOf:
            - type: string
            - type: 'null'
          title: Id
        target:
          $ref: '#/components/schemas/OutputTarget'
        condition:
          oneOf:
            - $ref: '#/components/schemas/ExistsCondition'
            - $ref: '#/components/schemas/NotExistsCondition'
            - $ref: '#/components/schemas/EqualsCondition'
            - $ref: '#/components/schemas/NotEqualsCondition'
            - $ref: '#/components/schemas/NumberCompareCondition'
            - $ref: '#/components/schemas/ContainsCondition'
            - $ref: '#/components/schemas/ObjectContainsCondition'
            - $ref: '#/components/schemas/ArrayContainsCondition'
            - $ref: '#/components/schemas/MatchesRegexCondition'
            - $ref: '#/components/schemas/JsonSchemaValidCondition'
            - $ref: '#/components/schemas/SimilarityGteCondition'
            - $ref: '#/components/schemas/LlmJudgedAsCondition'
            - $ref: '#/components/schemas/LlmNotJudgedAsCondition'
            - $ref: '#/components/schemas/NotContainsCondition'
            - $ref: '#/components/schemas/LengthCompareCondition'
            - $ref: '#/components/schemas/BetweenCondition'
            - $ref: '#/components/schemas/StartsWithCondition'
            - $ref: '#/components/schemas/EndsWithCondition'
            - $ref: '#/components/schemas/AllItemsMatchCondition'
            - $ref: '#/components/schemas/AnyItemMatchesCondition'
            - $ref: '#/components/schemas/SplitIOUCondition'
          title: Condition
          discriminator:
            propertyName: kind
            mapping:
              all_items_match:
                $ref: '#/components/schemas/AllItemsMatchCondition'
              any_item_matches:
                $ref: '#/components/schemas/AnyItemMatchesCondition'
              array_contains:
                $ref: '#/components/schemas/ArrayContainsCondition'
              between:
                $ref: '#/components/schemas/BetweenCondition'
              contains:
                $ref: '#/components/schemas/ContainsCondition'
              ends_with:
                $ref: '#/components/schemas/EndsWithCondition'
              equals:
                $ref: '#/components/schemas/EqualsCondition'
              exists:
                $ref: '#/components/schemas/ExistsCondition'
              json_schema_valid:
                $ref: '#/components/schemas/JsonSchemaValidCondition'
              length_compare:
                $ref: '#/components/schemas/LengthCompareCondition'
              llm_judged_as:
                $ref: '#/components/schemas/LlmJudgedAsCondition'
              llm_not_judged_as:
                $ref: '#/components/schemas/LlmNotJudgedAsCondition'
              matches_regex:
                $ref: '#/components/schemas/MatchesRegexCondition'
              not_contains:
                $ref: '#/components/schemas/NotContainsCondition'
              not_equals:
                $ref: '#/components/schemas/NotEqualsCondition'
              not_exists:
                $ref: '#/components/schemas/NotExistsCondition'
              number_compare:
                $ref: '#/components/schemas/NumberCompareCondition'
              object_contains:
                $ref: '#/components/schemas/ObjectContainsCondition'
              similarity_gte:
                $ref: '#/components/schemas/SimilarityGteCondition'
              split_iou_gte:
                $ref: '#/components/schemas/SplitIOUCondition'
              starts_with:
                $ref: '#/components/schemas/StartsWithCondition'
        label:
          anyOf:
            - type: string
            - type: 'null'
          title: Label
      type: object
      required:
        - condition
        - target
      title: AssertionSpec
      description: |-
        Block-eval assertion against one declared output handle.

        `target` is the only supported shape: an output handle id and an
        optional relative path inside that handle's payload.
    AssertionSchemaDep:
      properties:
        output_handle_id:
          anyOf:
            - type: string
            - type: 'null'
          title: Output Handle Id
        schema_path:
          type: string
          title: Schema Path
        subtree_hash:
          type: string
          title: Subtree Hash
        depends_on_root:
          type: boolean
          title: Depends On Root
          default: false
      type: object
      required:
        - schema_path
        - subtree_hash
      title: AssertionSchemaDep
      description: Single-rule schema dependency for Level 2 drift detection.
    ArtifactFreshness:
      properties:
        status:
          type: string
          enum:
            - fresh
            - stale
            - unknown
          title: Status
          default: unknown
        reasons:
          items:
            type: string
            enum:
              - validity_changed
              - inputs_changed
              - engine_changed
              - metrics_engine_changed
              - no_baseline
          type: array
          title: Reasons
          default: []
        validity_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Validity Fingerprint
        input_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Input Fingerprint
        baseline_run_id:
          anyOf:
            - type: string
            - type: 'null'
          title: Baseline Run Id
      type: object
      title: ArtifactFreshness
    ArtifactDrift:
      properties:
        status:
          type: string
          enum:
            - none
            - drifted
            - broken
            - unknown
          title: Status
          default: unknown
        affected_targets:
          items:
            type: string
          type: array
          title: Affected Targets
          default: []
        detail:
          anyOf:
            - type: string
            - type: 'null'
          title: Detail
      type: object
      title: ArtifactDrift
    LatestBlockEvalRunSummary:
      properties:
        run_record_id:
          type: string
          title: Run Record Id
        status:
          type: string
          enum:
            - pending
            - queued
            - running
            - completed
            - error
            - cancelled
          title: Status
        outcome:
          anyOf:
            - type: string
              enum:
                - passed
                - failed
                - blocked
            - type: 'null'
          title: Outcome
        started_at:
          type: string
          format: date-time
          title: Started At
        completed_at:
          anyOf:
            - type: string
              format: date-time
            - type: 'null'
          title: Completed At
        duration_ms:
          anyOf:
            - type: integer
            - type: 'null'
          title: Duration Ms
        workflow_draft_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Workflow Draft Fingerprint
        block_execution_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Block Execution Fingerprint
        validity_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Validity Fingerprint
        handle_inputs_fingerprint:
          anyOf:
            - type: string
            - type: 'null'
          title: Handle Inputs Fingerprint
        assertions_passed:
          type: integer
          title: Assertions Passed
          default: 0
        assertions_failed:
          type: integer
          title: Assertions Failed
          default: 0
        blocked_assertions:
          type: integer
          title: Blocked Assertions
          default: 0
      type: object
      required:
        - run_record_id
        - started_at
        - status
      title: LatestBlockEvalRunSummary
      description: |-
        Summary of the most recent block-eval run.

        Execution status and verdict outcome are exposed as separate fields.
        The summary is written on terminal-state transitions, so in practice
        `status` is one of `completed | error | cancelled` and `outcome` is
        populated when `status == "completed"`.
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
        input:
          title: Input
          default: null
        ctx:
          type: object
          title: Context
          default: {}
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
    EvalJsonHandleInput:
      properties:
        data:
          title: Data
          default: null
        type:
          type: string
          const: json
          title: Type
          default: json
      additionalProperties: false
      type: object
      title: EvalJsonHandleInput
    EvalFileHandleInput:
      properties:
        document:
          $ref: '#/components/schemas/EvalPublicFileRef'
        type:
          type: string
          const: file
          title: Type
          default: file
      additionalProperties: false
      type: object
      required:
        - document
      title: EvalFileHandleInput
    OutputTarget:
      properties:
        output_handle_id:
          type: string
          minLength: 1
          title: Output Handle Id
        path:
          anyOf:
            - type: string
            - type: 'null'
          title: Path
      additionalProperties: false
      type: object
      required:
        - output_handle_id
      title: OutputTarget
      description: Structured block-eval assertion target.
    ExistsCondition:
      properties:
        kind:
          type: string
          const: exists
          title: Kind
          default: exists
      type: object
      title: ExistsCondition
    NotExistsCondition:
      properties:
        kind:
          type: string
          const: not_exists
          title: Kind
          default: not_exists
      type: object
      title: NotExistsCondition
    EqualsCondition:
      properties:
        kind:
          type: string
          const: equals
          title: Kind
          default: equals
        expected:
          title: Expected
      type: object
      required:
        - expected
      title: EqualsCondition
    NotEqualsCondition:
      properties:
        kind:
          type: string
          const: not_equals
          title: Kind
          default: not_equals
        expected:
          title: Expected
      type: object
      required:
        - expected
      title: NotEqualsCondition
    NumberCompareCondition:
      properties:
        kind:
          type: string
          const: number_compare
          title: Kind
          default: number_compare
        op:
          type: string
          enum:
            - gt
            - gte
            - lt
            - lte
            - eq
            - neq
          title: Op
        expected:
          anyOf:
            - type: integer
            - type: number
          title: Expected
      type: object
      required:
        - expected
        - op
      title: NumberCompareCondition
    ContainsCondition:
      properties:
        kind:
          type: string
          const: contains
          title: Kind
          default: contains
        expected:
          title: Expected
      type: object
      required:
        - expected
      title: ContainsCondition
    ObjectContainsCondition:
      properties:
        kind:
          type: string
          const: object_contains
          title: Kind
          default: object_contains
        expected:
          additionalProperties: true
          type: object
          title: Expected
      type: object
      required:
        - expected
      title: ObjectContainsCondition
    ArrayContainsCondition:
      properties:
        kind:
          type: string
          const: array_contains
          title: Kind
          default: array_contains
        expected:
          additionalProperties: true
          type: object
          title: Expected
      type: object
      required:
        - expected
      title: ArrayContainsCondition
    MatchesRegexCondition:
      properties:
        kind:
          type: string
          const: matches_regex
          title: Kind
          default: matches_regex
        pattern:
          type: string
          title: Pattern
      type: object
      required:
        - pattern
      title: MatchesRegexCondition
    JsonSchemaValidCondition:
      properties:
        kind:
          type: string
          const: json_schema_valid
          title: Kind
          default: json_schema_valid
        schema:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          title: Schema
      type: object
      title: JsonSchemaValidCondition
    SimilarityGteCondition:
      properties:
        kind:
          type: string
          const: similarity_gte
          title: Kind
          default: similarity_gte
        reference:
          title: Reference
        threshold:
          type: number
          title: Threshold
        method:
          type: string
          enum:
            - levenshtein
            - embeddings
          title: Method
          default: levenshtein
      type: object
      required:
        - reference
        - threshold
      title: SimilarityGteCondition
    LlmJudgedAsCondition:
      properties:
        kind:
          type: string
          const: llm_judged_as
          title: Kind
          default: llm_judged_as
        rubric:
          type: string
          title: Rubric
        expected_label:
          anyOf:
            - type: string
            - type: 'null'
          title: Expected Label
      type: object
      required:
        - rubric
      title: LlmJudgedAsCondition
    LlmNotJudgedAsCondition:
      properties:
        kind:
          type: string
          const: llm_not_judged_as
          title: Kind
          default: llm_not_judged_as
        rubric:
          type: string
          title: Rubric
        expected_label:
          anyOf:
            - type: string
            - type: 'null'
          title: Expected Label
      type: object
      required:
        - rubric
      title: LlmNotJudgedAsCondition
    NotContainsCondition:
      properties:
        kind:
          type: string
          const: not_contains
          title: Kind
          default: not_contains
        expected:
          title: Expected
      type: object
      required:
        - expected
      title: NotContainsCondition
    LengthCompareCondition:
      properties:
        kind:
          type: string
          const: length_compare
          title: Kind
          default: length_compare
        op:
          type: string
          enum:
            - gt
            - gte
            - lt
            - lte
            - eq
            - neq
          title: Op
        expected:
          type: integer
          title: Expected
      type: object
      required:
        - expected
        - op
      title: LengthCompareCondition
    BetweenCondition:
      properties:
        kind:
          type: string
          const: between
          title: Kind
          default: between
        lower:
          anyOf:
            - type: integer
            - type: number
          title: Lower
        upper:
          anyOf:
            - type: integer
            - type: number
          title: Upper
        inclusive:
          type: boolean
          title: Inclusive
          default: true
      type: object
      required:
        - lower
        - upper
      title: BetweenCondition
    StartsWithCondition:
      properties:
        kind:
          type: string
          const: starts_with
          title: Kind
          default: starts_with
        expected:
          type: string
          title: Expected
      type: object
      required:
        - expected
      title: StartsWithCondition
    EndsWithCondition:
      properties:
        kind:
          type: string
          const: ends_with
          title: Kind
          default: ends_with
        expected:
          type: string
          title: Expected
      type: object
      required:
        - expected
      title: EndsWithCondition
    AllItemsMatchCondition:
      properties:
        kind:
          type: string
          const: all_items_match
          title: Kind
          default: all_items_match
        condition:
          oneOf:
            - $ref: '#/components/schemas/ExistsCondition'
            - $ref: '#/components/schemas/NotExistsCondition'
            - $ref: '#/components/schemas/EqualsCondition'
            - $ref: '#/components/schemas/NotEqualsCondition'
            - $ref: '#/components/schemas/NumberCompareCondition'
            - $ref: '#/components/schemas/ContainsCondition'
            - $ref: '#/components/schemas/ObjectContainsCondition'
            - $ref: '#/components/schemas/ArrayContainsCondition'
            - $ref: '#/components/schemas/MatchesRegexCondition'
            - $ref: '#/components/schemas/JsonSchemaValidCondition'
            - $ref: '#/components/schemas/SimilarityGteCondition'
            - $ref: '#/components/schemas/LlmJudgedAsCondition'
            - $ref: '#/components/schemas/LlmNotJudgedAsCondition'
            - $ref: '#/components/schemas/NotContainsCondition'
            - $ref: '#/components/schemas/LengthCompareCondition'
            - $ref: '#/components/schemas/BetweenCondition'
            - $ref: '#/components/schemas/StartsWithCondition'
            - $ref: '#/components/schemas/EndsWithCondition'
            - $ref: '#/components/schemas/AllItemsMatchCondition'
            - $ref: '#/components/schemas/AnyItemMatchesCondition'
            - $ref: '#/components/schemas/SplitIOUCondition'
          title: Condition
          discriminator:
            propertyName: kind
            mapping:
              all_items_match:
                $ref: '#/components/schemas/AllItemsMatchCondition'
              any_item_matches:
                $ref: '#/components/schemas/AnyItemMatchesCondition'
              array_contains:
                $ref: '#/components/schemas/ArrayContainsCondition'
              between:
                $ref: '#/components/schemas/BetweenCondition'
              contains:
                $ref: '#/components/schemas/ContainsCondition'
              ends_with:
                $ref: '#/components/schemas/EndsWithCondition'
              equals:
                $ref: '#/components/schemas/EqualsCondition'
              exists:
                $ref: '#/components/schemas/ExistsCondition'
              json_schema_valid:
                $ref: '#/components/schemas/JsonSchemaValidCondition'
              length_compare:
                $ref: '#/components/schemas/LengthCompareCondition'
              llm_judged_as:
                $ref: '#/components/schemas/LlmJudgedAsCondition'
              llm_not_judged_as:
                $ref: '#/components/schemas/LlmNotJudgedAsCondition'
              matches_regex:
                $ref: '#/components/schemas/MatchesRegexCondition'
              not_contains:
                $ref: '#/components/schemas/NotContainsCondition'
              not_equals:
                $ref: '#/components/schemas/NotEqualsCondition'
              not_exists:
                $ref: '#/components/schemas/NotExistsCondition'
              number_compare:
                $ref: '#/components/schemas/NumberCompareCondition'
              object_contains:
                $ref: '#/components/schemas/ObjectContainsCondition'
              similarity_gte:
                $ref: '#/components/schemas/SimilarityGteCondition'
              split_iou_gte:
                $ref: '#/components/schemas/SplitIOUCondition'
              starts_with:
                $ref: '#/components/schemas/StartsWithCondition'
      type: object
      required:
        - condition
      title: AllItemsMatchCondition
    AnyItemMatchesCondition:
      properties:
        kind:
          type: string
          const: any_item_matches
          title: Kind
          default: any_item_matches
        condition:
          oneOf:
            - $ref: '#/components/schemas/ExistsCondition'
            - $ref: '#/components/schemas/NotExistsCondition'
            - $ref: '#/components/schemas/EqualsCondition'
            - $ref: '#/components/schemas/NotEqualsCondition'
            - $ref: '#/components/schemas/NumberCompareCondition'
            - $ref: '#/components/schemas/ContainsCondition'
            - $ref: '#/components/schemas/ObjectContainsCondition'
            - $ref: '#/components/schemas/ArrayContainsCondition'
            - $ref: '#/components/schemas/MatchesRegexCondition'
            - $ref: '#/components/schemas/JsonSchemaValidCondition'
            - $ref: '#/components/schemas/SimilarityGteCondition'
            - $ref: '#/components/schemas/LlmJudgedAsCondition'
            - $ref: '#/components/schemas/LlmNotJudgedAsCondition'
            - $ref: '#/components/schemas/NotContainsCondition'
            - $ref: '#/components/schemas/LengthCompareCondition'
            - $ref: '#/components/schemas/BetweenCondition'
            - $ref: '#/components/schemas/StartsWithCondition'
            - $ref: '#/components/schemas/EndsWithCondition'
            - $ref: '#/components/schemas/AllItemsMatchCondition'
            - $ref: '#/components/schemas/AnyItemMatchesCondition'
            - $ref: '#/components/schemas/SplitIOUCondition'
          title: Condition
          discriminator:
            propertyName: kind
            mapping:
              all_items_match:
                $ref: '#/components/schemas/AllItemsMatchCondition'
              any_item_matches:
                $ref: '#/components/schemas/AnyItemMatchesCondition'
              array_contains:
                $ref: '#/components/schemas/ArrayContainsCondition'
              between:
                $ref: '#/components/schemas/BetweenCondition'
              contains:
                $ref: '#/components/schemas/ContainsCondition'
              ends_with:
                $ref: '#/components/schemas/EndsWithCondition'
              equals:
                $ref: '#/components/schemas/EqualsCondition'
              exists:
                $ref: '#/components/schemas/ExistsCondition'
              json_schema_valid:
                $ref: '#/components/schemas/JsonSchemaValidCondition'
              length_compare:
                $ref: '#/components/schemas/LengthCompareCondition'
              llm_judged_as:
                $ref: '#/components/schemas/LlmJudgedAsCondition'
              llm_not_judged_as:
                $ref: '#/components/schemas/LlmNotJudgedAsCondition'
              matches_regex:
                $ref: '#/components/schemas/MatchesRegexCondition'
              not_contains:
                $ref: '#/components/schemas/NotContainsCondition'
              not_equals:
                $ref: '#/components/schemas/NotEqualsCondition'
              not_exists:
                $ref: '#/components/schemas/NotExistsCondition'
              number_compare:
                $ref: '#/components/schemas/NumberCompareCondition'
              object_contains:
                $ref: '#/components/schemas/ObjectContainsCondition'
              similarity_gte:
                $ref: '#/components/schemas/SimilarityGteCondition'
              split_iou_gte:
                $ref: '#/components/schemas/SplitIOUCondition'
              starts_with:
                $ref: '#/components/schemas/StartsWithCondition'
      type: object
      required:
        - condition
      title: AnyItemMatchesCondition
    SplitIOUCondition:
      properties:
        kind:
          type: string
          const: split_iou_gte
          title: Kind
          default: split_iou_gte
        expected:
          additionalProperties: true
          type: object
          title: Expected
        threshold:
          type: number
          title: Threshold
          default: 1
      type: object
      required:
        - expected
      title: SplitIOUCondition
      description: |-
        Intersection-over-Union for split page assignments.

        `expected` uses the split payload shape:
        `{"splits": [{"name", "pages"}]}`
    EvalPublicFileRef:
      properties:
        id:
          type: string
          title: Id
          description: ID of the file
        filename:
          type: string
          title: Filename
          description: Filename of the file
        mime_type:
          type: string
          title: Mime Type
          description: MIME type of the file
      type: object
      required:
        - filename
        - id
        - mime_type
      title: EvalPublicFileRef
      description: Public/shared file reference used across SDK and customer-facing APIs.
  securitySchemes:
    HTTPBearer:
      type: http
      scheme: bearer

````