> ## Documentation Index
> Fetch the complete documentation index at: https://penseapp.vercel.app/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Get benchmark status

> Get the results of a benchmark run



## OpenAPI

````yaml /api-reference/openapi.json get /agent-tests/benchmark/{task_id}
openapi: 3.1.0
info:
  title: Calibrate Public API
  version: 0.1.0
  description: Programmatic API for CI/automation. Pass your key in the `X-API-Key` header.
servers:
  - url: https://api.calibrate.artpark.ai
    description: Production
security: []
paths:
  /agent-tests/benchmark/{task_id}:
    get:
      tags:
        - agent-tests
      summary: Get benchmark status
      description: Get the results of a benchmark run
      operationId: get_benchmark_status_agent_tests_benchmark__task_id__get
      parameters:
        - name: task_id
          in: path
          required: true
          schema:
            type: string
            description: Benchmark run to poll for status and results
            examples:
              - a3b2c1d0-e5f4-3210-abcd-ef1234567890
            title: Task Id
          description: Benchmark run to poll for status and results
        - name: only_failed
          in: query
          required: false
          schema:
            type: boolean
            description: >-
              Return only failing test cases for each model. Omit to return
              every case
            default: false
            title: Only Failed
          description: >-
            Return only failing test cases for each model. Omit to return every
            case
        - name: compact
          in: query
          required: false
          schema:
            type: boolean
            description: >-
              Return a compact response that omits heavy detail fields
              (`model_results.test_results`, `evaluators.output_config`),
              keeping only the lightweight decision fields. Omit for full detail
            default: false
            title: Compact
          description: >-
            Return a compact response that omits heavy detail fields
            (`model_results.test_results`, `evaluators.output_config`), keeping
            only the lightweight decision fields. Omit for full detail
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BenchmarkStatusResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - ApiKeyAuth: []
components:
  schemas:
    BenchmarkStatusResponse:
      properties:
        task_id:
          type: string
          maxLength: 36
          minLength: 36
          title: Task Id
          description: Benchmark run job ID
          examples:
            - a3b2c1d0-e5f4-3210-abcd-ef1234567890
        status:
          $ref: '#/components/schemas/TaskStatus'
          description: Current status of the run
        test_uuids:
          anyOf:
            - items:
                type: string
              type: array
            - type: 'null'
          title: Test Uuids
          description: IDs of the tests this benchmark executed, in run order
        evaluators:
          anyOf:
            - items:
                $ref: '#/components/schemas/TestRunEvaluator'
              type: array
            - type: 'null'
          title: Evaluators
          description: >-
            The evaluators used in this run. Each verdict in `judge_results`
            links to one of these by `evaluator_uuid`
        model_results:
          anyOf:
            - items:
                $ref: '#/components/schemas/ModelResult'
              type: array
            - type: 'null'
          title: Model Results
          description: Results for each model
        leaderboard_summary:
          anyOf:
            - items:
                additionalProperties: true
                type: object
              type: array
            - type: 'null'
          description: >-
            Leaderboard comparing the models, one row per model. Columns vary by
            benchmark: a `model` column plus pass/fail counts, latency, cost,
            and one score column per evaluator, keyed by evaluator name
          examples:
            - - cost: 0.0021
                failed: 2
                helpfulness: 0.92
                latency_p50: 480
                model: openai/gpt-4.1
                passed: 8
                safety: 1
              - cost: 0.0034
                failed: 1
                helpfulness: 0.95
                latency_p50: 520
                model: anthropic/claude-sonnet-4
                passed: 9
                safety: 1
        error:
          type: boolean
          title: Error
          description: True if the run failed
          default: false
        is_public:
          type: boolean
          title: Is Public
          description: Whether the run is shared publicly
          default: false
        share_token:
          anyOf:
            - type: string
            - type: 'null'
          title: Share Token
          description: Token for building the public share URL
      type: object
      required:
        - task_id
        - status
      title: BenchmarkStatusResponse
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    TaskStatus:
      type: string
      enum:
        - queued
        - in_progress
        - cancelled
        - done
        - failed
      title: TaskStatus
    TestRunEvaluator:
      properties:
        uuid:
          anyOf:
            - type: string
            - type: 'null'
          title: Uuid
          description: ID of the evaluator
        name:
          anyOf:
            - type: string
            - type: 'null'
          title: Name
          description: Name of the evaluator
        description:
          anyOf:
            - type: string
            - type: 'null'
          title: Description
          description: What the evaluator checks
        output_type:
          anyOf:
            - type: string
              enum:
                - binary
                - rating
            - type: 'null'
          title: Output Type
          description: |-
            The shape of the verdict:
            - `binary`: a pass/fail verdict
            - `rating`: a numeric score
        output_config:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: 'The rubric: the scale values, labels, and colors a verdict maps to'
          examples:
            - scale:
                - color: '#e5484d'
                  name: Wrong
                  value: false
                - color: '#30a46c'
                  name: Correct
                  value: true
        scale_min:
          anyOf:
            - type: number
            - type: 'null'
          title: Scale Min
          description: Lowest value on a rating scale
        scale_max:
          anyOf:
            - type: number
            - type: 'null'
          title: Scale Max
          description: Highest value on a rating scale
        version_number:
          anyOf:
            - type: integer
            - type: 'null'
          title: Version Number
          description: The evaluator version this run used
      type: object
      title: TestRunEvaluator
    ModelResult:
      properties:
        model:
          type: string
          title: Model
          description: Model name these results are for
          examples:
            - openai/gpt-4.1
        success:
          anyOf:
            - type: boolean
            - type: 'null'
          title: Success
          description: Whether this model's run succeeded
        message:
          type: string
          title: Message
          description: Status or result message for this model
        total_tests:
          anyOf:
            - type: integer
            - type: 'null'
          title: Total Tests
          description: Total test cases for this model
        passed:
          anyOf:
            - type: integer
            - type: 'null'
          title: Passed
          description: Number of test cases that passed
        failed:
          anyOf:
            - type: integer
            - type: 'null'
          title: Failed
          description: Number of test cases that failed
        evaluator_summary:
          anyOf:
            - items:
                additionalProperties: true
                type: object
              type: array
            - type: 'null'
          description: Aggregate summary for each evaluator for this model
        test_results:
          anyOf:
            - items:
                $ref: '#/components/schemas/TestCaseResult'
              type: array
            - type: 'null'
          title: Test Results
          description: Results for each test case for this model
        latency_ms:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: Aggregated latency in milliseconds, as `{p50, p95, p99, count}`
        cost:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: Aggregated cost as `{mean, min, max, count}` (USD)
        total_tokens:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: Aggregated token usage as `{mean, min, max, count}`
      type: object
      required:
        - model
        - message
      title: ModelResult
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
    TestCaseResult:
      properties:
        test_case_id:
          anyOf:
            - type: string
            - type: 'null'
          title: Test Case Id
          description: ID of the test case within the run
        name:
          anyOf:
            - type: string
            - type: 'null'
          title: Name
          description: Name of the test
        passed:
          anyOf:
            - type: boolean
            - type: 'null'
          title: Passed
          description: Whether the case passed
        reasoning:
          anyOf:
            - type: string
            - type: 'null'
          title: Reasoning
          description: The judge's reasoning, or the tool-call diff for a tool-call test
        output:
          anyOf:
            - $ref: '#/components/schemas/TestOutput'
            - type: 'null'
          description: The agent's output for this case
        test_case:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: The test case definition that was run
        judge_results:
          anyOf:
            - items:
                $ref: '#/components/schemas/JudgeResult'
              type: array
            - type: 'null'
          title: Judge Results
          description: One verdict for each evaluator
        latency_ms:
          anyOf:
            - type: number
            - type: 'null'
          title: Latency Ms
          description: How long the agent took to respond, in milliseconds
        cost:
          anyOf:
            - type: number
            - type: 'null'
          title: Cost
          description: Cost of this case (USD)
      type: object
      title: TestCaseResult
    TestOutput:
      properties:
        response:
          anyOf:
            - type: string
            - type: 'null'
          title: Response
          description: The reply the agent generated
        tool_calls:
          anyOf:
            - items:
                $ref: '#/components/schemas/ToolCallOutput'
              type: array
            - type: 'null'
          title: Tool Calls
          description: Tool calls the agent generated
      type: object
      title: TestOutput
    JudgeResult:
      properties:
        evaluator_uuid:
          anyOf:
            - type: string
              maxLength: 36
              minLength: 36
            - type: 'null'
          title: Evaluator Uuid
          description: ID of the evaluator that produced this verdict
        reasoning:
          anyOf:
            - type: string
            - type: 'null'
          title: Reasoning
          description: The judge's rationale for this verdict
        match:
          anyOf:
            - type: boolean
            - type: 'null'
          title: Match
          description: Pass/fail verdict, set for binary evaluators
        score:
          anyOf:
            - type: number
            - type: 'null'
          title: Score
          description: Numeric score, set for rating evaluators
        value_name:
          anyOf:
            - type: string
            - type: 'null'
          title: Value Name
          description: Readable label for the verdict, taken from the run's rubric
        variable_values:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: >-
            Values filled into the evaluator prompt's `{{variable}}`
            placeholders for this test case, keyed by variable name
          examples:
            - criteria: Is the reply correct and helpful?
      type: object
      title: JudgeResult
    ToolCallOutput:
      properties:
        tool:
          type: string
          title: Tool
          description: Name of the tool the agent called
        arguments:
          anyOf:
            - additionalProperties: true
              type: object
            - type: 'null'
          description: Arguments the agent passed to the tool
          examples:
            - city: Paris
        output:
          anyOf:
            - {}
            - type: 'null'
          title: Output
          description: >-
            Tool execution result, when the agent ran the tool and returned its
            result
      type: object
      required:
        - tool
      title: ToolCallOutput
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-Key
      description: API key for authentication

````