> ## Documentation Index
> Fetch the complete documentation index at: https://penseapp.vercel.app/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Create evaluator

> Create an evaluator along with its first version, which is set live



## OpenAPI

````yaml /api-reference/openapi.json post /evaluators
openapi: 3.1.0
info:
  title: Calibrate Public API
  version: 0.1.0
  description: Programmatic API for CI/automation. Pass your key in the `X-API-Key` header.
servers:
  - url: https://api.calibrate.artpark.ai
    description: Production
security: []
paths:
  /evaluators:
    post:
      tags:
        - evaluators
      summary: Create evaluator
      description: Create an evaluator along with its first version, which is set live
      operationId: create_evaluator_endpoint_evaluators_post
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluatorCreate'
      responses:
        '200':
          description: Successful Response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluatorCreateResponse'
        '422':
          description: Validation Error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/HTTPValidationError'
      security:
        - ApiKeyAuth: []
components:
  schemas:
    EvaluatorCreate:
      properties:
        name:
          type: string
          minLength: 1
          title: Name
          description: Evaluator name, unique within your workspace
        description:
          anyOf:
            - type: string
            - type: 'null'
          title: Description
          description: Description. Omit to leave blank
        evaluator_type:
          type: string
          enum:
            - tts
            - stt
            - llm
            - llm-general
            - conversation
          title: Evaluator Type
          description: |
            What the evaluator judges:

            - `tts`: TTS audio
            - `stt`: one transcript
            - `llm`: a reply with its conversation history
            - `llm-general`: a standalone input and output pair
            - `conversation`: a full conversation
          default: llm
        data_type:
          type: string
          enum:
            - text
            - audio
          title: Data Type
          description: |
            The modality the judge reads:

            - `text`
            - `audio`
          default: text
        output_type:
          type: string
          enum:
            - binary
            - rating
          title: Output Type
          description: |
            How the evaluator scores:

            - `binary`: pass or fail
            - `rating`: a numeric score, using the scale in `output_config`
          default: binary
        version:
          $ref: '#/components/schemas/EvaluatorVersionCreate'
          description: >-
            The evaluator's first version. Set as live when you create the
            evaluator
      type: object
      required:
        - name
        - version
      title: EvaluatorCreate
    EvaluatorCreateResponse:
      properties:
        uuid:
          type: string
          maxLength: 36
          minLength: 36
          title: Uuid
          description: ID of the created evaluator
          examples:
            - f47ac10b-58cc-4372-a567-0e02b2c3d479
        version_uuid:
          type: string
          maxLength: 36
          minLength: 36
          title: Version Uuid
          description: ID of its initial version
          examples:
            - 6ba7b810-9dad-11d1-80b4-00c04fd430c8
      type: object
      required:
        - uuid
        - version_uuid
      title: EvaluatorCreateResponse
    HTTPValidationError:
      properties:
        detail:
          items:
            $ref: '#/components/schemas/ValidationError'
          type: array
          title: Detail
      type: object
      title: HTTPValidationError
    EvaluatorVersionCreate:
      properties:
        judge_model:
          type: string
          title: Judge Model
          description: >-
            The model that runs the judge, named the way its provider does, for
            example `openai/gpt-4.1` or `anthropic/claude-sonnet-4`
        system_prompt:
          type: string
          title: System Prompt
          description: Judge system prompt. May contain `{{variable}}` placeholders
        output_config:
          anyOf:
            - $ref: '#/components/schemas/OutputConfig'
            - type: 'null'
          description: >-
            The scale points and their labels. Required for a `rating`
            evaluator. A `binary` evaluator uses the default Correct/Wrong
            labels unless you set your own
        variables:
          anyOf:
            - items:
                $ref: '#/components/schemas/VariableSpec'
              type: array
            - type: 'null'
          title: Variables
          description: >-
            Declared prompt variables. Omit if the prompt has no
            `{{placeholders}}`
      type: object
      required:
        - judge_model
        - system_prompt
      title: EvaluatorVersionCreate
      description: >-
        One version of an evaluator: its judge prompt, model, variables, and
        rubric.
    ValidationError:
      properties:
        loc:
          items:
            anyOf:
              - type: string
              - type: integer
          type: array
          title: Location
        msg:
          type: string
          title: Message
        type:
          type: string
          title: Error Type
      type: object
      required:
        - loc
        - msg
        - type
      title: ValidationError
    OutputConfig:
      properties:
        scale:
          anyOf:
            - items:
                $ref: '#/components/schemas/OutputScaleEntry'
              type: array
            - type: 'null'
          title: Scale
          description: >-
            The ordered scale points that make up the rubric, each with its
            label
      type: object
      title: OutputConfig
    VariableSpec:
      properties:
        name:
          type: string
          title: Name
          description: Name of a `{{placeholder}}` used in the system prompt
        description:
          anyOf:
            - type: string
            - type: 'null'
          title: Description
          description: What the variable is for. Omit if self-evident
        default:
          anyOf:
            - type: string
            - type: 'null'
          title: Default
          description: Default value used when you omit this variable. Omit for no default
      type: object
      required:
        - name
      title: VariableSpec
    OutputScaleEntry:
      properties:
        value:
          title: Value
          description: >-
            The value for this scale point. Use a boolean for a `binary`
            evaluator, a number for a `rating` one
        name:
          type: string
          title: Name
          description: Short label for this scale point
        description:
          anyOf:
            - type: string
            - type: 'null'
          title: Description
          description: >-
            Rubric text for this level, added to the judge prompt. Omit to leave
            this level undescribed
        color:
          anyOf:
            - type: string
            - type: 'null'
          title: Color
          description: Color to show for this level. Omit for the default
      type: object
      required:
        - value
        - name
      title: OutputScaleEntry
      description: >-
        One entry in an evaluator's output_config.scale. `value` is
        bool|number|string

        depending on output_type.
  securitySchemes:
    ApiKeyAuth:
      type: apiKey
      in: header
      name: X-API-Key
      description: API key for authentication

````