LLM Evaluation

calibrate llm tests run -c <config_file> -o <output_dir> -m <model> -p <provider>

calibrate llm tests run -c ./config.json -o ./out

calibrate llm tests run -c ./config.json -o ./out -m openai/gpt-4.1 -p openrouter

calibrate llm tests run -c ./config.json -o ./out -m gpt-4.1 -p openai

{
  "system_prompt": "You are a helpful assistant filling out a form...",
  "tools": [
    {
      "type": "client",
      "name": "plan_next_question",
      "description": "Plan the next question to ask",
      "parameters": [
        {
          "id": "next_unanswered_question_index",
          "type": "integer",
          "description": "Index of next question",
          "required": true
        }
      ]
    }
  ],
  "test_cases": [
    {
      "history": [
        { "role": "assistant", "content": "Hello! What is your name?" },
        { "role": "user", "content": "Aman Dalmia" }
      ],
      "evaluation": {
        "type": "tool_call",
        "tool_calls": [
          {
            "tool": "plan_next_question",
            "arguments": {
              "next_unanswered_question_index": 2,
              "questions_answered": [1]
            }
          }
        ]
      }
    },
    {
      "history": [
        { "role": "assistant", "content": "What is your phone number?" },
        { "role": "user", "content": "Can I skip this question?" }
      ],
      "evaluation": {
        "type": "response",
        "criteria": "The assistant should allow the user to skip."
      }
    }
  ]
}

/path/to/output/<test_config_name>/<model_name>
├── results.json      # Detailed results for each test case
├── metrics.json      # Summary statistics (total, passed)
└── logs

calibrate llm tests leaderboard -o <output_dir> -s <save_dir>

calibrate llm tests leaderboard -o ./out -s ./leaderboard

# For OpenAI provider
export OPENAI_API_KEY=your_key

# For OpenRouter provider
export OPENROUTER_API_KEY=your_key