Text Simulations

calibrate llm simulations run -c <config_file> -o <output_dir> -m <model> -p <provider> -n <parallel>

calibrate llm simulations run -c ./config.json -o ./out

calibrate llm simulations run -c ./config.json -o ./out -n 4

calibrate llm simulations run -c ./config.json -o ./out -m openai/gpt-4.1 -p openrouter

{
  "system_prompt": "You are a helpful nurse filling out a form...",
  "tools": [
    {
      "type": "client",
      "name": "plan_next_question",
      "description": "Plan the next question",
      "parameters": [
        {
          "id": "next_unanswered_question_index",
          "type": "integer",
          "description": "Next question index",
          "required": true
        }
      ]
    }
  ],
  "personas": [
    {
      "characteristics": "A shy mother named Geeta, 39 years old, gives short answers",
      "gender": "female",
      "language": "english"
    },
    {
      "characteristics": "An elderly farmer who speaks slowly and asks for clarification",
      "gender": "male",
      "language": "hindi"
    }
  ],
  "scenarios": [
    { "description": "User completes the form without any issues" },
    { "description": "User hesitates and wants to skip some questions" }
  ],
  "evaluation_criteria": [
    {
      "name": "question_completeness",
      "description": "Whether all the questions in the form were covered"
    },
    {
      "name": "assistant_behavior",
      "description": "Whether the assistant asks one concise question per turn"
    }
  ],
  "settings": {
    "agent_speaks_first": true,
    "max_turns": 50
  }
}

/path/to/output/
├── simulation_persona_1_scenario_1/
│   ├── transcript.json          # Full conversation
│   ├── evaluation_results.csv   # Per-criterion results
│   ├── config.json              # Persona and scenario used
│   ├── results.log
│   └── logs
├── simulation_persona_1_scenario_2/
│   └── ...
├── results.csv                  # Aggregated results
└── metrics.json                 # Summary statistics

[
  { "role": "assistant", "content": "Hello! What is your name?" },
  { "role": "user", "content": "My name is Geeta." },
  { "role": "assistant", "content": "Thank you, Geeta. What is your address?" }
]

calibrate llm simulations leaderboard -o <output_dir> -s <save_dir>

calibrate llm simulations leaderboard -o ./out -s ./leaderboard

# For OpenAI provider
export OPENAI_API_KEY=your_key

# For OpenRouter provider
export OPENROUTER_API_KEY=your_key