Evaluations API

GET /evaluations/tasks

curl -s https://agi.akhara.ai/evaluations/tasks | python3 -m json.tool | head

POST /evaluations/run

{
  "env_id": "amazon_gym_android",
  "model": "my-agent-v1",
  "task_ids": ["vu.checkout.t098", "vu.search.add.t309"],
  "rollouts_per_example": 1,
  "webhook_url": null,
  "endpoint": {
    "url": "https://your-agent.example.com",
    "key": null,
    "model": "gpt-4.1",
    "use_wrapper": false,
    "send_tool_definitions": false,
    "use_openai_chat": false,
    "minimal_observation": null,
    "additional_params": null
  }
}

{
  "run_id": "e9eb19e7-187d-4bc4-bb59-a187807e4568",
  "status": "RUNNING",
  "status_url": "/evaluations/run/e9eb19e7-187d-4bc4-bb59-a187807e4568/status",
  "message": "Evaluation started. Poll status_url for progress and results."
}

GET /evaluations/run/{run_id}/status

curl -s "https://agi.akhara.ai/evaluations/run/${RUN_ID}/status" | python3 -m json.tool

{
  "run_id": "e9eb19e7-187d-4bc4-bb59-a187807e4568",
  "status": "COMPLETED",
  "progress": {
    "completed_rollouts": 2,
    "total_rollouts": 2,
    "current_episode_id": null,
    "current_task_id": null
  },
  "env_id": "amazon_gym_android",
  "model": "my-agent-v1",
  "started_at": "2026-07-30T01:00:00Z",
  "completed_at": "2026-07-30T01:12:00Z",
  "evaluation_id": "a1b2c3d4",
  "run_output_path": "outputs/runs/e9eb19e7-187d-4bc4-bb59-a187807e4568",
  "evaluations": [
    {
      "id": "a1b2c3d4",
      "results": [
        {
          "example_id": 0,
          "rollout_id": "e9eb19e7-187d-4bc4-bb59-a187807e4568_0_0",
          "episode_id": "ep_c8c22030d0",
          "task": "vu.checkout.t098",
          "reward": 1.0,
          "task_passed": true,
          "task_outcome": "pass",
          "is_completed": true,
          "is_truncated": false,
          "stop_condition": "terminal_action",
          "verifiers": [],
          "error": null
        }
      ]
    }
  ]
}

DELETE /evaluations/run/{run_id}

GET /evaluations?environment_id=amazon_gym_android&limit=20
GET /evaluations/{evaluation_id}

POST /runs/open

{
  "env_id": "amazon_gym_android",
  "task_definition": "Browse headphones and add any pair to cart.",
  "max_steps": 20,
  "profile_id": "U_001",
  "rollouts": 1,
  "endpoint": {
    "url": "https://your-agent.example.com"
  }
}