Introduction to Akhara Evaluation

from akhara import Akhara

client = Akhara(api_key="your-api-key")

client.samples.create(
    dataset="support-agent-golden",
    input={"query": "I need to cancel my order and get a refund."},
    output={
        "response": "I can help with that. I've started the cancellation and refund.",
        "actions": ["cancel_order", "issue_refund"],
    },
    expected={
        "must_include": ["cancellation", "refund"],
        "must_not": ["upsell"],
    },
)

evaluation = client.evaluations.create(
    name="support-agent-regression",
    dataset="support-agent-golden",
    evaluators=[
        {"type": "contains_all", "config": {"fields": ["must_include"]}},
        {"type": "llm_judge", "config": {"rubric": "task_completion_v1"}},
    ],
)

print(evaluation.id)