Deploy evaluations in CI/CD

name: Agent Regression Gate
version: 1.2.0

evaluators:
  - type: task_completion
    version: 1.0.0
  - type: policy_adherence
    version: 1.0.0
    config:
      policy_set: agent_policy_v3

gates:
  - name: task_completion_gate
    metric: task_completion
    operator: gte
    threshold: 0.85

  - name: policy_gate
    metric: policy_adherence
    operator: gte
    threshold: 0.95

baseline:
  type: production
  model_tag: production-current
  max_regression:
    task_completion: 0.02
    policy_adherence: 0.01

#!/usr/bin/env python3
"""Run Akhara evaluation as part of CI/CD pipeline."""

import argparse
import json
import os
import subprocess
import sys
import yaml
from akhara import Akhara

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--model-path', required=True)
    parser.add_argument('--config', required=True)
    parser.add_argument('--dataset', required=True)
    parser.add_argument('--output', required=True)
    parser.add_argument('--timeout', type=int, default=3600)
    args = parser.parse_args()

    with open(args.config) as f:
        config = yaml.safe_load(f)

    client = Akhara()
    model_version = get_model_version(args.model_path)

    evaluation = client.evaluations.create(
        name=f"CI/CD Evaluation - {model_version}",
        dataset=args.dataset,
        model_version=model_version,
        evaluators=config['evaluators'],
        ci_mode=True,
        tags=["ci-cd", f"commit:{get_git_commit()}"],
        metadata={
            "git_commit": get_git_commit(),
            "git_branch": get_git_branch(),
            "pipeline_run": os.environ.get('GITHUB_RUN_ID', os.environ.get('CI_PIPELINE_ID', 'local')),
        },
    )

    try:
        evaluation.wait(timeout=args.timeout, stage="automated")
    except TimeoutError:
        print(f"Evaluation timed out after {args.timeout}s")
        sys.exit(1)

    results = client.evaluations.get(evaluation.id)
    gates_passed = check_gates(results, config['gates'])
    regression_check = None
    if 'baseline' in config:
        regression_check = check_regression(client, results, config['baseline'])

    output = {
        "evaluation_id": evaluation.id,
        "model_version": model_version,
        "scores": getattr(results, "scores", {}),
        "gates_passed": gates_passed,
        "regression_check": regression_check,
        "report_url": getattr(results, "dashboard_url", None),
    }

    with open(args.output, 'w') as f:
        json.dump(output, f, indent=2, default=str)

    if not gates_passed:
        sys.exit(1)
    if regression_check and not regression_check['passed']:
        sys.exit(1)


def check_gates(results, gates):
    all_passed = True
    scores = results.scores if hasattr(results, "scores") else results.get("scores", {})
    for gate in gates:
        metric_value = getattr(scores, gate['metric'], None)
        if metric_value is None and isinstance(scores, dict):
            metric_value = scores.get(gate['metric'])
        if gate['operator'] == 'gte':
            passed = metric_value >= gate['threshold']
        elif gate['operator'] == 'lte':
            passed = metric_value <= gate['threshold']
        elif gate['operator'] == 'eq':
            passed = metric_value == gate['threshold']
        else:
            passed = False
        print(f"{'PASS' if passed else 'FAIL'} {gate['name']}: {metric_value}")
        if not passed:
            all_passed = False
    return all_passed


def check_regression(client, results, baseline_config):
    baseline = client.evaluations.get_by_tag(baseline_config['model_tag'])
    if not baseline:
        print("No baseline found, skipping regression check")
        return None
    regression_found = False
    details = {}
    for metric, max_drop in baseline_config['max_regression'].items():
        current = getattr(results.scores, metric)
        previous = getattr(baseline.scores, metric)
        diff = current - previous
        if diff < -max_drop:
            regression_found = True
        details[metric] = {"baseline": previous, "current": current, "diff": diff}
    return {"passed": not regression_found, "details": details}


def get_model_version(path):
    import hashlib
    return f"model-{hashlib.sha256(open(path, 'rb').read()).hexdigest()[:8]}"

def get_git_commit():
    return subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip()[:8]

def get_git_branch():
    return subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode().strip()


if __name__ == '__main__':
    main()

name: Model Evaluation

on:
  pull_request:
    paths:
      - 'models/**'
      - 'prompts/**'
      - 'evaluations/**'
      - 'data/**'
      - 'scripts/run_evaluation.py'
  push:
    branches: [main]
    paths:
      - 'models/**'
      - 'prompts/**'
      - 'evaluations/**'
      - 'data/**'
      - 'scripts/run_evaluation.py'

env:
  AKHARA_API_KEY: ${{ secrets.AKHARA_API_KEY }}

jobs:
  evaluate:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Akhara SDK
        run: pip install akhara pyyaml

      - name: Run evaluation suite
        run: |
          python scripts/run_evaluation.py \
            --model-path models/candidate.bin \
            --config evaluations/regression_gate.yaml \
            --dataset golden-set \
            --output evaluation_results.json

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: evaluation-report
          path: evaluation_results.json

      - name: Comment on PR
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const results = JSON.parse(fs.readFileSync('./evaluation_results.json', 'utf8'));
            const scores = results.scores || {};
            const lines = Object.entries(scores)
              .map(([k, v]) => `| ${k} | ${typeof v === 'number' ? v.toFixed(3) : v} |`)
              .join('\n');
            const passed = results.gates_passed ? 'passed' : 'failed';
            const body = `## Evaluation results\n\nGates **${passed}**.\n\n| Metric | Value |\n|--------|-------|\n${lines}\n\n[View in dashboard](${results.report_url || 'https://app.akhara.ai'})`;
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body,
            });

stages:
  - evaluate
  - deploy

evaluate:
  stage: evaluate
  image: python:3.11
  variables:
    AKHARA_API_KEY: $AKHARA_API_KEY
  script:
    - pip install akhara pyyaml
    - >
      python scripts/run_evaluation.py
      --model-path models/candidate.bin
      --config evaluations/regression_gate.yaml
      --dataset golden-set
      --output evaluation_results.json
  artifacts:
    paths:
      - evaluation_results.json
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == "main"

deploy:
  stage: deploy
  script:
    - ./scripts/deploy.sh
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  needs:
    - evaluate