#!/usr/bin/env python3
"""
run_eval.py — Run agent evaluation using Vertex AI Gen AI Evaluation Service.
Metrics: coherence, groundedness, tool_call_quality
CI/CD gate: fails build if groundedness < 0.8
"""

import vertexai
from vertexai.evaluation import EvalTask

PROJECT_ID = "propane-will-491900-m5"
LOCATION = "us-central1"

vertexai.init(project=PROJECT_ID, location=LOCATION)

EVAL_DATASET = [
    {
        "prompt": "What GCP region should all resources use?",
        "reference": "us-central1",
    },
    {
        "prompt": "What command tears down all Agent Runtimes?",
        "reference": "Run 03-teardown.sh",
    },
    {
        "prompt": "What ADK version is required for Memory Bank?",
        "reference": "google-adk >= 2.0.0",
    },
]

METRICS = ["coherence", "groundedness", "tool_call_quality"]

eval_task = EvalTask(
    dataset=EVAL_DATASET,
    metrics=METRICS,
    experiment="oavauco-agent-eval",
)

result = eval_task.evaluate(
    model="gemini-2.5-flash",
    prompt_template="{prompt}",
)

print(result.summary_metrics)

# CI/CD gate
if result.summary_metrics.get("groundedness/mean", 1.0) < 0.8:
    print("EVAL FAILED: groundedness below 0.8 threshold")
    raise SystemExit(1)

print("EVAL PASSED")