05030edf61
Per the task in `docs/TODO.md`, this change runs `git update-index --chmod=+x` on all relevant `.sh` and `.py` files in the repository. This prevents intermittent 'Permission denied' errors when scripts are run in CI/CD environments or after being edited via the GitHub web interface, which can strip file permissions.
52 lines
1.2 KiB
Python
Executable File
52 lines
1.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
run_eval.py — Run agent evaluation using Vertex AI Gen AI Evaluation Service.
|
|
Metrics: coherence, groundedness, tool_call_quality
|
|
CI/CD gate: fails build if groundedness < 0.8
|
|
"""
|
|
|
|
import vertexai
|
|
from vertexai.evaluation import EvalTask
|
|
|
|
PROJECT_ID = "propane-will-491900-m5"
|
|
LOCATION = "us-central1"
|
|
|
|
vertexai.init(project=PROJECT_ID, location=LOCATION)
|
|
|
|
EVAL_DATASET = [
|
|
{
|
|
"prompt": "What GCP region should all resources use?",
|
|
"reference": "us-central1",
|
|
},
|
|
{
|
|
"prompt": "What command tears down all Agent Runtimes?",
|
|
"reference": "Run 03-teardown.sh",
|
|
},
|
|
{
|
|
"prompt": "What ADK version is required for Memory Bank?",
|
|
"reference": "google-adk >= 2.0.0",
|
|
},
|
|
]
|
|
|
|
METRICS = ["coherence", "groundedness", "tool_call_quality"]
|
|
|
|
eval_task = EvalTask(
|
|
dataset=EVAL_DATASET,
|
|
metrics=METRICS,
|
|
experiment="oavauco-agent-eval",
|
|
)
|
|
|
|
result = eval_task.evaluate(
|
|
model="gemini-2.5-flash",
|
|
prompt_template="{prompt}",
|
|
)
|
|
|
|
print(result.summary_metrics)
|
|
|
|
# CI/CD gate
|
|
if result.summary_metrics.get("groundedness/mean", 1.0) < 0.8:
|
|
print("EVAL FAILED: groundedness below 0.8 threshold")
|
|
raise SystemExit(1)
|
|
|
|
print("EVAL PASSED")
|