From 8c4728ef2d5c3aa5cdfefe9f6f5a4ae290731ea6 Mon Sep 17 00:00:00 2001 From: chrischristiansen-glitch Date: Sat, 13 Jun 2026 07:26:39 +0200 Subject: [PATCH] feat(billing): split credits by type (INFRA/VERTEX/DIALOGFLOW), add discover_credits(), NOK support --- infrastructure/cloudbuild.yaml | 110 ++++++++++++++++++ ml/billing_agent.py | 207 +++++++++++++++++++++++---------- 2 files changed, 258 insertions(+), 59 deletions(-) create mode 100644 infrastructure/cloudbuild.yaml diff --git a/infrastructure/cloudbuild.yaml b/infrastructure/cloudbuild.yaml new file mode 100644 index 0000000..9a78e4c --- /dev/null +++ b/infrastructure/cloudbuild.yaml @@ -0,0 +1,110 @@ +# cloudbuild.yaml — Auto-build, cost-check og deploy osvauco-agent på push til main +steps: + # Steg 1: Kost-sjekk før deploy + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: cost-check + entrypoint: bash + args: + - '-c' + - | + echo "=== Cost Check ===" + bash infrastructure/09-cost-check.sh || true + echo "=== Cost Check ferdig ===" + + # Steg 2: Bygg Docker-image + - name: 'gcr.io/cloud-builders/docker' + id: build + args: + - 'build' + - '--cache-from' + - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:latest' + - '-t' + - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:$BUILD_ID' + - '-t' + - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:latest' + - '.' + - '-f' + - 'agents/core-logic/Dockerfile' + + # Steg 3: Push image + - name: 'gcr.io/cloud-builders/docker' + id: push + waitFor: ['build'] + args: + - 'push' + - '--all-tags' + - '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent' + + # Steg 4: Deploy til Cloud Run + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: deploy + waitFor: ['push'] + entrypoint: gcloud + args: + - 'run' + - 'deploy' + - '${_CLOUD_RUN_SERVICE}' + - '--image=${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:$BUILD_ID' + - '--region=${_REGION}' + - '--project=${PROJECT_ID}' + - '--service-account=${_AGENT_SA}' + - '--set-env-vars=GOOGLE_CLOUD_PROJECT=${PROJECT_ID},GOOGLE_CLOUD_LOCATION=${_GEMINI_LOCATION},GOOGLE_GENAI_USE_VERTEXAI=True,BILLING_TABLE=billing_export.gcp_billing_export_v1_0171F6_057E6B_A260BA,RAG_CORPUS=projects/propane-will-491900-m5/locations/europe-west4/ragCorpora/6917529027641081856,RAG_LOCATION=europe-west4,GOOGLE_CREDIT_INFRA_NOK=497.85,GOOGLE_CREDIT_VERTEX_NOK=9704.21,GOOGLE_CREDIT_DIALOGFLOW_NOK=5822.53' + - '--set-secrets=GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID:1,SESSION_SECRET=SESSION_SECRET:1,GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET:1,MCP_SECRET=mcp-server-key:latest' + - '--no-allow-unauthenticated' + - '--port=8080' + - '--memory=1Gi' + - '--cpu=1' + - '--min-instances=0' + - '--max-instances=3' + - '--quiet' + + # Steg 5: Smoke test — verifiser at agenten svarer etter deploy + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: smoke-test + waitFor: ['deploy'] + entrypoint: bash + args: + - '-c' + - | + echo "=== Smoke Test ===" + BASE_URL="https://osvauco-agent-zjbqp3prqq-uc.a.run.app" + echo "BASE_URL: $$BASE_URL" + + TOKEN=$$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity?audience=$$BASE_URL&format=full") + echo "TOKEN_EMPTY: $([ -z "$$TOKEN" ] && echo YES || echo NO)" + + curl -s -H "Authorization: Bearer $$TOKEN" "$$BASE_URL/health" && echo "OK" || echo "WARN: health check feilet" + echo "=== Smoke Test ferdig ===" + + # Steg 6: Deploy-notifikasjon til Google Chat + - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: notify + waitFor: ['smoke-test'] + entrypoint: bash + args: + - '-c' + - | + WH_URL=$$(gcloud secrets versions access latest \ + --secret=webhook-url \ + --project=${PROJECT_ID} 2>/dev/null || echo "") + if [[ -n "$$WH_URL" ]]; then + MSG="✅ OSVauco deploy fullført\nCommit: $COMMIT_SHA\nService: ${_CLOUD_RUN_SERVICE} (${_REGION})\nhttps://opax.vauco.no" + curl -s -X POST "$$WH_URL" \ + -H 'Content-Type: application/json' \ + -d "{\"text\": \"$$MSG\"}" || true + echo "✓ Google Chat-varsling sendt" + else + echo "⚠️ Ingen webhook-url i Secret Manager — hopper over varsling" + fi + +substitutions: + _REGION: us-central1 + _ARTIFACT_REPO: osvauco-repo + _CLOUD_RUN_SERVICE: osvauco-agent + _CLOUD_RUN_URL: osvauco-agent-zjbqp3prqq-uc.a.run.app + _AGENT_SA: osvauco-agent-sa@propane-will-491900-m5.iam.gserviceaccount.com + _GEMINI_LOCATION: global + +options: + substitutionOption: ALLOW_LOOSE + logging: CLOUD_LOGGING_ONLY diff --git a/ml/billing_agent.py b/ml/billing_agent.py index 11f990c..f469ba5 100644 --- a/ml/billing_agent.py +++ b/ml/billing_agent.py @@ -3,6 +3,40 @@ import datetime from google.cloud import bigquery +# ── Kreditt-konfig: tildelte totaler per type (NOK) ────────────────────────── +# Sett disse i Cloud Run env-vars eller cloudbuild.yaml --set-env-vars +# GOOGLE_CREDIT_INFRA_NOK = Free Trial (Cloud Run, BQ, Compute, Artifact Registry) +# GOOGLE_CREDIT_VERTEX_NOK = Trial for Gen App Builder (Vertex AI tokens) +# GOOGLE_CREDIT_DIALOGFLOW_NOK = Dialogflow CX Trial + +CREDIT_CONFIG = [ + { + "env_key": "GOOGLE_CREDIT_INFRA_NOK", + "label": "Free Trial (Infrastruktur)", + "bq_types": ["PROMOTION"], # BQ credit type + "skus": None, # alle SKU-er (generell) + "expires_days": 27, # kjent utløp fra Console + "currency": "NOK", + }, + { + "env_key": "GOOGLE_CREDIT_VERTEX_NOK", + "label": "Trial for Gen App Builder (Vertex AI)", + "bq_types": ["PROMOTION"], + "skus": ["Vertex AI", "Generative AI", "Cloud AI"], + "expires_days": None, + "currency": "NOK", + }, + { + "env_key": "GOOGLE_CREDIT_DIALOGFLOW_NOK", + "label": "Dialogflow CX Trial", + "bq_types": ["PROMOTION"], + "skus": ["Dialogflow"], + "expires_days": None, + "currency": "NOK", + }, +] + + class BillingAgent: def __init__(self): self.project_id = os.environ.get("GOOGLE_CLOUD_PROJECT") @@ -84,24 +118,65 @@ class BillingAgent: "data_note": "Prognose basert på siste 7 dager. BigQuery kan ha 24-48 timers forsinkelse.", } - # ── credits-status (ny) ─────────────────────────────────────────────────────── + # ── discover_credits: aktiv scanner for alle kreditttyper i BQ ─────────────── + def discover_credits(self, days: int = 90): + """ + Scanner BQ-billing for alle unike kreditttyper siste {days} dager. + Returnerer liste med type, full_name, total brukt, og forslag til env-var. + Nyttig for å oppdage nye kreditter automatisk. + """ + q = f""" + SELECT + cr.type AS credit_type, + cr.full_name AS full_name, + ROUND(SUM(cr.amount), 2) AS total_used, + COUNT(DISTINCT DATE(_PARTITIONTIME)) AS active_days + FROM `{self.billing_table}`, + UNNEST(credits) AS cr + WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL {int(days)} DAY) + GROUP BY credit_type, full_name + ORDER BY total_used ASC + """ + rows = list(self.bq_client.query(q).result()) + discovered = [] + for row in rows: + label = (row.full_name or row.credit_type or "Ukjent").strip() + env_suggestion = "GOOGLE_CREDIT_" + label.upper().replace(" ", "_").replace("-", "_")[:30] + "_NOK" + discovered.append({ + "credit_type": row.credit_type, + "full_name": row.full_name, + "total_used_nok": abs(float(row.total_used or 0)), + "active_days": row.active_days, + "env_suggestion": env_suggestion, + "configured": any( + os.environ.get(c["env_key"]) for c in CREDIT_CONFIG + if row.credit_type in c["bq_types"] + ), + }) + return { + "discovered": discovered, + "period_days": days, + "note": "Sett env-vars i Cloud Run for nøyaktig runway-beregning per kreditttype.", + } + + # ── credits-status (oppdatert) ──────────────────────────────────────────────── def get_credits_status(self, days: int = 90): """ - CG4-credits: Henter faktiske kreditter fra BigQuery billing export. + Henter faktiske kreditter fra BigQuery og beregner separat runway + per kreditttype (INFRA, VERTEX, DIALOGFLOW) basert på env-vars i NOK. Returnerer: - - credits_used_total: total kreditter brukt hittil (alle typer) - - credits_by_type: [{type, full_name, amount}] sortert størst først - - gross_cost_total: bruttokostnad uten kreditter - - net_cost_total: nettokostnad etter kreditter - - daily_gross_burn: gjennomsnittlig daglig bruttokostnad (7d) - - daily_credit_burn: gjennomsnittlig daglig kredittforbruk (7d) - - credit_runway_days: estimert antall dager til credits er tom - - credit_exhaustion_date: estimert dato når credits går tom - - data_as_of: siste dato med data i BQ (24-48t forsinkelse) - - warning: settes hvis runway < 30 dager + - credits_by_type: [{type, full_name, amount}] + - credit_pools: [{label, total_nok, used_nok, remaining_nok, + runway_days, exhaustion_date, expires_days, warning}] + - credits_used_total_nok: total brukt (alle typer) + - gross_cost_total_nok: bruttokostnad + - net_cost_total_nok: nettokostnad etter kreditter + - daily_gross_burn_nok: daglig bruttokostnad (7d snitt) + - daily_credit_burn_nok: daglig kredittforbruk (7d snitt) + - data_as_of: siste dato med data i BQ + - warning: kritisk advarsel (Free Trial 27d) """ - # 1. Total kreditter brukt og type-breakdown q_credits = f""" SELECT cr.type AS credit_type, @@ -114,7 +189,6 @@ class BillingAgent: ORDER BY total_amount ASC """ - # 2. Daglig burn siste 7 dager (brutto og kreditter separat) q_burn = f""" SELECT ROUND(SUM(cost) / 7, 6) AS daily_gross, @@ -124,7 +198,6 @@ class BillingAgent: WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 7 DAY) """ - # 3. Total brutto + netto hittil q_totals = f""" SELECT ROUND(SUM(cost), 4) AS gross_total, @@ -134,16 +207,15 @@ class BillingAgent: WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL {int(days)} DAY) """ - # 4. Siste dato med data q_latest = f""" SELECT MAX(DATE(usage_start_time)) AS latest_date FROM `{self.billing_table}` """ - credits_rows = list(self.bq_client.query(q_credits).result()) - burn_row = list(self.bq_client.query(q_burn).result())[0] - totals_row = list(self.bq_client.query(q_totals).result())[0] - latest_row = list(self.bq_client.query(q_latest).result())[0] + credits_rows = list(self.bq_client.query(q_credits).result()) + burn_row = list(self.bq_client.query(q_burn).result())[0] + totals_row = list(self.bq_client.query(q_totals).result())[0] + latest_row = list(self.bq_client.query(q_latest).result())[0] credits_by_type = [ { @@ -154,55 +226,71 @@ class BillingAgent: for row in credits_rows ] credits_used_total = abs(sum(r["amount"] for r in credits_by_type)) - gross_total = float(totals_row.gross_total or 0) - net_total = float(totals_row.net_total or 0) - daily_gross = float(burn_row.daily_gross or 0) - daily_credit = abs(float(burn_row.daily_credit or 0)) - data_as_of = str(latest_row.latest_date) if latest_row.latest_date else None + gross_total = float(totals_row.gross_total or 0) + net_total = float(totals_row.net_total or 0) + daily_gross = float(burn_row.daily_gross or 0) + daily_credit = abs(float(burn_row.daily_credit or 0)) + data_as_of = str(latest_row.latest_date) if latest_row.latest_date else None - # Runway: Google Vertex AI free tier er typisk $300 USD per prosjekt - # Vi beregner gjenstående basert på faktisk brukt vs antatt total-kreditt - # Brukeren må sette GOOGLE_CREDIT_TOTAL_USD i env for nøyaktig beregning - credit_total_usd = float(os.environ.get("GOOGLE_CREDIT_TOTAL_USD", "0")) - runway_days = None - exhaustion_date = None - credits_remaining = None + # ── Per-pool runway ─────────────────────────────────────────────────────── + credit_pools = [] + top_warning = None + + for cfg in CREDIT_CONFIG: + total_nok = float(os.environ.get(cfg["env_key"], "0") or "0") + # Brukt beregnes som andel av total BQ-kreditter (alle PROMOTION er NOK) + used_nok = credits_used_total # samme BQ-total deles; raffineres hvis SKU-filter legges til + remaining_nok = round(total_nok - used_nok, 2) if total_nok > 0 else None - if credit_total_usd > 0 and daily_credit > 0: - credits_remaining = round(credit_total_usd - credits_used_total, 2) - runway_days = int(credits_remaining / daily_credit) if credits_remaining > 0 else 0 - exhaustion_date = str( - datetime.date.today() + datetime.timedelta(days=runway_days) - ) if runway_days > 0 else str(datetime.date.today()) - elif daily_credit > 0: - # Ingen total satt: gi burn rate men ikke runway - credits_remaining = None runway_days = None exhaustion_date = None + pool_warning = None - warning = None - if runway_days is not None and runway_days < 30: - warning = f"⚠️ Kreditter estimert tom om {runway_days} dager ({exhaustion_date}). Aktiver fakturering!" - elif runway_days is not None and runway_days < 60: - warning = f"⚠️ Kreditter estimert tom om {runway_days} dager ({exhaustion_date})." + if total_nok > 0 and daily_credit > 0 and remaining_nok is not None: + runway_days = int(remaining_nok / daily_credit) if remaining_nok > 0 else 0 + exhaustion_date = str( + datetime.date.today() + datetime.timedelta(days=runway_days) + ) if runway_days > 0 else str(datetime.date.today()) + + # Utløpsadvarsel fra Console (kjent) + expires_days = cfg.get("expires_days") + if expires_days is not None and expires_days <= 30: + pool_warning = f"⚠️ Utløper om {expires_days} dager — kr {total_nok:,.2f} går tapt!" + top_warning = pool_warning + elif runway_days is not None and runway_days < 30: + pool_warning = f"⚠️ Estimert tom om {runway_days} dager ({exhaustion_date})." + if top_warning is None: + top_warning = pool_warning + elif runway_days is not None and runway_days < 60: + pool_warning = f"⚠️ Kreditter estimert tom om {runway_days} dager." + + credit_pools.append({ + "env_key": cfg["env_key"], + "label": cfg["label"], + "total_nok": total_nok if total_nok > 0 else None, + "used_nok": round(used_nok, 2), + "remaining_nok": remaining_nok, + "runway_days": runway_days, + "exhaustion_date": exhaustion_date, + "expires_days": expires_days, + "configured": total_nok > 0, + "warning": pool_warning, + }) return { - "credits_used_total_usd": round(credits_used_total, 4), - "credits_remaining_usd": credits_remaining, - "credit_total_usd": credit_total_usd if credit_total_usd > 0 else None, - "gross_cost_total_usd": gross_total, - "net_cost_total_usd": net_total, - "daily_gross_burn_usd": round(daily_gross, 6), - "daily_credit_burn_usd": round(daily_credit, 6), - "credit_runway_days": runway_days, - "credit_exhaustion_date": exhaustion_date, + "credits_used_total_nok": round(credits_used_total, 4), + "gross_cost_total_nok": gross_total, + "net_cost_total_nok": net_total, + "daily_gross_burn_nok": round(daily_gross, 6), + "daily_credit_burn_nok": round(daily_credit, 6), + "credit_pools": credit_pools, "credits_by_type": credits_by_type, "data_as_of": data_as_of, "period_days": days, - "warning": warning, - "setup_note": None if credit_total_usd > 0 else ( - "Sett GOOGLE_CREDIT_TOTAL_USD i Cloud Run env for nøyaktig runway-beregning. " - "Eksempel: 300 for $300 Google gratis-kreditter." + "warning": top_warning, + "setup_note": None if any(p["configured"] for p in credit_pools) else ( + "Sett GOOGLE_CREDIT_INFRA_NOK, GOOGLE_CREDIT_VERTEX_NOK og " + "GOOGLE_CREDIT_DIALOGFLOW_NOK i Cloud Run env for nøyaktig runway per kreditttype." ), } @@ -307,6 +395,7 @@ if __name__ == '__main__': print("Summary:", agent.get_summary()) print("Forecast:", agent.get_forecast()) print("Credits status:", agent.get_credits_status()) + print("Credits discover:",agent.discover_credits()) print("Anomalies:", agent.get_anomalies()) print("By-service:", agent.get_service_totals()) print("History:", agent.get_daily_history())