From 72d94c203227bd72819064d775d8ae7e29d9a711 Mon Sep 17 00:00:00 2001 From: Chris Christiansen Date: Tue, 26 May 2026 12:58:22 +0000 Subject: [PATCH] auto: sync --- infrastructure/00-authcheck.sh | 0 .../modules/vauco-bootstrap/main.tf | 340 +++++++++++++++++ .../modules/vauco-bootstrap/outputs.tf | 14 + .../modules/vauco-bootstrap/variables.tf | 35 ++ .../modules/vauco-bootstrap/versions.tf | 14 + infrastructure/terraform/main.tf | 349 +----------------- infrastructure/terraform/outputs.tf | 23 +- 7 files changed, 424 insertions(+), 351 deletions(-) mode change 100644 => 100755 infrastructure/00-authcheck.sh create mode 100644 infrastructure/modules/vauco-bootstrap/main.tf create mode 100644 infrastructure/modules/vauco-bootstrap/outputs.tf create mode 100644 infrastructure/modules/vauco-bootstrap/variables.tf create mode 100644 infrastructure/modules/vauco-bootstrap/versions.tf diff --git a/infrastructure/00-authcheck.sh b/infrastructure/00-authcheck.sh old mode 100644 new mode 100755 diff --git a/infrastructure/modules/vauco-bootstrap/main.tf b/infrastructure/modules/vauco-bootstrap/main.tf new file mode 100644 index 0000000..a31c4a2 --- /dev/null +++ b/infrastructure/modules/vauco-bootstrap/main.tf @@ -0,0 +1,340 @@ +resource "google_project_service" "apis" { + for_each = toset([ + "aiplatform.googleapis.com", + "storage.googleapis.com", + "cloudbilling.googleapis.com", + "cloudresourcemanager.googleapis.com", + "iam.googleapis.com", + "run.googleapis.com", + "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", + "cloudbuildv2.googleapis.com", + "secretmanager.googleapis.com", + "monitoring.googleapis.com", + "logging.googleapis.com", + "cloudtrace.googleapis.com", + "pubsub.googleapis.com", + "cloudfunctions.googleapis.com", + ]) + service = each.key + disable_dependent_services = true + project = var.project_id +} + +resource "google_service_account" "agent_sa" { + project = var.project_id + account_id = "${var.customer_id}-agent-sa" + display_name = "${var.customer_id} Agent Runner SA" +} + +resource "google_project_iam_member" "agent_sa_roles" { + for_each = toset([ + "roles/aiplatform.user", + "roles/storage.objectAdmin", + "roles/logging.logWriter", + "roles/cloudtrace.agent", + "roles/monitoring.metricWriter", + "roles/secretmanager.secretAccessor", + "roles/run.invoker", + ]) + project = var.project_id + role = each.key + member = "serviceAccount:${google_service_account.agent_sa.email}" +} + +resource "google_pubsub_topic" "billing_alerts" { + project = var.project_id + name = "${var.customer_id}-billing-alert-auto-teardown" +} + +resource "google_storage_bucket" "agent_staging" { + project = var.project_id + name = "${var.project_id}-${var.customer_id}-agent-staging" + location = var.region + force_destroy = true + + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 7 + } + } +} + +resource "google_billing_budget" "prod_budget" { + billing_account = var.billing_account_id + display_name = "${var.customer_id}-Agent-Budget-500USD" + + amount { + specified_amount { + currency_code = "NOK" + units = "2500" + } + } + + threshold_rules { + threshold_percent = 0.5 + } + + threshold_rules { + threshold_percent = 0.8 + } + + threshold_rules { + threshold_percent = 1.0 + } + + all_updates_rule { + pubsub_topic = google_pubsub_topic.billing_alerts.id + schema_version = "1.0" + } +} + +resource "google_billing_budget" "dev_budget" { + billing_account = var.billing_account_id + display_name = "${var.customer_id}-Dev-Budget-75USD" + + amount { + specified_amount { + currency_code = "NOK" + units = "500" + } + } + + threshold_rules { + threshold_percent = 0.7 + } + + threshold_rules { + threshold_percent = 0.9 + } + + threshold_rules { + threshold_percent = 1.0 + } + + all_updates_rule { + pubsub_topic = google_pubsub_topic.billing_alerts.id + schema_version = "1.0" + } +} + +resource "google_artifact_registry_repository" "repo" { + project = var.project_id + location = var.region + repository_id = "${var.customer_id}-agent-images" + format = "DOCKER" +} + +resource "google_cloud_run_v2_service" "agent_service" { + project = var.project_id + name = "${var.customer_id}-agent" + location = var.region + + template { + service_account = google_service_account.agent_sa.email + + scaling { + min_instance_count = 1 + max_instance_count = 10 + } + + containers { + image = var.container_image + + resources { + limits = { + cpu = "1" + memory = "1Gi" + } + } + + env { + name = "PROJECT_ID" + value = var.project_id + } + env { + name = "REGION" + value = var.region + } + } + + timeout = "300s" + max_instance_request_concurrency = 80 + } + + labels = { + env = "prod" + team = "osvaucoe" + customer = var.customer_id + } + + depends_on = [google_project_service.apis] +} + +resource "google_cloud_run_v2_service_iam_member" "invoker" { + project = var.project_id + location = google_cloud_run_v2_service.agent_service.location + name = google_cloud_run_v2_service.agent_service.name + role = "roles/run.invoker" + member = "user:chris.christiansen@vauco.no" # This should probably be a variable +} + +resource "google_bigquery_dataset" "logs" { + project = var.project_id + dataset_id = "${var.customer_id}_logs" + friendly_name = "${var.customer_id} Logs" + location = var.region + default_table_expiration_ms = 7776000000 # 90 dager +} + +resource "google_bigquery_table" "cloud_run_logs" { + project = var.project_id + dataset_id = google_bigquery_dataset.logs.dataset_id + table_id = "cloud_run_logs" + deletion_protection = false + + time_partitioning { + type = "DAY" + field = "timestamp" + expiration_ms = 7776000000 + } + + schema = jsonencode([ + { name = "timestamp", type = "TIMESTAMP", mode = "REQUIRED" }, + { name = "severity", type = "STRING", mode = "NULLABLE" }, + { name = "service_name", type = "STRING", mode = "NULLABLE" }, + { name = "revision", type = "STRING", mode = "NULLABLE" }, + { name = "http_method", type = "STRING", mode = "NULLABLE" }, + { name = "http_url", type = "STRING", mode = "NULLABLE" }, + { name = "http_status", type = "INTEGER", mode = "NULLABLE" }, + { name = "latency_ms", type = "FLOAT", mode = "NULLABLE" }, + { name = "message", type = "STRING", mode = "NULLABLE" }, + { name = "trace", type = "STRING", mode = "NULLABLE" }, + { name = "labels", type = "JSON", mode = "NULLABLE" } + ]) +} + +resource "google_project_iam_member" "bq_log_writer" { + project = var.project_id + role = "roles/bigquery.dataEditor" + member = "serviceAccount:${google_service_account.agent_sa.email}" +} + +resource "google_logging_project_sink" "cloudrun_to_bq" { + project = var.project_id + name = "${var.customer_id}-cloudrun-sink" + destination = "bigquery.googleapis.com/projects/${var.project_id}/datasets/${google_bigquery_dataset.logs.dataset_id}" + filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.customer_id}-agent\"" + unique_writer_identity = true +} + +resource "google_project_iam_member" "log_sink_bq_writer" { + project = var.project_id + role = "roles/bigquery.dataEditor" + member = google_logging_project_sink.cloudrun_to_bq.writer_identity +} + +resource "google_monitoring_notification_channel" "email" { + project = var.project_id + display_name = "${var.customer_id} Alert Email" + type = "email" + labels = { + email_address = var.alert_email + } +} + +resource "google_logging_metric" "agent_error_count" { + project = var.project_id + name = "${var.customer_id}-agent-error-count" + description = "Count of ERROR severity logs from agent" + filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"${var.customer_id}-agent\" AND severity=ERROR" +} + +resource "google_monitoring_uptime_check_config" "cloud_run_uptime" { + project = var.project_id + display_name = "${var.customer_id}-uptime" + http_check { + path = "/health" + port = 443 + use_ssl = true + } + monitored_resource { + type = "uptime_url" + labels = { + host = replace(google_cloud_run_v2_service.agent_service.uri, "https://", "") + } + } + timeout = "10s" + period = "60s" +} + +data "archive_file" "cost_guard_fn_zip" { + type = "zip" + source_dir = "${path.module}/cost-guard-fn" + output_path = "${path.module}/cost-guard-fn.zip" +} + +resource "google_storage_bucket_object" "cost_guard_fn_source" { + name = "${var.customer_id}-cost-guard-fn-source.zip" + bucket = google_storage_bucket.agent_staging.name + source = data.archive_file.cost_guard_fn_zip.output_path +} + +resource "google_cloudfunctions2_function" "billing_auto_teardown" { + project = var.project_id + name = "${var.customer_id}-billing-auto-teardown" + location = var.region + + build_config { + runtime = "python312" + entry_point = "billing_alert" + source { + storage_source { + bucket = google_storage_bucket.agent_staging.name + object = google_storage_bucket_object.cost_guard_fn_source.name + } + } + } + + service_config { + service_account_email = google_service_account.agent_sa.email + environment_variables = { + PROJECT_ID = var.project_id + CLOUD_RUN_SERVICE = "${var.customer_id}-agent" + REGION = var.region + } + } + + event_trigger { + trigger_region = "global" + event_type = "google.cloud.pubsub.topic.v1.messagePublished" + pubsub_topic = google_pubsub_topic.billing_alerts.id + } +} + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_project_iam_member" "function_sa_run_admin" { + project = var.project_id + role = "roles/run.admin" + member = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" +} + +resource "google_project_iam_member" "function_sa_secret_accessor" { + project = var.project_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" +} + +resource "google_billing_account_iam_member" "billing_viewers" { + for_each = toset(var.billing_viewer_emails) + + billing_account_id = var.billing_account_id + role = "roles/billing.viewer" + member = "user:${each.key}" +} diff --git a/infrastructure/modules/vauco-bootstrap/outputs.tf b/infrastructure/modules/vauco-bootstrap/outputs.tf new file mode 100644 index 0000000..2dea4d2 --- /dev/null +++ b/infrastructure/modules/vauco-bootstrap/outputs.tf @@ -0,0 +1,14 @@ +output "cloud_run_service_url" { + description = "The URL of the Cloud Run service." + value = google_cloud_run_v2_service.agent_service.uri +} + +output "service_account_email" { + description = "The email of the service account." + value = google_service_account.agent_sa.email +} + +output "artifact_registry_url" { + description = "The URL of the Artifact Registry repository." + value = "${google_artifact_registry_repository.repo.location}-docker.pkg.dev/${google_artifact_registry_repository.repo.project}/${google_artifact_registry_repository.repo.repository_id}" +} diff --git a/infrastructure/modules/vauco-bootstrap/variables.tf b/infrastructure/modules/vauco-bootstrap/variables.tf new file mode 100644 index 0000000..6098a91 --- /dev/null +++ b/infrastructure/modules/vauco-bootstrap/variables.tf @@ -0,0 +1,35 @@ +variable "customer_id" { + description = "The unique ID for the customer." + type = string +} + +variable "project_id" { + description = "The project ID to deploy the resources to." + type = string +} + +variable "region" { + description = "The region to deploy the resources to." + type = string + default = "europe-west1" +} + +variable "billing_account_id" { + description = "The billing account ID to associate with the project." + type = string +} + +variable "alert_email" { + description = "The email address for billing alerts." + type = string +} + +variable "billing_viewer_emails" { + description = "A list of user emails to be granted billing viewer role." + type = list(string) +} + +variable "container_image" { + description = "The container image to deploy." + type = string +} diff --git a/infrastructure/modules/vauco-bootstrap/versions.tf b/infrastructure/modules/vauco-bootstrap/versions.tf new file mode 100644 index 0000000..a20b1a3 --- /dev/null +++ b/infrastructure/modules/vauco-bootstrap/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.0" + } + archive = { + source = "hashicorp/archive" + version = ">= 2.2.0" + } + } +} diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 7ee386d..3b0ddfe 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -3,344 +3,15 @@ provider "google" { region = var.region } -resource "google_project_service" "apis" { - for_each = toset([ - "aiplatform.googleapis.com", - "storage.googleapis.com", - "cloudbilling.googleapis.com", - "cloudresourcemanager.googleapis.com", - "iam.googleapis.com", - "run.googleapis.com", - "artifactregistry.googleapis.com", - "cloudbuild.googleapis.com", - "cloudbuildv2.googleapis.com", - "secretmanager.googleapis.com", - "monitoring.googleapis.com", - "logging.googleapis.com", - "cloudtrace.googleapis.com", - "pubsub.googleapis.com", - "cloudfunctions.googleapis.com", - ]) - service = each.key - disable_dependent_services = true +module "vauco_bootstrap_medioteq" { + source = "../modules/vauco-bootstrap" + + customer_id = "medioteq" + project_id = var.project_id + region = var.region + billing_account_id = var.billing_account_id + alert_email = "billing@vauco.no" + billing_viewer_emails = ["chris.christiansen@vauco.no", "jason.vauger@vauco.no"] + container_image = "europe-west1-docker.pkg.dev/propane-will-491900-m5/agent-images/osvauco-agent:latest" } -resource "google_service_account" "agent_sa" { - account_id = "osvauco-agent-sa" - display_name = "OSVauco Agent Runner SA" -} - -resource "google_project_iam_member" "agent_sa_roles" { - for_each = toset([ - "roles/aiplatform.user", - "roles/storage.objectAdmin", - "roles/logging.logWriter", - "roles/cloudtrace.agent", - "roles/monitoring.metricWriter", - "roles/secretmanager.secretAccessor", - "roles/run.invoker", - ]) - project = var.project_id - role = each.key - member = "serviceAccount:${google_service_account.agent_sa.email}" -} - -resource "google_pubsub_topic" "billing_alerts" { - name = "billing-alert-auto-teardown" -} - -resource "google_storage_bucket" "agent_staging" { - name = "${var.project_id}-agent-staging" - location = var.region - force_destroy = true // Set to true for ephemeral staging buckets - - lifecycle_rule { - action { - type = "Delete" - } - condition { - age = 7 - } - } -} - -resource "google_billing_budget" "prod_budget" { - billing_account = var.billing_account_id - display_name = "OSVauco-Agent-Budget-500USD" - - amount { - specified_amount { - currency_code = "NOK" - units = "2500" - } - } - - threshold_rules { - threshold_percent = 0.5 - } - - threshold_rules { - threshold_percent = 0.8 - } - - threshold_rules { - threshold_percent = 1.0 - } - - all_updates_rule { - pubsub_topic = google_pubsub_topic.billing_alerts.id - schema_version = "1.0" - } -} - -resource "google_billing_budget" "dev_budget" { - billing_account = var.billing_account_id - display_name = "OSVauco-Dev-Budget-75USD" - - amount { - specified_amount { - currency_code = "NOK" - units = "500" - } - } - - threshold_rules { - threshold_percent = 0.7 - } - - threshold_rules { - threshold_percent = 0.9 - } - - threshold_rules { - threshold_percent = 1.0 - } - - all_updates_rule { - pubsub_topic = google_pubsub_topic.billing_alerts.id - schema_version = "1.0" - } -} - -# Managed via Cloud Budget alerts — not Pub/Sub push -# resource "google_pubsub_subscription" "billing_alerts_sub" { -# name = "billing-alert-subscription" -# topic = google_pubsub_topic.billing_alerts.name -# -# # Push to an email address -# push_config { -# push_endpoint = "mailto:${var.alert_email}" -# } -# } - -resource "google_artifact_registry_repository" "repo" { - location = var.artifact_region - repository_id = var.artifact_repo_name - format = "DOCKER" -} - -resource "google_cloud_run_v2_service" "agent_service" { - name = var.cloud_run_service_name - location = var.cloud_run_region - - template { - service_account = google_service_account.agent_sa.email - - scaling { - min_instance_count = 1 - max_instance_count = 10 - } - - containers { - image = var.container_image - - resources { - limits = { - cpu = "1" - memory = "1Gi" - } - } - - env { - name = "PROJECT_ID" - value = var.project_id - } - env { - name = "REGION" - value = var.region - } - } - - timeout = "300s" - max_instance_request_concurrency = 80 - - } - - labels = { - env = "prod" - team = "osvaucoe" - agent = "orchestrator" - } - - depends_on = [google_project_service.apis] -} - -resource "google_cloud_run_v2_service_iam_member" "invoker" { - project = var.project_id - location = google_cloud_run_v2_service.agent_service.location - name = google_cloud_run_v2_service.agent_service.name - role = "roles/run.invoker" - member = "user:chris.christiansen@vauco.no" -} - -# BigQuery — Cloud Run structured logs -resource "google_bigquery_dataset" "logs" { - dataset_id = "osvauco_logs" - friendly_name = "OSVauco Logs" - location = var.region - default_table_expiration_ms = 7776000000 # 90 dager -} - -resource "google_bigquery_table" "cloud_run_logs" { - dataset_id = google_bigquery_dataset.logs.dataset_id - table_id = "cloud_run_logs" - deletion_protection = false - - time_partitioning { - type = "DAY" - field = "timestamp" - expiration_ms = 7776000000 - } - - schema = jsonencode([ - { name = "timestamp", type = "TIMESTAMP", mode = "REQUIRED" }, - { name = "severity", type = "STRING", mode = "NULLABLE" }, - { name = "service_name", type = "STRING", mode = "NULLABLE" }, - { name = "revision", type = "STRING", mode = "NULLABLE" }, - { name = "http_method", type = "STRING", mode = "NULLABLE" }, - { name = "http_url", type = "STRING", mode = "NULLABLE" }, - { name = "http_status", type = "INTEGER", mode = "NULLABLE" }, - { name = "latency_ms", type = "FLOAT", mode = "NULLABLE" }, - { name = "message", type = "STRING", mode = "NULLABLE" }, - { name = "trace", type = "STRING", mode = "NULLABLE" }, - { name = "labels", type = "JSON", mode = "NULLABLE" } - ]) -} - -resource "google_project_iam_member" "bq_log_writer" { - project = var.project_id - role = "roles/bigquery.dataEditor" - member = "serviceAccount:${google_service_account.agent_sa.email}" -} - -resource "google_logging_project_sink" "cloudrun_to_bq" { - name = "osvauco-cloudrun-sink" - destination = "bigquery.googleapis.com/projects/${var.project_id}/datasets/${google_bigquery_dataset.logs.dataset_id}" - filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"osvauco-agent\"" - unique_writer_identity = true -} - -resource "google_project_iam_member" "log_sink_bq_writer" { - project = var.project_id - role = "roles/bigquery.dataEditor" - member = google_logging_project_sink.cloudrun_to_bq.writer_identity -} - -# From 04-observability-setup.sh -resource "google_monitoring_notification_channel" "email" { - display_name = "OSVauco Alert Email" - type = "email" - labels = { - email_address = var.alert_email - } -} - -resource "google_logging_metric" "agent_error_count" { - name = "agent-error-count" - description = "Count of ERROR severity logs from agent" - filter = "resource.type=\"cloud_run_revision\" severity=ERROR" -} - -resource "google_monitoring_uptime_check_config" "cloud_run_uptime" { - display_name = "${var.cloud_run_service_name}-uptime" - http_check { - path = "/health" - port = 443 - use_ssl = true - } - monitored_resource { - type = "uptime_url" - labels = { - host = replace(google_cloud_run_v2_service.agent_service.uri, "https://", "") - } - } - timeout = "10s" - period = "60s" -} - -# From 10-cost-guard.sh -data "archive_file" "cost_guard_fn_zip" { - type = "zip" - source_dir = "${path.module}/cost-guard-fn" - output_path = "${path.module}/cost-guard-fn.zip" -} - -resource "google_storage_bucket_object" "cost_guard_fn_source" { - name = "cost-guard-fn-source.zip" - bucket = google_storage_bucket.agent_staging.name - source = data.archive_file.cost_guard_fn_zip.output_path -} - -resource "google_cloudfunctions2_function" "billing_auto_teardown" { - name = "billing-auto-teardown" - location = var.region - - build_config { - runtime = "python312" - entry_point = "billing_alert" - source { - storage_source { - bucket = google_storage_bucket.agent_staging.name - object = google_storage_bucket_object.cost_guard_fn_source.name - } - } - } - - service_config { - service_account_email = google_service_account.agent_sa.email - environment_variables = { - PROJECT_ID = var.project_id - CLOUD_RUN_SERVICE = var.cloud_run_service_name - REGION = var.region - } - } - - event_trigger { - trigger_region = "global" - event_type = "google.cloud.pubsub.topic.v1.messagePublished" - pubsub_topic = google_pubsub_topic.billing_alerts.id - } -} - -data "google_project" "project" {} - -resource "google_project_iam_member" "function_sa_run_admin" { - project = var.project_id - role = "roles/run.admin" - member = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" -} - -resource "google_project_iam_member" "function_sa_secret_accessor" { - project = var.project_id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com" -} - -# From 11-billing-iam-hardening.sh -resource "google_billing_account_iam_member" "billing_viewers" { - for_each = toset(var.billing_viewer_emails) - - billing_account_id = var.billing_account_id - role = "roles/billing.viewer" - member = "user:${each.key}" -} diff --git a/infrastructure/terraform/outputs.tf b/infrastructure/terraform/outputs.tf index 7c0ee8a..194eca2 100644 --- a/infrastructure/terraform/outputs.tf +++ b/infrastructure/terraform/outputs.tf @@ -1,15 +1,14 @@ -# Managed via Cloud Build CI/CD — not Terraform -# output "cloud_run_url" { -# description = "The URL of the Cloud Run service." -# value = google_cloud_run_v2_service.agent_service.uri -# } - -output "service_account_email" { - description = "The email of the service account." - value = google_service_account.agent_sa.email +output "medioteq_cloud_run_service_url" { + description = "The URL of the Medioteq Cloud Run service." + value = module.vauco_bootstrap_medioteq.cloud_run_service_url } -output "artifact_registry_url" { - description = "The URL of the Artifact Registry repository." - value = "${google_artifact_registry_repository.repo.location}-docker.pkg.dev/${google_artifact_registry_repository.repo.project}/${google_artifact_registry_repository.repo.repository_id}" +output "medioteq_service_account_email" { + description = "The email of the Medioteq service account." + value = module.vauco_bootstrap_medioteq.service_account_email +} + +output "medioteq_artifact_registry_url" { + description = "The URL of the Medioteq Artifact Registry repository." + value = module.vauco_bootstrap_medioteq.artifact_registry_url }