diff --git a/ml/anomaly_detector.py b/ml/anomaly_detector.py index ba2dc88..d2a14c0 100644 --- a/ml/anomaly_detector.py +++ b/ml/anomaly_detector.py @@ -10,43 +10,45 @@ class AnomalyDetector: if not self.project_id: raise ValueError("GOOGLE_CLOUD_PROJECT environment variable not set.") + self.billing_table = os.environ.get("BILLING_TABLE") + if not self.billing_table: + raise ValueError("BILLING_TABLE environment variable not set.") + self.pubsub_topic_name = os.environ.get("PUBSUB_TOPIC", "billing_alerts") self.bq_client = bigquery.Client(project=self.project_id) self.publisher = pubsub_v1.PublisherClient() self.topic_path = self.publisher.topic_path(self.project_id, self.pubsub_topic_name) - self.billing_table = f"{self.project_id}.billing_export.gcp_billing_export_v1_*" def detect_anomalies(self): """ - Detects anomalies in billing data. + Detects anomalies in billing data by comparing today's cost to the 7-day average. """ query = f""" - WITH - cost_last_7_days AS ( - SELECT - service.description as service, - SUM(cost) as total_cost - FROM `{self.billing_table}` - WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 8 DAY) - AND _PARTITIONTIME < TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY) - GROUP BY 1 + WITH daily_costs AS ( + SELECT + service.description AS service, + DATE(_PARTITIONTIME) AS usage_date, + SUM(cost) AS daily_cost + FROM `{self.billing_table}` + WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 14 DAY) + GROUP BY 1, 2 ), - cost_last_1_day AS ( - SELECT - service.description as service, - SUM(cost) as total_cost - FROM `{self.billing_table}` - WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 DAY) - GROUP BY 1 + costs_with_avg AS ( + SELECT + service, + usage_date, + daily_cost, + AVG(daily_cost) OVER (PARTITION BY service ORDER BY usage_date ROWS BETWEEN 7 PRECEDING AND 1 PRECEDING) AS avg_7day + FROM daily_costs ) SELECT - c1.service, - c1.total_cost as today_cost, - c7.total_cost / 7 as avg_7day, - c1.total_cost / (c7.total_cost / 7) as ratio - FROM cost_last_1_day c1 - JOIN cost_last_7_days c7 ON c1.service = c7.service - WHERE c1.total_cost > 2 * (c7.total_cost / 7) + service, + daily_cost AS today_cost, + avg_7day, + daily_cost / avg_7day AS ratio + FROM costs_with_avg + WHERE usage_date = CURRENT_DATE() + AND daily_cost > 2.0 * avg_7day """ query_job = self.bq_client.query(query) results = query_job.result() @@ -56,7 +58,7 @@ class AnomalyDetector: anomalies.append({ "service": row.service, "today_cost": row.today_cost, - "avg_7day": row.avg_7day, + "avg_7d": row.avg_7day, "ratio": row.ratio }) @@ -73,7 +75,7 @@ class AnomalyDetector: message_data = { "service": anomaly["service"], "today_cost": anomaly["today_cost"], - "avg_7day": anomaly["avg_7day"], + "avg_7d": anomaly["avg_7d"], "ratio": anomaly["ratio"], "timestamp": datetime.datetime.now().isoformat() }