88 lines
3.1 KiB
Python
88 lines
3.1 KiB
Python
import os
|
|
import json
|
|
import datetime
|
|
from google.cloud import bigquery
|
|
from google.cloud import pubsub_v1
|
|
|
|
class AnomalyDetector:
|
|
def __init__(self):
|
|
self.project_id = os.environ.get("GOOGLE_CLOUD_PROJECT")
|
|
if not self.project_id:
|
|
raise ValueError("GOOGLE_CLOUD_PROJECT environment variable not set.")
|
|
|
|
self.billing_table = os.environ.get("BILLING_TABLE")
|
|
if not self.billing_table:
|
|
raise ValueError("BILLING_TABLE environment variable not set.")
|
|
|
|
self.pubsub_topic_name = os.environ.get("ANOMALY_TOPIC", "osvauco-billing-anomalies")
|
|
self.bq_client = bigquery.Client(project=self.project_id)
|
|
self.publisher = pubsub_v1.PublisherClient()
|
|
self.topic_path = self.publisher.topic_path(self.project_id, self.pubsub_topic_name)
|
|
|
|
def detect_anomalies(self):
|
|
"""
|
|
Detects anomalies in billing data by comparing today's cost to the 7-day average.
|
|
"""
|
|
query = f"""
|
|
WITH daily_costs AS (
|
|
SELECT
|
|
service.description AS service,
|
|
DATE(_PARTITIONTIME) AS usage_date,
|
|
SUM(cost) AS daily_cost
|
|
FROM `{self.billing_table}`
|
|
WHERE _PARTITIONTIME >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 14 DAY)
|
|
GROUP BY 1, 2
|
|
),
|
|
costs_with_avg AS (
|
|
SELECT
|
|
service,
|
|
usage_date,
|
|
daily_cost,
|
|
AVG(daily_cost) OVER (PARTITION BY service ORDER BY usage_date ROWS BETWEEN 7 PRECEDING AND 1 PRECEDING) AS avg_7day
|
|
FROM daily_costs
|
|
)
|
|
SELECT
|
|
service,
|
|
daily_cost AS today_cost,
|
|
avg_7day,
|
|
daily_cost / avg_7day AS ratio
|
|
FROM costs_with_avg
|
|
WHERE usage_date = CURRENT_DATE()
|
|
AND daily_cost > 2.0 * avg_7day
|
|
"""
|
|
query_job = self.bq_client.query(query)
|
|
results = query_job.result()
|
|
|
|
anomalies = []
|
|
for row in results:
|
|
anomalies.append({
|
|
"service": row.service,
|
|
"today_cost": row.today_cost,
|
|
"avg_7d": row.avg_7day,
|
|
"ratio": row.ratio
|
|
})
|
|
|
|
if anomalies:
|
|
self.publish_alerts(anomalies)
|
|
|
|
return {"anomalies": anomalies}
|
|
|
|
def publish_alerts(self, anomalies):
|
|
"""
|
|
Publishes anomaly alerts to a Pub/Sub topic.
|
|
"""
|
|
for anomaly in anomalies:
|
|
message_data = {
|
|
"service": anomaly["service"],
|
|
"today_cost": anomaly["today_cost"],
|
|
"avg_7d": anomaly["avg_7d"],
|
|
"ratio": anomaly["ratio"],
|
|
"timestamp_utc": datetime.datetime.utcnow().isoformat()
|
|
}
|
|
message_bytes = json.dumps(message_data).encode("utf-8")
|
|
self.publisher.publish(self.topic_path, data=message_bytes)
|
|
|
|
if __name__ == '__main__':
|
|
detector = AnomalyDetector()
|
|
print("Anomalies:", detector.detect_anomalies())
|