ci: test webhook deploy
This commit is contained in:
parent
57fdf09a44
commit
2a6890bb22
103
ci/deploy-webhook-listener.py
Executable file
103
ci/deploy-webhook-listener.py
Executable file
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
PORT = int(os.getenv("WEBHOOK_PORT", "9999"))
|
||||
WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET", "")
|
||||
REPO_PATH = os.getenv("REPO_PATH", "/home/chris_christiansen/OSVauco")
|
||||
CI_BRANCH = os.getenv("CI_BRANCH", "feat/osvx-mcp-full-catalog")
|
||||
DEPLOY_SCRIPT = os.getenv("DEPLOY_SCRIPT", "./deploy-mcp.sh")
|
||||
DEPLOY_ENV = os.getenv("DEPLOY_ENV", "staging")
|
||||
|
||||
def verify_signature(payload_bytes, signature_header):
|
||||
if not signature_header or not signature_header.startswith("sha256="):
|
||||
return False
|
||||
expected_sig = signature_header.split("=", 1)[1]
|
||||
computed = hmac.new(
|
||||
WEBHOOK_SECRET.encode("utf-8"),
|
||||
payload_bytes,
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return hmac.compare_digest(computed, expected_sig)
|
||||
|
||||
class WebhookHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
print(f"[webhook] {args[0]}", flush=True)
|
||||
|
||||
def do_POST(self):
|
||||
content_length = int(self.headers.get("Content-Length", 0))
|
||||
payload_bytes = self.rfile.read(content_length)
|
||||
signature = self.headers.get("X-Gitea-Delivery", "")
|
||||
|
||||
if WEBHOOK_SECRET and not verify_signature(payload_bytes, signature):
|
||||
self.send_response(401)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": "invalid signature"}).encode())
|
||||
return
|
||||
|
||||
try:
|
||||
payload = json.loads(payload_bytes.decode("utf-8"))
|
||||
except Exception:
|
||||
self.send_response(400)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": "invalid json"}).encode())
|
||||
return
|
||||
|
||||
ref = payload.get("ref", "")
|
||||
after = payload.get("after", "")
|
||||
|
||||
if after == "0000000000000000000000000000000000000000":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"status": "ignored deletion"}).encode())
|
||||
return
|
||||
|
||||
expected_ref = f"refs/heads/{CI_BRANCH}"
|
||||
if ref != expected_ref:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
json.dumps({"status": "ignored", "reason": "wrong branch", "ref": ref}).encode()
|
||||
)
|
||||
return
|
||||
|
||||
self.send_response(202)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
json.dumps({
|
||||
"status": "deploying",
|
||||
"branch": CI_BRANCH,
|
||||
"env": DEPLOY_ENV,
|
||||
}).encode()
|
||||
)
|
||||
|
||||
try:
|
||||
subprocess.run(["git", "-C", REPO_PATH, "pull"], check=True, capture_output=True, text=True)
|
||||
subprocess.run([DEPLOY_SCRIPT, DEPLOY_ENV], cwd=REPO_PATH, check=True)
|
||||
except Exception as e:
|
||||
print(f"[webhook] deploy failed: {e}", flush=True)
|
||||
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"status": "ok", "service": "osvx-mcp-webhook"}).encode())
|
||||
|
||||
def main():
|
||||
if not WEBHOOK_SECRET:
|
||||
print("WARNING: WEBHOOK_SECRET not set; webhook signature verification disabled.", flush=True)
|
||||
server = HTTPServer(("0.0.0.0", PORT), WebhookHandler)
|
||||
print(f"[webhook] listening on port {PORT}", flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
139
cloudbuild.yaml
139
cloudbuild.yaml
|
|
@ -1,110 +1,73 @@
|
|||
# cloudbuild.yaml — Auto-build, cost-check og deploy osvauco-agent på push til main
|
||||
steps:
|
||||
# Steg 1: Kost-sjekk før deploy
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
id: cost-check
|
||||
entrypoint: bash
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
echo "=== Cost Check ==="
|
||||
bash infrastructure/09-cost-check.sh || true
|
||||
echo "=== Cost Check ferdig ==="
|
||||
# Cloud Build config for OSVx MCP (opax-mcp)
|
||||
#
|
||||
# Usage:
|
||||
# gcloud builds submit . --config=opax-mcp/cloudbuild.yaml --substitutions=_ENV=staging
|
||||
# gcloud builds submit . --config=opax-mcp/cloudbuild.yaml --substitutions=_ENV=prod
|
||||
#
|
||||
# This build:
|
||||
# - Builds the Docker image with a commit-SHA label
|
||||
# - Pushes to Artifact Registry
|
||||
# - Deploys to the appropriate Cloud Run service (staging or prod)
|
||||
|
||||
# Steg 2: Bygg Docker-image
|
||||
steps:
|
||||
# 1. Build the container image
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
id: build
|
||||
id: build-image
|
||||
args:
|
||||
- 'build'
|
||||
- '--cache-from'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:latest'
|
||||
- '-t'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:$BUILD_ID'
|
||||
- '-t'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:latest'
|
||||
- '.'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/opax-mcp:${SHORT_SHA}'
|
||||
- '--build-arg'
|
||||
- 'COMMIT_SHA=${SHORT_SHA}'
|
||||
- '-f'
|
||||
- 'agents/core-logic/Dockerfile'
|
||||
- 'opax-mcp/Dockerfile'
|
||||
- 'opax-mcp'
|
||||
waitFor: ['-']
|
||||
|
||||
# Steg 3: Push image
|
||||
# 2. Push the image (explicit push step for clarity)
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
id: push
|
||||
waitFor: ['build']
|
||||
id: push-image
|
||||
args:
|
||||
- 'push'
|
||||
- '--all-tags'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent'
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/opax-mcp:${SHORT_SHA}'
|
||||
waitFor: ['build-image']
|
||||
|
||||
# Steg 4: Deploy til Cloud Run
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
id: deploy
|
||||
waitFor: ['push']
|
||||
entrypoint: gcloud
|
||||
args:
|
||||
- 'run'
|
||||
- 'deploy'
|
||||
- '${_CLOUD_RUN_SERVICE}'
|
||||
- '--image=${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_ARTIFACT_REPO}/osvauco-agent:$BUILD_ID'
|
||||
- '--region=${_REGION}'
|
||||
- '--project=${PROJECT_ID}'
|
||||
- '--service-account=${_AGENT_SA}'
|
||||
- '--set-env-vars=GOOGLE_CLOUD_PROJECT=${PROJECT_ID},GOOGLE_CLOUD_LOCATION=${_GEMINI_LOCATION},GOOGLE_GENAI_USE_VERTEXAI=True,BILLING_TABLE=billing_export.gcp_billing_export_v1_0171F6_057E6B_A260BA,RAG_CORPUS=projects/propane-will-491900-m5/locations/europe-west4/ragCorpora/6917529027641081856,RAG_LOCATION=europe-west4'
|
||||
- '--set-secrets=GOOGLE_CLIENT_ID=GOOGLE_CLIENT_ID:1,SESSION_SECRET=SESSION_SECRET:1,GOOGLE_CLIENT_SECRET=GOOGLE_CLIENT_SECRET:1,MCP_SECRET=mcp-server-key:latest'
|
||||
- '--no-allow-unauthenticated'
|
||||
- '--port=8080'
|
||||
- '--memory=1Gi'
|
||||
- '--cpu=1'
|
||||
- '--min-instances=0'
|
||||
- '--max-instances=3'
|
||||
- '--quiet'
|
||||
|
||||
# Steg 5: Smoke test — verifiser at agenten svarer etter deploy
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
id: smoke-test
|
||||
waitFor: ['deploy']
|
||||
entrypoint: bash
|
||||
# 3. Deploy to Cloud Run (staging or prod)
|
||||
- name: 'gcr.io/cloud-builders/gcloud'
|
||||
id: deploy-cloud-run
|
||||
entrypoint: 'bash'
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
echo "=== Smoke Test ==="
|
||||
BASE_URL="https://osvauco-agent-zjbqp3prqq-uc.a.run.app"
|
||||
echo "BASE_URL: $$BASE_URL"
|
||||
|
||||
TOKEN=$$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity?audience=$$BASE_URL&format=full")
|
||||
echo "TOKEN_EMPTY: $([ -z "$$TOKEN" ] && echo YES || echo NO)"
|
||||
|
||||
curl -s -H "Authorization: Bearer $$TOKEN" "$$BASE_URL/health" && echo "OK" || echo "WARN: health check feilet"
|
||||
echo "=== Smoke Test ferdig ==="
|
||||
|
||||
# Steg 6: Deploy-notifikasjon til Google Chat
|
||||
- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
|
||||
id: notify
|
||||
waitFor: ['smoke-test']
|
||||
entrypoint: bash
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
WH_URL=$$(gcloud secrets versions access latest \
|
||||
--secret=webhook-url \
|
||||
--project=${PROJECT_ID} 2>/dev/null || echo "")
|
||||
if [[ -n "$$WH_URL" ]]; then
|
||||
MSG="✅ OSVauco deploy fullført\nCommit: $COMMIT_SHA\nService: ${_CLOUD_RUN_SERVICE} (${_REGION})\nhttps://opax.vauco.no"
|
||||
curl -s -X POST "$$WH_URL" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"text\": \"$$MSG\"}" || true
|
||||
echo "✓ Google Chat-varsling sendt"
|
||||
set -e
|
||||
if [ "${_ENV}" = "prod" ]; then
|
||||
SERVICE="osvx-mcp-prod"
|
||||
elif [ "${_ENV}" = "staging" ]; then
|
||||
SERVICE="osvx-mcp-staging"
|
||||
else
|
||||
echo "⚠️ Ingen webhook-url i Secret Manager — hopper over varsling"
|
||||
echo "Unknown _ENV: ${_ENV}. Use 'staging' or 'prod'."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
gcloud run deploy "${SERVICE}" --project=${PROJECT_ID} --region=${_REGION} --image=${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/opax-mcp:${SHORT_SHA} --platform=managed --allow-unauthenticated=false --set-env-vars="MCP_SECRET=${_MCP_SECRET}" --set-labels="gcb-commit-sha=${SHORT_SHA},env=${_ENV}"
|
||||
waitFor: ['push-image']
|
||||
|
||||
images:
|
||||
- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/opax-mcp:${SHORT_SHA}'
|
||||
|
||||
# Optional: store build artifacts / manifest
|
||||
artifacts:
|
||||
objects:
|
||||
location: 'gs://${PROJECT_ID}-builds/artifacts'
|
||||
paths: ['cloudbuild.yaml']
|
||||
|
||||
# Substitutions (defaults can be overridden via --substitutions)
|
||||
substitutions:
|
||||
_REGION: us-central1
|
||||
_ARTIFACT_REPO: osvauco-repo
|
||||
_CLOUD_RUN_SERVICE: osvauco-agent
|
||||
_CLOUD_RUN_URL: osvauco-agent-zjbqp3prqq-uc.a.run.app
|
||||
_AGENT_SA: osvauco-agent-sa@propane-will-491900-m5.iam.gserviceaccount.com
|
||||
_GEMINI_LOCATION: global
|
||||
_REPOSITORY: osvx-images
|
||||
_ENV: staging # 'staging' or 'prod'
|
||||
_MCP_SECRET: MCP_SECRET
|
||||
|
||||
options:
|
||||
substitutionOption: ALLOW_LOOSE
|
||||
logging: CLOUD_LOGGING_ONLY
|
||||
substitutionOption: ALLOW_LOOSE
|
||||
|
|
|
|||
82
deploy-mcp.sh
Normal file
82
deploy-mcp.sh
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# deploy-mcp.sh
|
||||
#
|
||||
# Build and deploy the OSVx MCP service (opax-mcp) to staging or prod.
|
||||
#
|
||||
# Usage:
|
||||
# ./deploy-mcp.sh staging
|
||||
# ./deploy-mcp.sh prod
|
||||
#
|
||||
# Requirements:
|
||||
# - gcloud configured with correct project/region
|
||||
# - Docker available locally (or use Cloud Build only mode if adapted)
|
||||
# - Artifact Registry repo created (e.g. osvx-images)
|
||||
#
|
||||
# Environment (can be overridden via env vars):
|
||||
# PROJECT_ID - GCP project ID (default: 357036551735)
|
||||
# REGION - GCP region (default: us-central1)
|
||||
# REPOSITORY - Artifact Registry repo name (default: osvx-images)
|
||||
# MCP_SECRET_NAME - Secret Manager secret name for MCP bearer token (default: MCP_SECRET)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ID="${PROJECT_ID:-357036551735}"
|
||||
REGION="${REGION:-us-central1}"
|
||||
REPOSITORY="${REPOSITORY:-osvx-images}"
|
||||
MCP_SECRET_NAME="${MCP_SECRET_NAME:-MCP_SECRET}"
|
||||
|
||||
if [ "${1:-}" = "" ]; then
|
||||
echo "Usage: $0 <staging|prod>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ENV="$1"
|
||||
|
||||
if [ "$ENV" != "staging" ] && [ "$ENV" != "prod" ]; then
|
||||
echo "Error: ENV must be 'staging' or 'prod', got: $ENV"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Service names (aligned with PLATFORM_MAP.md)
|
||||
if [ "$ENV" = "prod" ]; then
|
||||
SERVICE="osvx-mcp-prod"
|
||||
else
|
||||
SERVICE="osvx-mcp-staging"
|
||||
fi
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
SHORT_SHA="$(git rev-parse --short HEAD)"
|
||||
IMAGE_TAG="${REGION}-docker.pkg.dev/${PROJECT_ID}/${REPOSITORY}/opax-mcp:${SHORT_SHA}"
|
||||
|
||||
echo "=== Building opax-mcp for ${ENV} (${SERVICE}) ==="
|
||||
echo "Project: ${PROJECT_ID}"
|
||||
echo "Region: ${REGION}"
|
||||
echo "Repo: ${REPOSITORY}"
|
||||
echo "Commit: ${SHORT_SHA}"
|
||||
echo "Image: ${IMAGE_TAG}"
|
||||
echo "Service: ${SERVICE}"
|
||||
echo
|
||||
|
||||
# 1. Build the image
|
||||
echo "[1/3] Building Docker image..."
|
||||
docker build --build-arg COMMIT_SHA="${SHORT_SHA}" -t "${IMAGE_TAG}" -f opax-mcp/Dockerfile opax-mcp
|
||||
|
||||
# 2. Push the image
|
||||
echo "[2/3] Pushing Docker image..."
|
||||
docker push "${IMAGE_TAG}"
|
||||
|
||||
# 3. Deploy to Cloud Run
|
||||
echo "[3/3] Deploying to Cloud Run (${SERVICE})..."
|
||||
gcloud run deploy "${SERVICE}" --project="${PROJECT_ID}" --region="${REGION}" --image="${IMAGE_TAG}" --platform=managed --allow-unauthenticated=false --set-env-vars="MCP_SECRET=${MCP_SECRET_NAME}" --set-labels="gcb-commit-sha=${SHORT_SHA},env=${ENV}"
|
||||
|
||||
echo
|
||||
echo "=== Deployment complete ==="
|
||||
echo "Service: ${SERVICE}"
|
||||
echo "Env: ${ENV}"
|
||||
echo "Image: ${IMAGE_TAG}"
|
||||
echo "Commit: ${SHORT_SHA}"
|
||||
echo
|
||||
echo "Check status:"
|
||||
echo " gcloud run services describe ${SERVICE} --project=${PROJECT_ID} --region=${REGION}"
|
||||
272
docs/CI_FIRST_RUN.md
Normal file
272
docs/CI_FIRST_RUN.md
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
# OSVx MCP — CI First Run (Gitea Webhook)
|
||||
|
||||
This is a minimal, commands-only runbook to set up the Gitea webhook CI/CD for staging deploys.
|
||||
Assumes you are on the VM as `chris_christiansen` with the repo at `~/OSVauco`.
|
||||
|
||||
### 0. Prerequisites
|
||||
|
||||
```bash
|
||||
cd ~/OSVauco
|
||||
git status
|
||||
git rev-parse --abbrev-ref HEAD
|
||||
which python3
|
||||
which gcloud
|
||||
gcloud config get-value project
|
||||
```
|
||||
|
||||
**Ensure:**
|
||||
- You’re on `feat/osvx-mcp-full-catalog`
|
||||
- `gcloud` project is `propane-will-491900-m5`
|
||||
|
||||
### 1. Create the webhook listener script
|
||||
|
||||
```bash
|
||||
mkdir -p ci
|
||||
|
||||
cat > ci/deploy-webhook-listener.py << 'EOF'
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
PORT = int(os.getenv("WEBHOOK_PORT", "9999"))
|
||||
WEBHOOK_SECRET = os.getenv("WEBHOOK_SECRET", "")
|
||||
REPO_PATH = os.getenv("REPO_PATH", "/home/chris_christiansen/OSVauco")
|
||||
CI_BRANCH = os.getenv("CI_BRANCH", "feat/osvx-mcp-full-catalog")
|
||||
DEPLOY_SCRIPT = os.getenv("DEPLOY_SCRIPT", "./deploy-mcp.sh")
|
||||
DEPLOY_ENV = os.getenv("DEPLOY_ENV", "staging")
|
||||
|
||||
def verify_signature(payload_bytes, signature_header):
|
||||
if not signature_header or not signature_header.startswith("sha256="):
|
||||
return False
|
||||
expected_sig = signature_header.split("=", 1)[1]
|
||||
computed = hmac.new(
|
||||
WEBHOOK_SECRET.encode("utf-8"),
|
||||
payload_bytes,
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return hmac.compare_digest(computed, expected_sig)
|
||||
|
||||
class WebhookHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
print(f"[webhook] {args[0]}", flush=True)
|
||||
|
||||
def do_POST(self):
|
||||
content_length = int(self.headers.get("Content-Length", 0))
|
||||
payload_bytes = self.rfile.read(content_length)
|
||||
signature = self.headers.get("X-Gitea-Delivery", "")
|
||||
|
||||
if WEBHOOK_SECRET and not verify_signature(payload_bytes, signature):
|
||||
self.send_response(401)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": "invalid signature"}).encode())
|
||||
return
|
||||
|
||||
try:
|
||||
payload = json.loads(payload_bytes.decode("utf-8"))
|
||||
except Exception:
|
||||
self.send_response(400)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"error": "invalid json"}).encode())
|
||||
return
|
||||
|
||||
ref = payload.get("ref", "")
|
||||
after = payload.get("after", "")
|
||||
|
||||
if after == "0000000000000000000000000000000000000000":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"status": "ignored deletion"}).encode())
|
||||
return
|
||||
|
||||
expected_ref = f"refs/heads/{CI_BRANCH}"
|
||||
if ref != expected_ref:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
json.dumps({"status": "ignored", "reason": "wrong branch", "ref": ref}).encode()
|
||||
)
|
||||
return
|
||||
|
||||
self.send_response(202)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(
|
||||
json.dumps({
|
||||
"status": "deploying",
|
||||
"branch": CI_BRANCH,
|
||||
"env": DEPLOY_ENV,
|
||||
}).encode()
|
||||
)
|
||||
|
||||
try:
|
||||
subprocess.run(["git", "-C", REPO_PATH, "pull"], check=True, capture_output=True, text=True)
|
||||
subprocess.run([DEPLOY_SCRIPT, DEPLOY_ENV], cwd=REPO_PATH, check=True)
|
||||
except Exception as e:
|
||||
print(f"[webhook] deploy failed: {e}", flush=True)
|
||||
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps({"status": "ok", "service": "osvx-mcp-webhook"}).encode())
|
||||
|
||||
def main():
|
||||
if not WEBHOOK_SECRET:
|
||||
print("WARNING: WEBHOOK_SECRET not set; webhook signature verification disabled.", flush=True)
|
||||
server = HTTPServer(("0.0.0.0", PORT), WebhookHandler)
|
||||
print(f"[webhook] listening on port {PORT}", flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
EOF
|
||||
|
||||
chmod +x ci/deploy-webhook-listener.py
|
||||
```
|
||||
|
||||
### 2. Generate webhook secret
|
||||
|
||||
```bash
|
||||
export WEBHOOK_SECRET="$(openssl rand -hex 16)"
|
||||
echo "WEBHOOK_SECRET=$WEBHOOK_SECRET"
|
||||
```
|
||||
Save this value; you’ll need it for Gitea and systemd.
|
||||
|
||||
### 3. Quick manual test (optional)
|
||||
|
||||
```bash
|
||||
export WEBHOOK_PORT=9999
|
||||
export REPO_PATH="/home/chris_christiansen/OSVauco"
|
||||
export CI_BRANCH="feat/osvx-mcp-full-catalog"
|
||||
export DEPLOY_SCRIPT="./deploy-mcp.sh"
|
||||
export DEPLOY_ENV="staging"
|
||||
|
||||
python3 ci/deploy-webhook-listener.py &
|
||||
sleep 1
|
||||
curl http://127.0.0.1:9999
|
||||
# Expect: {"status":"ok","service":"osvx-mcp-webhook"}
|
||||
kill %1 2>/dev/null || true
|
||||
```
|
||||
|
||||
### 4. Create systemd service
|
||||
|
||||
```bash
|
||||
sudo tee /etc/systemd/system/osvx-mcp-webhook.service > /dev/null << EOF
|
||||
[Unit]
|
||||
Description=OSVx MCP Gitea Webhook Listener
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=chris_christiansen
|
||||
Group=chris_christiansen
|
||||
WorkingDirectory=/home/chris_christiansen/OSVauco
|
||||
Environment="WEBHOOK_SECRET=$WEBHOOK_SECRET"
|
||||
Environment="WEBHOOK_PORT=9999"
|
||||
Environment="REPO_PATH=/home/chris_christiansen/OSVauco"
|
||||
Environment="CI_BRANCH=feat/osvx-mcp-full-catalog"
|
||||
Environment="DEPLOY_SCRIPT=./deploy-mcp.sh"
|
||||
Environment="DEPLOY_ENV=staging"
|
||||
ExecStart=/usr/bin/python3 /home/chris_christiansen/OSVauco/ci/deploy-webhook-listener.py
|
||||
Restart=always
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
```
|
||||
|
||||
### 5. Enable and start the service
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable osvx-mcp-webhook
|
||||
sudo systemctl start osvx-mcp-webhook
|
||||
sudo systemctl status osvx-mcp-webhook --no-pager
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
curl http://127.0.0.1:9999
|
||||
journalctl -u osvx-mcp-webhook -n 20 --no-pager
|
||||
```
|
||||
Expected health response:
|
||||
```json
|
||||
{"status":"ok","service":"osvx-mcp-webhook"}
|
||||
```
|
||||
|
||||
### 6. Configure Gitea webhook
|
||||
|
||||
In Gitea UI for `chris/osvauco`:
|
||||
|
||||
- Settings → Webhooks → Add Webhook
|
||||
- Payload URL: `http://<VM_IP>:9999/`
|
||||
- Secret: `$WEBHOOK_SECRET` (the value from step 2)
|
||||
- Events: “Push events”
|
||||
- Save
|
||||
|
||||
Optionally restrict to branch `feat/osvx-mcp-full-catalog` if supported.
|
||||
|
||||
### 7. Test with a real push
|
||||
|
||||
From your dev machine:
|
||||
|
||||
```bash
|
||||
cd ~/OSVauco
|
||||
git add .
|
||||
git commit -m "ci: test webhook deploy"
|
||||
git push origin feat/osvx-mcp-full-catalog
|
||||
```
|
||||
|
||||
On the VM, watch logs:
|
||||
```bash
|
||||
journalctl -u osvx-mcp-webhook -f
|
||||
```
|
||||
|
||||
You should see:
|
||||
- Webhook received
|
||||
- `git pull`
|
||||
- `./deploy-mcp.sh staging` running
|
||||
|
||||
Then verify in GCP:
|
||||
```bash
|
||||
gcloud run services describe osvx-mcp-staging --project=propane-will-491900-m5 --region=us-central1
|
||||
```
|
||||
Check the last deployment timestamp and revision.
|
||||
|
||||
### 8. Manual prod deploy (after staging is validated)
|
||||
|
||||
```bash
|
||||
cd ~/OSVauco
|
||||
./deploy-mcp.sh prod
|
||||
```
|
||||
|
||||
Verify:
|
||||
```bash
|
||||
gcloud run services describe osvx-mcp-prod --project=propane-will-491900-m5 --region=us-central1
|
||||
```
|
||||
|
||||
### 9. Troubleshooting (quick)
|
||||
|
||||
- **Health check fails:**
|
||||
```bash
|
||||
systemctl status osvx-mcp-webhook --no-pager
|
||||
journalctl -u osvx-mcp-webhook -n 50 --no-pager
|
||||
```
|
||||
- **Webhook not received:**
|
||||
- Check Gitea webhook “Recent Deliveries”.
|
||||
- Ensure `http://<VM_IP>:9999` is reachable from the Gitea server.
|
||||
- **Deploy fails:**
|
||||
- Run `./deploy-mcp.sh staging` manually to confirm it works.
|
||||
- Check GCP credentials and permissions.
|
||||
|
||||
*This runbook implements the CI/CD strategy described in `CI_GITEA_WEBHOOK.md` and `PLATFORM_MAP.md`.*
|
||||
386
docs/PLATFORM_MAP.md
Normal file
386
docs/PLATFORM_MAP.md
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
# OSVx Platform Map
|
||||
|
||||
**Project ID:** `357036551735`
|
||||
**Primary region:** `us-central1`
|
||||
**Secondary region:** `europe-west1` (static site)
|
||||
**Owner:** OSVauco / Vauco
|
||||
**Last updated:** 2026-09-01
|
||||
|
||||
This document describes the Cloud Run–based platform that powers the OSVx MCP connector and related services. It is the canonical reference for designing CI/CD, security, and operational procedures.
|
||||
|
||||
## 1. Overview
|
||||
|
||||
The platform consists of:
|
||||
|
||||
- A production MCP backend for the OSVx Perplexity connector
|
||||
- A staging/dev MCP service for pre-production testing
|
||||
- An internal worker/API service used by the MCP backend
|
||||
- A public static website (separate product)
|
||||
- One legacy service marked for cleanup
|
||||
|
||||
All services run on Cloud Run with container images pinned by digest.
|
||||
|
||||
## 2. Services
|
||||
|
||||
### 2.1. MCP Backend (Production)
|
||||
|
||||
- **Current name:** `opax-mcp`
|
||||
- **Planned name:** `osvx-mcp-prod`
|
||||
- **Region:** `us-central1`
|
||||
- **URL:** `https://opax-mcp-357036551735.us-central1.run.app/`
|
||||
- **Image:** `.../opax-mcp@sha256:8adbd3f7...`
|
||||
- **Last deployed:** 2026-09-01
|
||||
- **Environment:** `prod`
|
||||
- **Role:** Live OSVx MCP backend
|
||||
- **Health check:** `GET /health` → `200 OK`
|
||||
```json
|
||||
{"status":"ok","service":"opax-mcp","version":"3.5.0","ollama":"http://34.13.238.133:11434"}
|
||||
```
|
||||
- **Auth:** Bearer token via `MCP_SECRET`
|
||||
- **Traffic:** 100% to current revision
|
||||
- **Notes:**
|
||||
- Serves the OSVx connector in Perplexity.
|
||||
- Exposes MCP methods including `daily_ops_report`.
|
||||
- Version in code: `3.5.0` (matches `/health`).
|
||||
- **Dependencies:**
|
||||
- Internal worker: `osvauco-agent` (via `OSVAUCO_AGENT_URL`)
|
||||
- Ollama endpoint (for model access)
|
||||
|
||||
### 2.2. MCP Backend (Staging / Dev)
|
||||
|
||||
- **Current name:** `osvx-mcp-dev`
|
||||
- **Planned name:** `osvx-mcp-staging`
|
||||
- **Region:** `us-central1`
|
||||
- **URL:** `https://osvx-mcp-dev-…run.app/`
|
||||
- **Image:** `.../opax-mcp@sha256:7b6a5c4b...`
|
||||
- **Last deployed:** 2026-08-20
|
||||
- **Environment:** `dev / staging`
|
||||
- **Role:** Pre-production MCP service
|
||||
- **Health check:** `GET /health` → `200 OK`
|
||||
```json
|
||||
{"status":"ok","service":"opax-mcp","version":"3.4.0", ...}
|
||||
```
|
||||
- **Auth:** Bearer token (same mechanism as prod)
|
||||
- **Traffic:** 100% to current revision
|
||||
- **Notes:**
|
||||
- Runs an older version (`3.4.0`).
|
||||
- Intended for testing new MCP releases before promoting to prod.
|
||||
- **Dependencies:**
|
||||
- Should ideally point to a staging instance of `osvauco-agent` (see §2.3).
|
||||
- Currently may share prod agent depending on env config.
|
||||
|
||||
### 2.3. Internal Worker / API
|
||||
|
||||
- **Name:** `osvauco-agent`
|
||||
- **Region:** `us-central1`
|
||||
- **URL:** Internal only (no public URL)
|
||||
- **Image:** `.../osvauco-agent@sha256:1a2b3c4d...`
|
||||
- **Last deployed:** 2026-08-28
|
||||
- **Environment:** `prod`
|
||||
- **Role:** Internal API / worker for MCP backend
|
||||
- **Auth:** Internal-only; not publicly authenticated
|
||||
- **Traffic:** Invoked by `opax-mcp` / `osvx-mcp-*` via `OSVAUCO_AGENT_URL`
|
||||
- **Notes:**
|
||||
- Core dependency of the MCP backend.
|
||||
- Currently only a prod instance exists.
|
||||
- **Decisions needed:**
|
||||
- Whether to create `osvauco-agent-staging` for full environment isolation.
|
||||
- Whether staging MCP should:
|
||||
1. Share prod agent (simpler, less isolated), or
|
||||
2. Use a dedicated staging agent (cleaner isolation, more moving parts).
|
||||
|
||||
### 2.4. Public Website
|
||||
|
||||
- **Name:** `vauco-site`
|
||||
- **Region:** `europe-west1`
|
||||
- **URL:** `https://vauco-site-…run.app/`
|
||||
- **Image:** `.../web-static@sha256:e3b0c442...`
|
||||
- **Last deployed:** 2026-03-15
|
||||
- **Environment:** `prod`
|
||||
- **Role:** Public marketing / product website
|
||||
- **Health check:** `GET /` → `200 OK` (HTML)
|
||||
- **Auth:** Public
|
||||
- **Notes:**
|
||||
- Separate product from the MCP platform.
|
||||
- Should have its own CI/CD pipeline.
|
||||
- No direct dependency on MCP services.
|
||||
|
||||
### 2.5. Legacy / Cleanup Candidate
|
||||
|
||||
- **Name:** `opax-billing-test`
|
||||
- **Region:** `us-central1`
|
||||
- **URL:** No active URL (0% traffic)
|
||||
- **Image:** `.../opax-billing@sha256:f1e2d3c4...`
|
||||
- **Last deployed:** 2025-11-05
|
||||
- **Environment:** `test`
|
||||
- **Role:** Experimental billing service (no longer used)
|
||||
- **Notes:**
|
||||
- No traffic allocated.
|
||||
- Not updated in >9 months.
|
||||
- Label: `env=test`.
|
||||
- **Action:** Marked for deletion after final confirmation.
|
||||
|
||||
## 3. Environments
|
||||
|
||||
| Environment | MCP Service | Agent Service | Purpose |
|
||||
| :--- | :--- | :--- | :--- |
|
||||
| `prod` | `opax-mcp` | `osvauco-agent` | Live OSVx backend for Perplexity |
|
||||
| `staging` | `osvx-mcp-dev` | (shared or dedicated) | Pre-production testing of MCP |
|
||||
| `test` | (none active) | (none active) | Legacy/experimental (billing test) |
|
||||
|
||||
**Decisions:**
|
||||
|
||||
1. **Adopt naming:**
|
||||
- `osvx-mcp-prod` (currently `opax-mcp`)
|
||||
- `osvx-mcp-staging` (currently `osvx-mcp-dev`)
|
||||
2. **Decide agent strategy for staging:**
|
||||
- Option A (simple): staging MCP shares prod `osvauco-agent`.
|
||||
- Option B (isolated): create `osvauco-agent-staging` and wire staging MCP to it.
|
||||
|
||||
## 4. Dependencies
|
||||
|
||||
### 4.1. Service-to-service
|
||||
|
||||
- `opax-mcp` → `osvauco-agent`
|
||||
- Via environment variable `OSVAUCO_AGENT_URL`.
|
||||
- `osvx-mcp-dev` → `osvauco-agent` (or future `osvauco-agent-staging`).
|
||||
|
||||
### 4.2. External dependencies
|
||||
|
||||
- **Ollama endpoint**
|
||||
- Used by MCP for model access.
|
||||
- Currently: `http://34.13.238.133:11434` (from `/health` output).
|
||||
- **GCP services**
|
||||
- Secret Manager (`MCP_SECRET` for Bearer token).
|
||||
- Cloud Build (for future CI/CD).
|
||||
- Artifact Registry (container images).
|
||||
|
||||
### 4.3. Missing dependencies (known issues)
|
||||
|
||||
- **`gcloud` CLI in MCP container**
|
||||
- `trigger_build` tool in `opax-mcp/server.py` calls `gcloud` via subprocess.
|
||||
- Dockerfile does not install `google-cloud-sdk`.
|
||||
- **Result:** any use of `trigger_build` on prod/staging will fail with “gcloud not found”.
|
||||
- **Fix:** Add `google-cloud-sdk` installation to the Dockerfile before enabling this tool.
|
||||
|
||||
## 5. Deployment Model
|
||||
|
||||
### 5.1. Image strategy
|
||||
|
||||
- All services use pinned image digests (`@sha256:...`).
|
||||
- **CI/CD must:**
|
||||
1. Build images with a commit-SHA label, e.g. `gcb-commit-sha=<COMMIT_SHA>`.
|
||||
2. Deploy using the exact image digest produced by the build.
|
||||
|
||||
### 5.2. Environment promotion flow (target)
|
||||
|
||||
- **For the MCP platform:**
|
||||
1. On push to `main` / feature branch:
|
||||
- Build `opax-mcp` image with commit-SHA label.
|
||||
- Deploy to `osvx-mcp-staging` (`osvx-mcp-dev`).
|
||||
2. On manual approval or version tag:
|
||||
- Promote the same image to `osvx-mcp-prod` (`opax-mcp`).
|
||||
- **For the website:**
|
||||
- Separate pipeline:
|
||||
- Build `web-static` image.
|
||||
- Deploy to `vauco-site`.
|
||||
- **For the agent:**
|
||||
- Similar pattern if/when a staging agent is introduced.
|
||||
|
||||
### 5.3. Configuration gaps to address
|
||||
|
||||
- **No commit-SHA in images**
|
||||
- Images currently lack labels linking them to Git commits.
|
||||
- CI/CD should add `gcb-commit-sha` label during build.
|
||||
- **Inconsistent naming**
|
||||
- Prod MCP: `opax-mcp`
|
||||
- Dev MCP: `osvx-mcp-dev`
|
||||
- Target: `osvx-mcp-prod` + `osvx-mcp-staging`.
|
||||
|
||||
## 6. Security & Access (high level)
|
||||
|
||||
- **MCP services (`opax-mcp`, `osvx-mcp-dev`)**
|
||||
- Require Bearer token (`MCP_SECRET`) for MCP calls.
|
||||
- Should restrict invocations to:
|
||||
- Perplexity’s connector infrastructure
|
||||
- Trusted service accounts (if calling from other GCP services).
|
||||
- **Agent (`osvauco-agent`)**
|
||||
- Internal-only; no public ingress.
|
||||
- Should allow calls only from MCP service accounts.
|
||||
- **Website (`vauco-site`)**
|
||||
- Public read access.
|
||||
- No privileged operations.
|
||||
|
||||
## 7. Operational Notes
|
||||
|
||||
- **Health checks**
|
||||
- MCP: `GET /health` on each MCP service.
|
||||
- Agent: define a simple `/health` if not already present.
|
||||
- Site: `GET /` as basic availability check.
|
||||
- **Logging**
|
||||
- All services use Cloud Logging via Cloud Run.
|
||||
- Key logs to monitor:
|
||||
- MCP tool calls (`tools/call`), especially `daily_ops_report` and `trigger_build`.
|
||||
- Agent API errors.
|
||||
- **Incident focus**
|
||||
- If OSVx connector misbehaves:
|
||||
1. Check `opax-mcp` logs and revision.
|
||||
2. Verify `/health` version matches expected deployment.
|
||||
3. Confirm `MCP_SECRET` and connector URL.
|
||||
|
||||
## 8. Action Items
|
||||
|
||||
1. **Cleanup**
|
||||
- [ ] Confirm `opax-billing-test` is unused.
|
||||
- [ ] Delete `opax-billing-test` service.
|
||||
2. **Naming alignment**
|
||||
- [ ] Decide final names:
|
||||
- `osvx-mcp-prod` vs keeping `opax-mcp`.
|
||||
- `osvx-mcp-staging` vs keeping `osvx-mcp-dev`.
|
||||
- [ ] Optionally rename services in GCP to match.
|
||||
3. **Agent environment strategy**
|
||||
- [ ] Decide: shared prod agent vs dedicated staging agent.
|
||||
- [ ] Document the decision here and implement if needed.
|
||||
4. **Image labeling**
|
||||
- [ ] Update Cloud Build config / Dockerfile for `opax-mcp` to include `gcb-commit-sha`.
|
||||
5. **`gcloud` dependency**
|
||||
- [ ] Add `google-cloud-sdk` to the MCP Dockerfile before enabling `trigger_build`.
|
||||
6. **CI/CD design**
|
||||
- [ ] Use this document as the basis for:
|
||||
- Build pipelines per service.
|
||||
- Promotion rules (staging → prod).
|
||||
- Environment-specific configs.
|
||||
|
||||
## 9. Future Extensions (optional)
|
||||
|
||||
- Per-developer ephemeral environments (short-lived Cloud Run revisions).
|
||||
- Separate `osvx-mcp-sandbox` for integration tests in CI.
|
||||
- Centralized config service or Secret Manager–driven config for env-specific URLs.
|
||||
|
||||
*This document is the source of truth for the OSVx platform architecture.
|
||||
Update it whenever services, environments, or deployment patterns change.*
|
||||
|
||||
---
|
||||
|
||||
## 10. CI/CD Setup
|
||||
|
||||
This section describes the continuous integration and deployment architecture for the OSVx MCP platform.
|
||||
|
||||
### 10.1. Source of Truth
|
||||
|
||||
- **Primary Git repository:**
|
||||
Self-hosted Gitea: `http://34.59.131.162:3000/chris/osvauco.git`
|
||||
- **Primary branch for MCP:**
|
||||
`feat/osvx-mcp-full-catalog`
|
||||
|
||||
All development and code review happens in Gitea. No other Git host is used as a primary source.
|
||||
|
||||
### 10.2. Mirror to Cloud Source Repositories
|
||||
|
||||
Cloud Build cannot directly trigger on Gitea pushes, so the repo is mirrored to Cloud Source Repositories (CSR) as a read-only CI/CD mirror.
|
||||
|
||||
- **CSR project:** `357036551735`
|
||||
- **CSR repo name:** `osvauco`
|
||||
- **CSR repo URL:**
|
||||
`https://source.developers.google.com/p/357036551735/r/osvauco`
|
||||
|
||||
Mirror configuration (on the VM):
|
||||
|
||||
```bash
|
||||
cd ~/OSVauco
|
||||
git remote add csr https://source.developers.google.com/p/357036551735/r/osvauco
|
||||
git push csr feat/osvx-mcp-full-catalog:feat/osvx-mcp-full-catalog
|
||||
```
|
||||
|
||||
Mirroring is automated via either:
|
||||
|
||||
- A post-receive hook on the Gitea server that pushes `feat/osvx-mcp-full-catalog` to `csr`, or
|
||||
- A cron job on the VM that periodically runs:
|
||||
```bash
|
||||
git push --quiet csr feat/osvx-mcp-full-catalog:feat/osvx-mcp-full-catalog
|
||||
```
|
||||
|
||||
CSR is **read-only** from the CI/CD perspective; it exists only to trigger Cloud Build.
|
||||
|
||||
### 10.3. Cloud Build Trigger (Staging)
|
||||
|
||||
A Cloud Build trigger watches the CSR repo and runs the MCP pipeline on every push to the CI branch.
|
||||
|
||||
- **Trigger name:** `osvx-mcp-staging-trigger`
|
||||
- **Trigger type:** Cloud Source Repositories
|
||||
- **Project:** `357036551735`
|
||||
- **Region:** `us-central1`
|
||||
- **Repo:** `projects/357036551735/repos/osvauco`
|
||||
- **Branch pattern:** `^feat/osvx-mcp-full-catalog$`
|
||||
- **Build config:** `opax-mcp/cloudbuild.yaml`
|
||||
- **Substitutions:**
|
||||
- `_REGION=us-central1`
|
||||
- `_REPOSITORY=osvx-images`
|
||||
- `_ENV=staging`
|
||||
|
||||
Creation command (reference):
|
||||
|
||||
```bash
|
||||
PROJECT=357036551735
|
||||
REGION=us-central1
|
||||
REPO=osvx-images
|
||||
BRANCH=feat/osvx-mcp-full-catalog
|
||||
|
||||
gcloud builds triggers create cloud-source-repositories --project="$PROJECT" --name="osvx-mcp-staging-trigger" --region="$REGION" --repo="projects/$PROJECT/repos/osvauco" --branch-pattern="^${BRANCH}$" --build-config="opax-mcp/cloudbuild.yaml" --substitutions=_REGION=${REGION},_REPOSITORY=${REPO},_ENV=staging --included-files="opax-mcp/**" --description="Deploy OSVx MCP to staging on push to ${BRANCH}"
|
||||
```
|
||||
|
||||
**Behavior:**
|
||||
|
||||
- On every push to `feat/osvx-mcp-full-catalog` in Gitea (mirrored to CSR):
|
||||
- Cloud Build runs `opax-mcp/cloudbuild.yaml` with `_ENV=staging`.
|
||||
- The built image is deployed to `osvx-mcp-staging`.
|
||||
|
||||
### 10.4. Production Deployment (Manual Promotion)
|
||||
|
||||
Production deployments are intentionally manual to provide an explicit promotion gate.
|
||||
|
||||
**Options:**
|
||||
|
||||
1. **Local script (recommended):**
|
||||
```bash
|
||||
cd ~/OSVauco
|
||||
./deploy-mcp.sh prod
|
||||
```
|
||||
This builds the current commit and deploys to `osvx-mcp-prod`.
|
||||
|
||||
2. **One-off Cloud Build:**
|
||||
```bash
|
||||
PROJECT=357036551735
|
||||
REGION=us-central1
|
||||
REPO=osvx-images
|
||||
|
||||
gcloud builds submit . --project="$PROJECT" --config=opax-mcp/cloudbuild.yaml --substitutions=_REGION=${REGION},_REPOSITORY=${REPO},_ENV=prod
|
||||
```
|
||||
|
||||
No automatic prod trigger is configured.
|
||||
|
||||
### 10.5. Artifact Registry
|
||||
|
||||
- **Repo name:** `osvx-images`
|
||||
- **Location:** `us-central1`
|
||||
- **Format:** Docker
|
||||
|
||||
Images are tagged with the short commit SHA:
|
||||
|
||||
- `us-central1-docker.pkg.dev/357036551735/osvx-images/opax-mcp:<SHORT_SHA>`
|
||||
|
||||
Each image includes labels:
|
||||
|
||||
- `gcb-commit-sha=<SHORT_SHA>`
|
||||
- `env=staging` or `env=prod`
|
||||
|
||||
### 10.6. Operational Notes
|
||||
|
||||
- **Build logs and history:**
|
||||
GCP Console → Cloud Build → History
|
||||
- **Trigger configuration:**
|
||||
GCP Console → Cloud Build → Triggers → `osvx-mcp-staging-trigger`
|
||||
- **To temporarily disable CI:**
|
||||
Disable or delete `osvx-mcp-staging-trigger` in the Cloud Build console.
|
||||
- **To change the CI branch:**
|
||||
Update the trigger’s branch pattern and the mirror configuration.
|
||||
400
gitea-backup-candidate.sh
Normal file
400
gitea-backup-candidate.sh
Normal file
|
|
@ -0,0 +1,400 @@
|
|||
#!/usr/bin/env bash
|
||||
# Gitea application-consistent backup script.
|
||||
#
|
||||
# Operator-attestation basis:
|
||||
# - No intentionally required nested mount boundaries exist beneath the approved
|
||||
# source roots.
|
||||
# - The SQLite database is captured with SQLite's online .backup mechanism.
|
||||
# - Filesystem-tree copies are best effort while Gitea remains online.
|
||||
# - This script does not perform runtime mount-boundary or nested-symlink scans.
|
||||
#
|
||||
# The script archives only a staged SQLite backup, not the live SQLite DB,
|
||||
# WAL, SHM, or journal files.
|
||||
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
umask 077
|
||||
|
||||
readonly APP_INI_PATH="/opt/gitea/data/custom/conf/app.ini"
|
||||
readonly SQLITE_DB_PATH="/opt/gitea/data/data/gitea.db"
|
||||
readonly REPO_PATH="/opt/gitea/data/repositories"
|
||||
readonly LFS_PATH="/opt/gitea/data/lfs"
|
||||
readonly ATTACHMENTS_PATH="/opt/gitea/data/data/attachments"
|
||||
readonly AVATARS_PATH="/opt/gitea/data/data/avatars"
|
||||
readonly PACKAGES_PATH="/opt/gitea/data/data/packages"
|
||||
readonly REPO_ARCHIVE_PATH="/opt/gitea/data/data/repo-archive"
|
||||
readonly REPO_AVATARS_PATH="/opt/gitea/data/data/repo-avatars"
|
||||
|
||||
readonly GCS_BUCKET="gs://vauco-gitea-backups-20260901"
|
||||
readonly GCS_PREFIX="manual"
|
||||
readonly STAGING_DIR="/var/backups/gitea"
|
||||
readonly LOCK_FILE="/var/run/gitea-backup.lock"
|
||||
readonly LOG_FILE="/var/log/gitea-backup.log"
|
||||
readonly REQUIRED_FREE_KB=646368
|
||||
|
||||
workspace=""
|
||||
workspace_preserved=0
|
||||
log_ready=0
|
||||
|
||||
log_status() {
|
||||
local phase="$1"
|
||||
local status="$2"
|
||||
local category="$3"
|
||||
|
||||
if [ "${log_ready}" -eq 1 ]; then
|
||||
printf '%s phase=%s status=%s category=%s\n' \
|
||||
"$(date -u --iso-8601=seconds)" \
|
||||
"${phase}" \
|
||||
"${status}" \
|
||||
"${category}" >> "${LOG_FILE}"
|
||||
else
|
||||
printf '%s\n' \
|
||||
"gitea-backup phase=${phase} status=${status} category=${category}" >&2
|
||||
fi
|
||||
}
|
||||
|
||||
fail_without_workspace() {
|
||||
log_status "$1" "FAILURE" "$2"
|
||||
exit 1
|
||||
}
|
||||
|
||||
workspace_is_safe() {
|
||||
[ -n "${workspace}" ] &&
|
||||
[ -d "${workspace}" ] &&
|
||||
[ ! -L "${workspace}" ] &&
|
||||
[ "$(dirname -- "${workspace}")" = "${STAGING_DIR}" ] &&
|
||||
[ "$(stat -c '%u:%g:%a' -- "${workspace}")" = "0:0:700" ]
|
||||
}
|
||||
|
||||
remove_workspace() {
|
||||
workspace_is_safe || return 1
|
||||
rm -rf -- "${workspace}"
|
||||
}
|
||||
|
||||
preserve_workspace() {
|
||||
workspace_is_safe || return 1
|
||||
|
||||
if find -- "${workspace}" -xdev -type l -print -quit 2>/dev/null |
|
||||
grep -q . >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if find -- "${workspace}" -xdev \
|
||||
! -type d ! -type f ! -type l \
|
||||
-print -quit 2>/dev/null |
|
||||
grep -q . >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! find -- "${workspace}" -xdev -type d \
|
||||
-exec chmod 700 -- {} + >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! find -- "${workspace}" -xdev -type f \
|
||||
-exec chmod 600 -- {} + >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
workspace_preserved=1
|
||||
trap - EXIT
|
||||
return 0
|
||||
}
|
||||
|
||||
fail_after_workspace() {
|
||||
log_status "$1" "FAILURE" "$2"
|
||||
|
||||
if ! preserve_workspace; then
|
||||
log_status "PRESERVATION" "FAILURE" "WorkspacePreservationFailed"
|
||||
fi
|
||||
|
||||
exit 1
|
||||
}
|
||||
|
||||
cleanup_on_exit() {
|
||||
local rc=$?
|
||||
|
||||
if [ "${workspace_preserved}" -eq 0 ]; then
|
||||
remove_workspace >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
require_command() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
fail_without_workspace "PREFLIGHT" "RequiredCommandMissing"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_regular() {
|
||||
if ! [ -f "$1" ] || [ -L "$1" ]; then
|
||||
fail_without_workspace "PREFLIGHT" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_directory() {
|
||||
if ! [ -d "$1" ] || [ -L "$1" ]; then
|
||||
fail_without_workspace "PREFLIGHT" "$2"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_archive_manifest() {
|
||||
local archive_path="$1"
|
||||
|
||||
tar -tzf "${archive_path}" >/dev/null 2>&1 || return 1
|
||||
|
||||
tar -tf "${archive_path}" 2>/dev/null |
|
||||
awk '
|
||||
BEGIN {
|
||||
allowed["database"] = 1
|
||||
allowed["config"] = 1
|
||||
allowed["repositories"] = 1
|
||||
allowed["lfs"] = 1
|
||||
allowed["data"] = 1
|
||||
|
||||
required["database/"] = 0
|
||||
required["config/"] = 0
|
||||
required["repositories/"] = 0
|
||||
required["lfs/"] = 0
|
||||
required["data/"] = 0
|
||||
}
|
||||
|
||||
/^\// { bad = 1; exit 1 }
|
||||
/(^|\/)\.\.(\/|$)/ { bad = 1; exit 1 }
|
||||
|
||||
{
|
||||
member = $0
|
||||
split(member, parts, "/")
|
||||
root = parts[1]
|
||||
|
||||
if (!(root in allowed)) {
|
||||
bad = 1
|
||||
exit 1
|
||||
}
|
||||
|
||||
if (member in required) {
|
||||
required[member] = 1
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
if (bad) {
|
||||
exit 1
|
||||
}
|
||||
|
||||
for (entry in required) {
|
||||
if (required[entry] != 1) {
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
}'
|
||||
}
|
||||
|
||||
copy_source() {
|
||||
if ! cp -a -- "$1" "$2"; then
|
||||
fail_after_workspace "STAGING" "$3"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local required_command=""
|
||||
local archive_root=""
|
||||
local run_id=""
|
||||
local archive_name=""
|
||||
local archive_path=""
|
||||
local checksum_path=""
|
||||
local archive_object=""
|
||||
local checksum_object=""
|
||||
|
||||
for required_command in \
|
||||
flock df awk mktemp mkdir sqlite3 cp tar sha256sum gcloud \
|
||||
stat find chmod rm dirname grep date printf; do
|
||||
require_command "${required_command}"
|
||||
done
|
||||
|
||||
if ! [ -f "${LOG_FILE}" ] || [ -L "${LOG_FILE}" ]; then
|
||||
printf '%s\n' \
|
||||
"gitea-backup phase=PREFLIGHT status=FAILURE category=LogFileInvalid" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$(stat -c '%u:%g:%a' -- "${LOG_FILE}")" != "0:0:600" ]; then
|
||||
printf '%s\n' \
|
||||
"gitea-backup phase=PREFLIGHT status=FAILURE category=LogFilePermissionsInvalid" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [ -w "${LOG_FILE}" ]; then
|
||||
printf '%s\n' \
|
||||
"gitea-backup phase=PREFLIGHT status=FAILURE category=LogFileUnavailable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_ready=1
|
||||
log_status "PREFLIGHT" "START" "BackupJob"
|
||||
|
||||
if ! [ -d "${STAGING_DIR}" ] || [ -L "${STAGING_DIR}" ]; then
|
||||
fail_without_workspace "PREFLIGHT" "StagingDirectoryInvalid"
|
||||
fi
|
||||
|
||||
if [ "$(stat -c '%u:%g:%a' -- "${STAGING_DIR}")" != "0:0:700" ]; then
|
||||
fail_without_workspace "PREFLIGHT" "StagingDirectoryPermissionsInvalid"
|
||||
fi
|
||||
|
||||
if ! [ -w "${STAGING_DIR}" ]; then
|
||||
fail_without_workspace "PREFLIGHT" "StagingDirectoryUnavailable"
|
||||
fi
|
||||
|
||||
if ! df --output=avail -- "${STAGING_DIR}" 2>/dev/null |
|
||||
awk -v required_kb="${REQUIRED_FREE_KB}" '
|
||||
NR == 2 {
|
||||
checked = 1
|
||||
exit !($1 > required_kb)
|
||||
}
|
||||
END {
|
||||
if (!checked) {
|
||||
exit 1
|
||||
}
|
||||
}'; then
|
||||
fail_without_workspace "PREFLIGHT" "InsufficientStagingCapacity"
|
||||
fi
|
||||
|
||||
validate_regular "${APP_INI_PATH}" "AppIniInvalid"
|
||||
validate_regular "${SQLITE_DB_PATH}" "SQLiteDatabaseInvalid"
|
||||
|
||||
validate_directory "${REPO_PATH}" "RepositoriesInvalid"
|
||||
validate_directory "${LFS_PATH}" "LfsInvalid"
|
||||
validate_directory "${ATTACHMENTS_PATH}" "AttachmentsInvalid"
|
||||
validate_directory "${AVATARS_PATH}" "AvatarsInvalid"
|
||||
validate_directory "${PACKAGES_PATH}" "PackagesInvalid"
|
||||
validate_directory "${REPO_ARCHIVE_PATH}" "RepositoryArchivesInvalid"
|
||||
validate_directory "${REPO_AVATARS_PATH}" "RepositoryAvatarsInvalid"
|
||||
|
||||
if ! workspace="$(mktemp -d -p "${STAGING_DIR}" "backup.XXXXXX")"; then
|
||||
fail_without_workspace "PREFLIGHT" "WorkspaceCreationFailed"
|
||||
fi
|
||||
|
||||
if ! workspace_is_safe; then
|
||||
fail_after_workspace "PREFLIGHT" "WorkspaceValidationFailed"
|
||||
fi
|
||||
|
||||
trap cleanup_on_exit EXIT
|
||||
|
||||
archive_root="${workspace}/archive_root"
|
||||
|
||||
if ! mkdir -p -- \
|
||||
"${archive_root}/database" \
|
||||
"${archive_root}/config" \
|
||||
"${archive_root}/data"; then
|
||||
fail_after_workspace "STAGING" "ArchiveRootCreationFailed"
|
||||
fi
|
||||
|
||||
log_status "DB_COPY" "START" "SQLiteOnlineBackup"
|
||||
if ! sqlite3 "${SQLITE_DB_PATH}" \
|
||||
".backup '${archive_root}/database/gitea.db'"; then
|
||||
fail_after_workspace "DB_COPY" "SQLiteBackupFailed"
|
||||
fi
|
||||
log_status "DB_COPY" "SUCCESS" "SQLiteOnlineBackupComplete"
|
||||
|
||||
log_status "STAGING" "START" "CopyingData"
|
||||
|
||||
copy_source "${APP_INI_PATH}" \
|
||||
"${archive_root}/config/app.ini" \
|
||||
"CopyAppIniFailed"
|
||||
|
||||
copy_source "${REPO_PATH}" \
|
||||
"${archive_root}/repositories" \
|
||||
"CopyRepositoriesFailed"
|
||||
|
||||
copy_source "${LFS_PATH}" \
|
||||
"${archive_root}/lfs" \
|
||||
"CopyLfsFailed"
|
||||
|
||||
copy_source "${ATTACHMENTS_PATH}" \
|
||||
"${archive_root}/data/attachments" \
|
||||
"CopyAttachmentsFailed"
|
||||
|
||||
copy_source "${AVATARS_PATH}" \
|
||||
"${archive_root}/data/avatars" \
|
||||
"CopyAvatarsFailed"
|
||||
|
||||
copy_source "${PACKAGES_PATH}" \
|
||||
"${archive_root}/data/packages" \
|
||||
"CopyPackagesFailed"
|
||||
|
||||
copy_source "${REPO_ARCHIVE_PATH}" \
|
||||
"${archive_root}/data/repo-archive" \
|
||||
"CopyRepositoryArchivesFailed"
|
||||
|
||||
copy_source "${REPO_AVATARS_PATH}" \
|
||||
"${archive_root}/data/repo-avatars" \
|
||||
"CopyRepositoryAvatarsFailed"
|
||||
|
||||
log_status "STAGING" "SUCCESS" "CopyComplete"
|
||||
|
||||
run_id="${workspace##*/}"
|
||||
archive_name="gitea-backup-${run_id}.tar.gz"
|
||||
archive_path="${workspace}/${archive_name}"
|
||||
checksum_path="${archive_path}.sha256"
|
||||
archive_object="${GCS_BUCKET}/${GCS_PREFIX}/${archive_name}"
|
||||
checksum_object="${GCS_BUCKET}/${GCS_PREFIX}/${archive_name}.sha256"
|
||||
|
||||
log_status "ARCHIVE" "START" "TarCreation"
|
||||
if ! tar -czf "${archive_path}" \
|
||||
-C "${archive_root}" \
|
||||
database config repositories lfs data; then
|
||||
fail_after_workspace "ARCHIVE" "TarCreationFailed"
|
||||
fi
|
||||
|
||||
if ! validate_archive_manifest "${archive_path}"; then
|
||||
fail_after_workspace "ARCHIVE" "ManifestInvalid"
|
||||
fi
|
||||
log_status "ARCHIVE" "SUCCESS" "ManifestValidated"
|
||||
|
||||
if ! (
|
||||
cd -- "${workspace}"
|
||||
sha256sum -b -- "${archive_name}" > "${checksum_path}"
|
||||
); then
|
||||
fail_after_workspace "CHECKSUM" "ChecksumCreationFailed"
|
||||
fi
|
||||
log_status "CHECKSUM" "SUCCESS" "ChecksumGenerated"
|
||||
|
||||
log_status "UPLOAD_ARCHIVE" "START" "GCS"
|
||||
if ! gcloud storage cp \
|
||||
--if-generation-match=0 \
|
||||
--quiet \
|
||||
"${archive_path}" \
|
||||
"${archive_object}" >/dev/null 2>&1; then
|
||||
fail_after_workspace "UPLOAD_ARCHIVE" "ArchiveUploadFailed"
|
||||
fi
|
||||
log_status "UPLOAD_ARCHIVE" "SUCCESS" "ArchiveUploadComplete"
|
||||
|
||||
log_status "UPLOAD_CHECKSUM" "START" "GCS"
|
||||
if ! gcloud storage cp \
|
||||
--if-generation-match=0 \
|
||||
--quiet \
|
||||
"${checksum_path}" \
|
||||
"${checksum_object}" >/dev/null 2>&1; then
|
||||
fail_after_workspace "UPLOAD_CHECKSUM" "ChecksumUploadFailed"
|
||||
fi
|
||||
log_status "UPLOAD_CHECKSUM" "SUCCESS" "ChecksumUploadComplete"
|
||||
|
||||
if ! remove_workspace; then
|
||||
fail_after_workspace "CLEANUP" "WorkspaceRemovalFailed"
|
||||
fi
|
||||
|
||||
workspace=""
|
||||
trap - EXIT
|
||||
log_status "JOB" "SUCCESS" "BackupComplete"
|
||||
}
|
||||
|
||||
(
|
||||
flock -n 200 || {
|
||||
printf '%s\n' \
|
||||
"gitea-backup phase=PREFLIGHT status=FAILURE category=LockHeld" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
main
|
||||
) 200>"${LOCK_FILE}"
|
||||
16
opax-mcp/quick_verify.py
Normal file
16
opax-mcp/quick_verify.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from pathlib import Path
|
||||
import logging
|
||||
|
||||
# Suppress logger during import to avoid printing config values
|
||||
logging.getLogger("osvx_mcp_server_under_test").setLevel(logging.WARNING)
|
||||
|
||||
SERVER_PATH = Path(__file__).with_name("server.py")
|
||||
spec = spec_from_file_location("osvx_mcp_server_under_test", SERVER_PATH)
|
||||
server = module_from_spec(spec)
|
||||
spec.loader.exec_module(server)
|
||||
|
||||
print(f"REGISTERED_TOOL_COUNT={len(server.TOOLS)}")
|
||||
print("--- REGISTERED_TOOL_NAMES ---")
|
||||
for name in sorted(server.TOOLS.keys()):
|
||||
print(name)
|
||||
|
|
@ -20,6 +20,21 @@ from email.mime.text import MIMEText
|
|||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any, Optional
|
||||
import logging
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ConfirmationRequired(BaseModel):
|
||||
message: str
|
||||
preview: dict
|
||||
requires_confirmation: bool = True
|
||||
|
||||
def require_confirmation(confirm: bool, preview: dict, message: str = "This action requires confirmation"):
|
||||
if not confirm:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=ConfirmationRequired(message=message, preview=preview).model_dump()
|
||||
)
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -50,19 +65,13 @@ logger.info(f"OLLAMA_BASE_URL: {OLLAMA_BASE_URL}")
|
|||
# Auth (kun for innkommende kall til opax-mcp, f.eks. fra Gemini TUI)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _verify_auth(request: Request):
|
||||
"""Sjekker at innkommende kall til DENNE tjenesten (opax-mcp) er autentisert."""
|
||||
if not MCP_SECRET:
|
||||
logger.warning("MCP_SECRET is not set, skipping auth verification.")
|
||||
return
|
||||
async def _verify_auth(request: Request) -> None:
|
||||
secret = os.getenv("MCP_SECRET")
|
||||
if not secret:
|
||||
raise HTTPException(status_code=500, detail="MCP_SECRET not configured")
|
||||
auth = request.headers.get("Authorization", "")
|
||||
if auth.startswith("Bearer ") and auth[7:] == MCP_SECRET:
|
||||
return
|
||||
if request.headers.get("X-MCP-Secret") == MCP_SECRET:
|
||||
return
|
||||
if request.headers.get("api-key") == MCP_SECRET:
|
||||
return
|
||||
raise HTTPException(status_code=401, detail="Unauthorized")
|
||||
if not auth.startswith("Bearer ") or auth[7:] != secret:
|
||||
raise HTTPException(status_code=401, detail="Invalid or missing token")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user