From 9cd43572b02419b7070d00972473a62de877f20b Mon Sep 17 00:00:00 2001 From: Chris Christiansen Date: Mon, 25 May 2026 12:49:52 +0000 Subject: [PATCH] chore: add untracked RAG, infra and session files --- agents/rag/sync_corpus.py | 132 +++++++++++++++++ docs/RAG-SECONDARY-PROJECT-PLAN.md | 84 +++++++++++ infrastructure/99-region-guard.sh | 58 ++++++++ ...-24_osvauco_rag_and_operator_experience.md | 137 ++++++++++++++++++ 4 files changed, 411 insertions(+) create mode 100644 agents/rag/sync_corpus.py create mode 100644 docs/RAG-SECONDARY-PROJECT-PLAN.md create mode 100644 infrastructure/99-region-guard.sh create mode 100644 protocols/sessions/2026-05-24_osvauco_rag_and_operator_experience.md diff --git a/agents/rag/sync_corpus.py b/agents/rag/sync_corpus.py new file mode 100644 index 0000000..de8022c --- /dev/null +++ b/agents/rag/sync_corpus.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +""" +sync_corpus.py — Synchronize RAG Engine corpus with GCS and perform health checks. +Project: propane-will-491900-m5 + +Capabilities: +- Locates existing corpus by display name. +- Lists current files in the corpus. +- Triggers incremental import from GCS. +- Performs a health-check retrieval query. +- Reports status and chunk counts. + +Graceful degradation: +- If corpus is not found (e.g. due to Spanner restriction), logs a warning and exits cleanly. +""" + +import os +import json +import time +import subprocess +import sys + +import vertexai +from vertexai import rag +from vertexai.rag import RagCorpus + +PROJECT_ID = "propane-will-491900-m5" +LOCATION = os.environ.get("RAG_LOCATION", "us-central1") +CORPUS_DISPLAY_NAME = os.environ.get("RAG_CORPUS_NAME", "osvauco-knowledge-base") +GCS_SOURCE = os.environ.get( + "RAG_GCS_SOURCE", + f"gs://{PROJECT_ID}-agent-staging/rag-docs/", +) + +def get_token() -> str: + return subprocess.check_output( + ["gcloud", "auth", "print-access-token"], text=True + ).strip() + +def find_corpus() -> RagCorpus: + """Search for the corpus in the configured location and fallback location.""" + locations_to_try = [LOCATION] + if LOCATION != "europe-west4": + locations_to_try.append("europe-west4") + + for loc in locations_to_try: + print(f"Searching for corpus '{CORPUS_DISPLAY_NAME}' in {loc}...") + vertexai.init(project=PROJECT_ID, location=loc) + try: + for c in rag.list_corpora(): + if c.display_name == CORPUS_DISPLAY_NAME: + print(f" \u2713 Found: {c.name}") + return c + except Exception as e: + print(f" \u26a0\ufe0f Notice: Could not list corpora in {loc}: {e}") + + return None + +def list_files(corpus: RagCorpus): + """List files currently in the corpus.""" + print(f"Listing files in corpus...") + files = list(rag.list_files(corpus_name=corpus.name)) + if not files: + print(" (Corpus is empty)") + return [] + + for f in files: + print(f" - {f.display_name} (state: {f.state.name})") + return files + +def sync_from_gcs(corpus: RagCorpus): + """Trigger incremental import from GCS.""" + print(f"Syncing from {GCS_SOURCE}...") + # import_files in SDK is essentially an incremental trigger + rag.import_files( + corpus.name, + paths=[GCS_SOURCE], + transformation_config=rag.TransformationConfig( + chunking_config=rag.ChunkingConfig( + chunk_size=512, + chunk_overlap=100, + ) + ), + ) + print(" \u2713 Sync job submitted.") + +def run_health_check(corpus: RagCorpus): + """Perform a retrieval query and report results.""" + print("Running health check retrieval...") + try: + response = rag.retrieval_query( + rag_resources=[rag.RagResource(rag_corpus=corpus.name)], + text="How does OSVauco handle operator experience?", + rag_retrieval_config=rag.RagRetrievalConfig(top_k=3), + ) + contexts = response.contexts.contexts + print(f" \u2713 Health check returned {len(contexts)} contexts.") + for i, ctx in enumerate(contexts): + print(f" [{i+1}] Source: {ctx.source_uri} (Score: {ctx.score:.4f})") + except Exception as e: + print(f" \u2718 Health check failed: {e}") + +def main(): + print(f"--- RAG Sync & Maintenance Tool ---") + + corpus = find_corpus() + if not corpus: + print(f"\n\u26a0\ufe0f WARN: Corpus '{CORPUS_DISPLAY_NAME}' not found.") + print(" RAG Engine might be restricted for this project or setup_corpus.py has not been run.") + print(" Skipping sync and health check.") + sys.exit(0) + + # Update global LOCATION if we found it in a different one + found_loc = corpus.name.split('/')[3] + print(f"Active Location: {found_loc}") + + list_files(corpus) + + # Always trigger a sync to ensure new files are picked up + sync_from_gcs(corpus) + + print("\nWaiting 5s for sync initialization...") + time.sleep(5) + + run_health_check(corpus) + + print(f"\nCOMPLETED") + print(f"RAG_LOCATION={found_loc}") + print(f"RAG_CORPUS={corpus.name}") + +if __name__ == "__main__": + main() diff --git a/docs/RAG-SECONDARY-PROJECT-PLAN.md b/docs/RAG-SECONDARY-PROJECT-PLAN.md new file mode 100644 index 0000000..eb7c569 --- /dev/null +++ b/docs/RAG-SECONDARY-PROJECT-PLAN.md @@ -0,0 +1,84 @@ +# Plan: Secondary RAG-Dedicated Project Migration + +## 1. Objective +To bypass the "Spanner mode" restriction currently blocking serverless RAG Engine corpus creation in `propane-will-491900-m5`. We will spin up a clean, dedicated project for RAG experimentation and wire OSVauco to it as a secondary data source. + +## 2. New Project Setup + +### Project Selection +* **Suggested Name:** `vauco-rag-lab-01` or `osvauco-memory-node`. +* **Organization:** Ensure it is created under the same billing account as the primary project. + +### Step 1: Bootstrap Command +```bash +NEW_RAG_PROJECT="vauco-rag-lab-01" +BILLING_ACCOUNT=$(gcloud billing projects describe propane-will-491900-m5 --format="value(billingAccountName)") + +# Create project +gcloud projects create ${NEW_RAG_PROJECT} --name="OSVauco RAG Lab" + +# Link billing +gcloud billing projects link ${NEW_RAG_PROJECT} --billing-account=${BILLING_ACCOUNT} +``` + +### Step 2: Minimal APIs +Enable only what is needed for RAG Engine and its data source: +```bash +gcloud services enable \ + aiplatform.googleapis.com \ + storage.googleapis.com \ + --project=${NEW_RAG_PROJECT} +``` + +### Step 3: Cross-Project IAM +Grant the primary OSVauco Agent (from `propane-will-491900-m5`) access to the new RAG project: +```bash +PRIMARY_SA="jason.vauger@propane-will-491900-m5.iam.gserviceaccount.com" + +# Grant Vertex AI User and Storage Admin on the NEW project +for ROLE in roles/aiplatform.user roles/storage.objectAdmin; do + gcloud projects add-iam-policy-binding ${NEW_RAG_PROJECT} \ + --member="serviceAccount:${PRIMARY_SA}" \ + --role="${ROLE}" +done +``` + +## 3. RAG Setup & Sync Refactoring + +We will use variants of our existing scripts, but modified to accept a `--project` override. + +### `setup_corpus_remote.py` +A variant of `setup_corpus.py` that: +1. Targets `NEW_RAG_PROJECT`. +2. Attempts the same "Serverless" PATCH in `us-central1` or `europe-west4`. +3. If it succeeds, it saves the resulting corpus name to a shared secret in the **Primary** project. + +### `sync_corpus_remote.py` +A variant of `sync_corpus.py` that reads from the Primary project's GCS bucket but writes/indexes to the Remote project's RAG Engine. + +## 4. Dual-Project Configuration Strategy + +OSVauco will be updated to handle "Project Context Switching." + +### Environment Variables +The following overrides will be added to `.env`: +* `PROJECT_ID`: `propane-will-491900-m5` (Primary logic/infra) +* **`RAG_PROJECT_ID`**: `vauco-rag-lab-01` (Knowledge base source) +* **`RAG_LOCATION`**: `europe-west4` (Region for the remote corpus) + +### Code Updates +1. **`agents/rag/*`**: All RAG scripts and agent tools will default to `RAG_PROJECT_ID` (falling back to `PROJECT_ID`) during `vertexai.init()`. +2. **`scripts/preflight.sh`**: Will check both projects. + * Infrastructure section -> Primary Project. + * Knowledge Base section -> Remote Project. +3. **`scripts/cost-lens.sh`**: Will aggregate costs from both projects if `HEAVY_MODE` is enabled. + +## 5. Migration Timeline + +1. **Project Creation**: Create `vauco-rag-lab-01` and link billing. +2. **IAM Wiring**: Establish the cross-project service account trust. +3. **Test Run**: Run `setup_corpus.py` targeting the new project to verify if "Spanner mode" restriction is absent. +4. **Integration**: Update `.env` and `preflight.sh` to recognize the secondary project. + +--- +*Senior Staff Engineer (Gemini CLI)* diff --git a/infrastructure/99-region-guard.sh b/infrastructure/99-region-guard.sh new file mode 100644 index 0000000..67ead79 --- /dev/null +++ b/infrastructure/99-region-guard.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# infrastructure/99-region-guard.sh — Regional availability intelligence +# Sourced by other infrastructure scripts to ensure regional compliance. + +# --- 1. Known Regional Capabilities (v1beta1) --- + +# Vertex AI RAG Engine (Serverless) availability +# Source: https://cloud.google.com/vertex-ai/docs/generative-ai/rag-overview +RAG_SUPPORTED_REGIONS=("us-central1" "us-east1" "us-east4" "europe-west1" "europe-west4" "europe-west9" "asia-northeast1") + +# Vertex AI Agent Engine (Memory Bank / Reasoning Engine) availability +# Most Vertex AI regions support this, but we track primary ones. +AGENT_ENGINE_SUPPORTED_REGIONS=("us-central1" "europe-west4") + +# Cloud Run +# Virtually all regions support Cloud Run, but we prefer those with low latency or specific compliance. +# Default for OSVauco is us-central1 (Management Plane). +# Default for clinical data is europe-north1. + +# --- 2. Helper Functions --- + +# Check if a value is in an array +contains_element() { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +# Validate RAG region +validate_rag_region() { + local TARGET_REGION="${1:-us-central1}" + if contains_element "$TARGET_REGION" "${RAG_SUPPORTED_REGIONS[@]}"; then + echo "$TARGET_REGION" + else + # Fallback to a safe region known to support serverless RAG + echo "europe-west4" + fi +} + +# Validate Agent Engine region +validate_agent_engine_region() { + local TARGET_REGION="${1:-us-central1}" + if contains_element "$TARGET_REGION" "${AGENT_ENGINE_SUPPORTED_REGIONS[@]}"; then + echo "$TARGET_REGION" + else + # Fallback + echo "us-central1" + fi +} + +# Log regional context +log_region_context() { + local SVC=$1 + local REGION=$2 + local STATUS=$3 + echo -e " [REGION-GUARD] Service: ${SVC} | Region: ${REGION} | Status: ${STATUS}" +} diff --git a/protocols/sessions/2026-05-24_osvauco_rag_and_operator_experience.md b/protocols/sessions/2026-05-24_osvauco_rag_and_operator_experience.md new file mode 100644 index 0000000..31259d3 --- /dev/null +++ b/protocols/sessions/2026-05-24_osvauco_rag_and_operator_experience.md @@ -0,0 +1,137 @@ +# Engineering Brief: RAG Robustness & Operator Experience +**Date:** 2026-05-24 +**Author:** Senior Staff Engineer (Gemini CLI) +**Status:** PROPOSED + +## 1. Vision & Roadmap Alignment + +OSVauco (OPAX) is evolving from a collection of scripts into a cohesive **AI Operating System**. The Roadmap (Phase 4-6) and Masterplan emphasize a transition from "getting it up" to "keeping it smooth, secure, and multi-client ready." + +The core of this experience is the **Operator Cockpit**—where Gemini, agents (Jason Vauger), and the TUI bridge converge to manage a complex GCP estate. + +### The Role of RAG +RAG is the "long-term memory" of OPAX. It shouldn't just be a one-time setup; it must be a living repository of the operator's intent, session logs, and architectural rules. + +## 2. Current RAG State (Baseline) + +* **What Works:** + * Serverless corpus creation via REST (bypassing SDK bugs). + * Automatic regional fallback (`us-central1` -> `europe-west4`) to avoid Spanner restrictions. + * Initial document import from GCS. +* **What's Fragile:** + * **Synchronization:** No automated way to re-sync if new files are added to GCS. + * **Health Checks:** No quick way to verify if the corpus is "alive" or if the embedding model is responding. + * **Discovery:** The corpus name is saved to Secret Manager but not easily discoverable by other agents without manual environment variable plumbing. + +## 3. Proposed "Heavy Lifts" + +### Lift 1: Automated RAG Sync & Maintenance Tool +**Goal:** Create `agents/rag/sync_corpus.py` (and a corresponding agent tool) that performs incremental imports and health checks. + +### Lift 2: Operator "Preflight" Dashboard (CLI) +**Goal:** A new script `scripts/preflight.sh` that provides a high-signal summary for the operator at the start of a session. + +### Lift 3: Jason Vauger "Cost-Lens" Tooling +**Goal:** Enhance the core agent logic to support structured cost queries. + +### Lift 4: Multi-Region Infrastructure Guard +**Goal:** Refactor `infrastructure/*.sh` to be fully region-agnostic. + +--- + +## Implementation Log + +### Heavy Lift 1: Automated RAG Sync & Maintenance Tool +**Status:** COMPLETED +**Date:** 2026-05-24 + +* **Changes:** + * Created `agents/rag/sync_corpus.py`. + * Implemented multi-location discovery logic (`us-central1` and `europe-west4`). + * Added file listing functionality to report on corpus content. + * Integrated incremental sync from GCS via `rag.import_files`. + * Added a health-check retrieval query to verify the end-to-end RAG pipeline. +* **Why:** To ensure the RAG corpus remains a "living" repository that stays in sync with GCS-based documentation and allows operators to verify system health. +* **How to run/test:** + ```bash + # 1. Ensure corpus exists (or create it) + python3 agents/rag/setup_corpus.py + + # 2. Sync and check health + python3 agents/rag/sync_corpus.py + ``` + +### Heavy Lift 2: Operator Preflight Dashboard +**Status:** COMPLETED +**Date:** 2026-05-24 + +* **Changes:** + * Created `scripts/preflight.sh`. + * Implemented Cloud Run service health check. + * Added RAG corpus discovery with automatic SDK fallback for environments missing `gcloud ai rag` commands. + * Integrated an "Idle Cost Risk" audit (VMs, SQL, Memory Bank). + * Added Secret Manager verification for critical tokens. +* **Why:** To give the operator immediate situational awareness and prevent "billing surprises" or broken deployments before starting a session. +* **How to run/test:** + ```bash + bash scripts/preflight.sh + ``` + +### Heavy Lift 3: Jason Vauger "Cost-Lens" Tooling +**Status:** COMPLETED +**Date:** 2026-05-24 + +* **Changes:** + * Created `scripts/cost-lens.sh`. + * Implemented "Idle Cost Risk" detection for VMs, SQL, Reasoning Engines, and Cloud Run min-instances. + * Added `get_price_hint` helper for rough monthly cost projections. + * Added 7-day cost trend visualization (with instructions for BigQuery billing export). + * Implemented `--heavy` mode for structured YAML/JSON output designed for LLM agent ingestion. +* **Why:** To empower the operator and the OPAX agent (Jason Vauger) with structured, actionable cost intelligence. +* **How to run/test:** + ```bash + # Human-readable report + bash scripts/cost-lens.sh + + # Machine-readable "Heavy Mode" + bash scripts/cost-lens.sh --heavy + ``` + +### Heavy Lift 4: Multi-Region Infrastructure Guard +**Status:** COMPLETED +**Date:** 2026-05-24 + +* **Changes:** + * Created `infrastructure/99-region-guard.sh` to centralize regional availability logic. + * Updated `infrastructure/07-rag-setup.sh` and `infrastructure/08-memorybank-setup.sh` to use the guard for automatic fallback to supported regions. + * Refactored `infrastructure/05-cloudrun-deploy.sh` to be fully region-agnostic, supporting `DEPLOY_REGION` overrides for cross-region deployments (e.g., clinical data in `europe-north1`). + * Updated `infrastructure/01-setupenv.sh` to integrate regional logging and validation. +* **Why:** To ensure the OPAX infrastructure can scale across multiple regions and clients while respecting GCP service availability and compliance requirements (e.g., data residency). +* **How to run/test:** + ```bash + # Normal setup (respects .env REGION) + bash infrastructure/01-setupenv.sh + + # Deployment to a specific region + DEPLOY_REGION=europe-north1 bash infrastructure/05-cloudrun-deploy.sh + ``` +* **Open Follow-ups:** + * Expand the guard to include VPC Service Controls and other compliance-heavy services. + * Add automated "region discovery" to suggest the best region based on latency from the operator's current location. + +## Docs Hardening – 2026-05-24 +**Status:** COMPLETED + +* **Updated Docs:** + * `docs/LEARNINGS.md`: Added **LEARNING-007** regarding Vertex AI RAG Engine Spanner restrictions and our "graceful degradation" strategy. + * `docs/SECRETS-SETUP.md`: Refreshed list of essential secrets (`github-token`, `webhook-url`, `rag-corpus-name`) with updated `gcloud` commands and service account mappings. + * `docs/IAP-SETUP.md`: Clarified current status—OPAX is on Cloud Run, with IAP planned for Phase 5. +* **Key Decisions:** + * **RAG Unavailable:** Formally recognized that RAG Engine is restricted to Spanner mode for this project. The platform now skips RAG operations cleanly rather than failing. + * **Operator Focus:** Shifted documentation priority towards new tools (`preflight.sh`, `cost-lens.sh`) to empower the operator despite the RAG limitation. +* **Open Follow-ups:** + * Update `MASTERPLAN.md` quick-start section to include `bash scripts/preflight.sh` as the first step. + * Document the `infrastructure/99-region-guard.sh` logic in `ARCHITECTURE.md`. + +--- +*Senior Staff Engineer (Gemini CLI)*