chore: add untracked RAG, infra and session files
This commit is contained in:
parent
b73d140032
commit
9cd43572b0
132
agents/rag/sync_corpus.py
Normal file
132
agents/rag/sync_corpus.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
sync_corpus.py — Synchronize RAG Engine corpus with GCS and perform health checks.
|
||||
Project: propane-will-491900-m5
|
||||
|
||||
Capabilities:
|
||||
- Locates existing corpus by display name.
|
||||
- Lists current files in the corpus.
|
||||
- Triggers incremental import from GCS.
|
||||
- Performs a health-check retrieval query.
|
||||
- Reports status and chunk counts.
|
||||
|
||||
Graceful degradation:
|
||||
- If corpus is not found (e.g. due to Spanner restriction), logs a warning and exits cleanly.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import vertexai
|
||||
from vertexai import rag
|
||||
from vertexai.rag import RagCorpus
|
||||
|
||||
PROJECT_ID = "propane-will-491900-m5"
|
||||
LOCATION = os.environ.get("RAG_LOCATION", "us-central1")
|
||||
CORPUS_DISPLAY_NAME = os.environ.get("RAG_CORPUS_NAME", "osvauco-knowledge-base")
|
||||
GCS_SOURCE = os.environ.get(
|
||||
"RAG_GCS_SOURCE",
|
||||
f"gs://{PROJECT_ID}-agent-staging/rag-docs/",
|
||||
)
|
||||
|
||||
def get_token() -> str:
|
||||
return subprocess.check_output(
|
||||
["gcloud", "auth", "print-access-token"], text=True
|
||||
).strip()
|
||||
|
||||
def find_corpus() -> RagCorpus:
|
||||
"""Search for the corpus in the configured location and fallback location."""
|
||||
locations_to_try = [LOCATION]
|
||||
if LOCATION != "europe-west4":
|
||||
locations_to_try.append("europe-west4")
|
||||
|
||||
for loc in locations_to_try:
|
||||
print(f"Searching for corpus '{CORPUS_DISPLAY_NAME}' in {loc}...")
|
||||
vertexai.init(project=PROJECT_ID, location=loc)
|
||||
try:
|
||||
for c in rag.list_corpora():
|
||||
if c.display_name == CORPUS_DISPLAY_NAME:
|
||||
print(f" \u2713 Found: {c.name}")
|
||||
return c
|
||||
except Exception as e:
|
||||
print(f" \u26a0\ufe0f Notice: Could not list corpora in {loc}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def list_files(corpus: RagCorpus):
|
||||
"""List files currently in the corpus."""
|
||||
print(f"Listing files in corpus...")
|
||||
files = list(rag.list_files(corpus_name=corpus.name))
|
||||
if not files:
|
||||
print(" (Corpus is empty)")
|
||||
return []
|
||||
|
||||
for f in files:
|
||||
print(f" - {f.display_name} (state: {f.state.name})")
|
||||
return files
|
||||
|
||||
def sync_from_gcs(corpus: RagCorpus):
|
||||
"""Trigger incremental import from GCS."""
|
||||
print(f"Syncing from {GCS_SOURCE}...")
|
||||
# import_files in SDK is essentially an incremental trigger
|
||||
rag.import_files(
|
||||
corpus.name,
|
||||
paths=[GCS_SOURCE],
|
||||
transformation_config=rag.TransformationConfig(
|
||||
chunking_config=rag.ChunkingConfig(
|
||||
chunk_size=512,
|
||||
chunk_overlap=100,
|
||||
)
|
||||
),
|
||||
)
|
||||
print(" \u2713 Sync job submitted.")
|
||||
|
||||
def run_health_check(corpus: RagCorpus):
|
||||
"""Perform a retrieval query and report results."""
|
||||
print("Running health check retrieval...")
|
||||
try:
|
||||
response = rag.retrieval_query(
|
||||
rag_resources=[rag.RagResource(rag_corpus=corpus.name)],
|
||||
text="How does OSVauco handle operator experience?",
|
||||
rag_retrieval_config=rag.RagRetrievalConfig(top_k=3),
|
||||
)
|
||||
contexts = response.contexts.contexts
|
||||
print(f" \u2713 Health check returned {len(contexts)} contexts.")
|
||||
for i, ctx in enumerate(contexts):
|
||||
print(f" [{i+1}] Source: {ctx.source_uri} (Score: {ctx.score:.4f})")
|
||||
except Exception as e:
|
||||
print(f" \u2718 Health check failed: {e}")
|
||||
|
||||
def main():
|
||||
print(f"--- RAG Sync & Maintenance Tool ---")
|
||||
|
||||
corpus = find_corpus()
|
||||
if not corpus:
|
||||
print(f"\n\u26a0\ufe0f WARN: Corpus '{CORPUS_DISPLAY_NAME}' not found.")
|
||||
print(" RAG Engine might be restricted for this project or setup_corpus.py has not been run.")
|
||||
print(" Skipping sync and health check.")
|
||||
sys.exit(0)
|
||||
|
||||
# Update global LOCATION if we found it in a different one
|
||||
found_loc = corpus.name.split('/')[3]
|
||||
print(f"Active Location: {found_loc}")
|
||||
|
||||
list_files(corpus)
|
||||
|
||||
# Always trigger a sync to ensure new files are picked up
|
||||
sync_from_gcs(corpus)
|
||||
|
||||
print("\nWaiting 5s for sync initialization...")
|
||||
time.sleep(5)
|
||||
|
||||
run_health_check(corpus)
|
||||
|
||||
print(f"\nCOMPLETED")
|
||||
print(f"RAG_LOCATION={found_loc}")
|
||||
print(f"RAG_CORPUS={corpus.name}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
84
docs/RAG-SECONDARY-PROJECT-PLAN.md
Normal file
84
docs/RAG-SECONDARY-PROJECT-PLAN.md
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# Plan: Secondary RAG-Dedicated Project Migration
|
||||
|
||||
## 1. Objective
|
||||
To bypass the "Spanner mode" restriction currently blocking serverless RAG Engine corpus creation in `propane-will-491900-m5`. We will spin up a clean, dedicated project for RAG experimentation and wire OSVauco to it as a secondary data source.
|
||||
|
||||
## 2. New Project Setup
|
||||
|
||||
### Project Selection
|
||||
* **Suggested Name:** `vauco-rag-lab-01` or `osvauco-memory-node`.
|
||||
* **Organization:** Ensure it is created under the same billing account as the primary project.
|
||||
|
||||
### Step 1: Bootstrap Command
|
||||
```bash
|
||||
NEW_RAG_PROJECT="vauco-rag-lab-01"
|
||||
BILLING_ACCOUNT=$(gcloud billing projects describe propane-will-491900-m5 --format="value(billingAccountName)")
|
||||
|
||||
# Create project
|
||||
gcloud projects create ${NEW_RAG_PROJECT} --name="OSVauco RAG Lab"
|
||||
|
||||
# Link billing
|
||||
gcloud billing projects link ${NEW_RAG_PROJECT} --billing-account=${BILLING_ACCOUNT}
|
||||
```
|
||||
|
||||
### Step 2: Minimal APIs
|
||||
Enable only what is needed for RAG Engine and its data source:
|
||||
```bash
|
||||
gcloud services enable \
|
||||
aiplatform.googleapis.com \
|
||||
storage.googleapis.com \
|
||||
--project=${NEW_RAG_PROJECT}
|
||||
```
|
||||
|
||||
### Step 3: Cross-Project IAM
|
||||
Grant the primary OSVauco Agent (from `propane-will-491900-m5`) access to the new RAG project:
|
||||
```bash
|
||||
PRIMARY_SA="jason.vauger@propane-will-491900-m5.iam.gserviceaccount.com"
|
||||
|
||||
# Grant Vertex AI User and Storage Admin on the NEW project
|
||||
for ROLE in roles/aiplatform.user roles/storage.objectAdmin; do
|
||||
gcloud projects add-iam-policy-binding ${NEW_RAG_PROJECT} \
|
||||
--member="serviceAccount:${PRIMARY_SA}" \
|
||||
--role="${ROLE}"
|
||||
done
|
||||
```
|
||||
|
||||
## 3. RAG Setup & Sync Refactoring
|
||||
|
||||
We will use variants of our existing scripts, but modified to accept a `--project` override.
|
||||
|
||||
### `setup_corpus_remote.py`
|
||||
A variant of `setup_corpus.py` that:
|
||||
1. Targets `NEW_RAG_PROJECT`.
|
||||
2. Attempts the same "Serverless" PATCH in `us-central1` or `europe-west4`.
|
||||
3. If it succeeds, it saves the resulting corpus name to a shared secret in the **Primary** project.
|
||||
|
||||
### `sync_corpus_remote.py`
|
||||
A variant of `sync_corpus.py` that reads from the Primary project's GCS bucket but writes/indexes to the Remote project's RAG Engine.
|
||||
|
||||
## 4. Dual-Project Configuration Strategy
|
||||
|
||||
OSVauco will be updated to handle "Project Context Switching."
|
||||
|
||||
### Environment Variables
|
||||
The following overrides will be added to `.env`:
|
||||
* `PROJECT_ID`: `propane-will-491900-m5` (Primary logic/infra)
|
||||
* **`RAG_PROJECT_ID`**: `vauco-rag-lab-01` (Knowledge base source)
|
||||
* **`RAG_LOCATION`**: `europe-west4` (Region for the remote corpus)
|
||||
|
||||
### Code Updates
|
||||
1. **`agents/rag/*`**: All RAG scripts and agent tools will default to `RAG_PROJECT_ID` (falling back to `PROJECT_ID`) during `vertexai.init()`.
|
||||
2. **`scripts/preflight.sh`**: Will check both projects.
|
||||
* Infrastructure section -> Primary Project.
|
||||
* Knowledge Base section -> Remote Project.
|
||||
3. **`scripts/cost-lens.sh`**: Will aggregate costs from both projects if `HEAVY_MODE` is enabled.
|
||||
|
||||
## 5. Migration Timeline
|
||||
|
||||
1. **Project Creation**: Create `vauco-rag-lab-01` and link billing.
|
||||
2. **IAM Wiring**: Establish the cross-project service account trust.
|
||||
3. **Test Run**: Run `setup_corpus.py` targeting the new project to verify if "Spanner mode" restriction is absent.
|
||||
4. **Integration**: Update `.env` and `preflight.sh` to recognize the secondary project.
|
||||
|
||||
---
|
||||
*Senior Staff Engineer (Gemini CLI)*
|
||||
58
infrastructure/99-region-guard.sh
Normal file
58
infrastructure/99-region-guard.sh
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env bash
|
||||
# infrastructure/99-region-guard.sh — Regional availability intelligence
|
||||
# Sourced by other infrastructure scripts to ensure regional compliance.
|
||||
|
||||
# --- 1. Known Regional Capabilities (v1beta1) ---
|
||||
|
||||
# Vertex AI RAG Engine (Serverless) availability
|
||||
# Source: https://cloud.google.com/vertex-ai/docs/generative-ai/rag-overview
|
||||
RAG_SUPPORTED_REGIONS=("us-central1" "us-east1" "us-east4" "europe-west1" "europe-west4" "europe-west9" "asia-northeast1")
|
||||
|
||||
# Vertex AI Agent Engine (Memory Bank / Reasoning Engine) availability
|
||||
# Most Vertex AI regions support this, but we track primary ones.
|
||||
AGENT_ENGINE_SUPPORTED_REGIONS=("us-central1" "europe-west4")
|
||||
|
||||
# Cloud Run
|
||||
# Virtually all regions support Cloud Run, but we prefer those with low latency or specific compliance.
|
||||
# Default for OSVauco is us-central1 (Management Plane).
|
||||
# Default for clinical data is europe-north1.
|
||||
|
||||
# --- 2. Helper Functions ---
|
||||
|
||||
# Check if a value is in an array
|
||||
contains_element() {
|
||||
local e match="$1"
|
||||
shift
|
||||
for e; do [[ "$e" == "$match" ]] && return 0; done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Validate RAG region
|
||||
validate_rag_region() {
|
||||
local TARGET_REGION="${1:-us-central1}"
|
||||
if contains_element "$TARGET_REGION" "${RAG_SUPPORTED_REGIONS[@]}"; then
|
||||
echo "$TARGET_REGION"
|
||||
else
|
||||
# Fallback to a safe region known to support serverless RAG
|
||||
echo "europe-west4"
|
||||
fi
|
||||
}
|
||||
|
||||
# Validate Agent Engine region
|
||||
validate_agent_engine_region() {
|
||||
local TARGET_REGION="${1:-us-central1}"
|
||||
if contains_element "$TARGET_REGION" "${AGENT_ENGINE_SUPPORTED_REGIONS[@]}"; then
|
||||
echo "$TARGET_REGION"
|
||||
else
|
||||
# Fallback
|
||||
echo "us-central1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Log regional context
|
||||
log_region_context() {
|
||||
local SVC=$1
|
||||
local REGION=$2
|
||||
local STATUS=$3
|
||||
echo -e " [REGION-GUARD] Service: ${SVC} | Region: ${REGION} | Status: ${STATUS}"
|
||||
}
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
# Engineering Brief: RAG Robustness & Operator Experience
|
||||
**Date:** 2026-05-24
|
||||
**Author:** Senior Staff Engineer (Gemini CLI)
|
||||
**Status:** PROPOSED
|
||||
|
||||
## 1. Vision & Roadmap Alignment
|
||||
|
||||
OSVauco (OPAX) is evolving from a collection of scripts into a cohesive **AI Operating System**. The Roadmap (Phase 4-6) and Masterplan emphasize a transition from "getting it up" to "keeping it smooth, secure, and multi-client ready."
|
||||
|
||||
The core of this experience is the **Operator Cockpit**—where Gemini, agents (Jason Vauger), and the TUI bridge converge to manage a complex GCP estate.
|
||||
|
||||
### The Role of RAG
|
||||
RAG is the "long-term memory" of OPAX. It shouldn't just be a one-time setup; it must be a living repository of the operator's intent, session logs, and architectural rules.
|
||||
|
||||
## 2. Current RAG State (Baseline)
|
||||
|
||||
* **What Works:**
|
||||
* Serverless corpus creation via REST (bypassing SDK bugs).
|
||||
* Automatic regional fallback (`us-central1` -> `europe-west4`) to avoid Spanner restrictions.
|
||||
* Initial document import from GCS.
|
||||
* **What's Fragile:**
|
||||
* **Synchronization:** No automated way to re-sync if new files are added to GCS.
|
||||
* **Health Checks:** No quick way to verify if the corpus is "alive" or if the embedding model is responding.
|
||||
* **Discovery:** The corpus name is saved to Secret Manager but not easily discoverable by other agents without manual environment variable plumbing.
|
||||
|
||||
## 3. Proposed "Heavy Lifts"
|
||||
|
||||
### Lift 1: Automated RAG Sync & Maintenance Tool
|
||||
**Goal:** Create `agents/rag/sync_corpus.py` (and a corresponding agent tool) that performs incremental imports and health checks.
|
||||
|
||||
### Lift 2: Operator "Preflight" Dashboard (CLI)
|
||||
**Goal:** A new script `scripts/preflight.sh` that provides a high-signal summary for the operator at the start of a session.
|
||||
|
||||
### Lift 3: Jason Vauger "Cost-Lens" Tooling
|
||||
**Goal:** Enhance the core agent logic to support structured cost queries.
|
||||
|
||||
### Lift 4: Multi-Region Infrastructure Guard
|
||||
**Goal:** Refactor `infrastructure/*.sh` to be fully region-agnostic.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Log
|
||||
|
||||
### Heavy Lift 1: Automated RAG Sync & Maintenance Tool
|
||||
**Status:** COMPLETED
|
||||
**Date:** 2026-05-24
|
||||
|
||||
* **Changes:**
|
||||
* Created `agents/rag/sync_corpus.py`.
|
||||
* Implemented multi-location discovery logic (`us-central1` and `europe-west4`).
|
||||
* Added file listing functionality to report on corpus content.
|
||||
* Integrated incremental sync from GCS via `rag.import_files`.
|
||||
* Added a health-check retrieval query to verify the end-to-end RAG pipeline.
|
||||
* **Why:** To ensure the RAG corpus remains a "living" repository that stays in sync with GCS-based documentation and allows operators to verify system health.
|
||||
* **How to run/test:**
|
||||
```bash
|
||||
# 1. Ensure corpus exists (or create it)
|
||||
python3 agents/rag/setup_corpus.py
|
||||
|
||||
# 2. Sync and check health
|
||||
python3 agents/rag/sync_corpus.py
|
||||
```
|
||||
|
||||
### Heavy Lift 2: Operator Preflight Dashboard
|
||||
**Status:** COMPLETED
|
||||
**Date:** 2026-05-24
|
||||
|
||||
* **Changes:**
|
||||
* Created `scripts/preflight.sh`.
|
||||
* Implemented Cloud Run service health check.
|
||||
* Added RAG corpus discovery with automatic SDK fallback for environments missing `gcloud ai rag` commands.
|
||||
* Integrated an "Idle Cost Risk" audit (VMs, SQL, Memory Bank).
|
||||
* Added Secret Manager verification for critical tokens.
|
||||
* **Why:** To give the operator immediate situational awareness and prevent "billing surprises" or broken deployments before starting a session.
|
||||
* **How to run/test:**
|
||||
```bash
|
||||
bash scripts/preflight.sh
|
||||
```
|
||||
|
||||
### Heavy Lift 3: Jason Vauger "Cost-Lens" Tooling
|
||||
**Status:** COMPLETED
|
||||
**Date:** 2026-05-24
|
||||
|
||||
* **Changes:**
|
||||
* Created `scripts/cost-lens.sh`.
|
||||
* Implemented "Idle Cost Risk" detection for VMs, SQL, Reasoning Engines, and Cloud Run min-instances.
|
||||
* Added `get_price_hint` helper for rough monthly cost projections.
|
||||
* Added 7-day cost trend visualization (with instructions for BigQuery billing export).
|
||||
* Implemented `--heavy` mode for structured YAML/JSON output designed for LLM agent ingestion.
|
||||
* **Why:** To empower the operator and the OPAX agent (Jason Vauger) with structured, actionable cost intelligence.
|
||||
* **How to run/test:**
|
||||
```bash
|
||||
# Human-readable report
|
||||
bash scripts/cost-lens.sh
|
||||
|
||||
# Machine-readable "Heavy Mode"
|
||||
bash scripts/cost-lens.sh --heavy
|
||||
```
|
||||
|
||||
### Heavy Lift 4: Multi-Region Infrastructure Guard
|
||||
**Status:** COMPLETED
|
||||
**Date:** 2026-05-24
|
||||
|
||||
* **Changes:**
|
||||
* Created `infrastructure/99-region-guard.sh` to centralize regional availability logic.
|
||||
* Updated `infrastructure/07-rag-setup.sh` and `infrastructure/08-memorybank-setup.sh` to use the guard for automatic fallback to supported regions.
|
||||
* Refactored `infrastructure/05-cloudrun-deploy.sh` to be fully region-agnostic, supporting `DEPLOY_REGION` overrides for cross-region deployments (e.g., clinical data in `europe-north1`).
|
||||
* Updated `infrastructure/01-setupenv.sh` to integrate regional logging and validation.
|
||||
* **Why:** To ensure the OPAX infrastructure can scale across multiple regions and clients while respecting GCP service availability and compliance requirements (e.g., data residency).
|
||||
* **How to run/test:**
|
||||
```bash
|
||||
# Normal setup (respects .env REGION)
|
||||
bash infrastructure/01-setupenv.sh
|
||||
|
||||
# Deployment to a specific region
|
||||
DEPLOY_REGION=europe-north1 bash infrastructure/05-cloudrun-deploy.sh
|
||||
```
|
||||
* **Open Follow-ups:**
|
||||
* Expand the guard to include VPC Service Controls and other compliance-heavy services.
|
||||
* Add automated "region discovery" to suggest the best region based on latency from the operator's current location.
|
||||
|
||||
## Docs Hardening – 2026-05-24
|
||||
**Status:** COMPLETED
|
||||
|
||||
* **Updated Docs:**
|
||||
* `docs/LEARNINGS.md`: Added **LEARNING-007** regarding Vertex AI RAG Engine Spanner restrictions and our "graceful degradation" strategy.
|
||||
* `docs/SECRETS-SETUP.md`: Refreshed list of essential secrets (`github-token`, `webhook-url`, `rag-corpus-name`) with updated `gcloud` commands and service account mappings.
|
||||
* `docs/IAP-SETUP.md`: Clarified current status—OPAX is on Cloud Run, with IAP planned for Phase 5.
|
||||
* **Key Decisions:**
|
||||
* **RAG Unavailable:** Formally recognized that RAG Engine is restricted to Spanner mode for this project. The platform now skips RAG operations cleanly rather than failing.
|
||||
* **Operator Focus:** Shifted documentation priority towards new tools (`preflight.sh`, `cost-lens.sh`) to empower the operator despite the RAG limitation.
|
||||
* **Open Follow-ups:**
|
||||
* Update `MASTERPLAN.md` quick-start section to include `bash scripts/preflight.sh` as the first step.
|
||||
* Document the `infrastructure/99-region-guard.sh` logic in `ARCHITECTURE.md`.
|
||||
|
||||
---
|
||||
*Senior Staff Engineer (Gemini CLI)*
|
||||
Loading…
Reference in New Issue
Block a user