chore: commit modified infra, RAG and session log files

agents/rag/setup_corpus.py

@@ -3,27 +3,20 @@
 setup_corpus.py — Create a Vertex AI RAG Engine corpus and import documents.
 Project: propane-will-491900-m5
 
-SDK 1.153.1 has a bug where backend_config crashes when provided.
+SDK 1.153.1 has bugs in backend_config and RagManagedDbConfig.
-We call the REST API directly for corpus creation.
+We use the REST API (v1beta1) for corpus creation and engine configuration.
 
-v1beta1 REST schema facts (verified from Google docs):
+Changes implemented:
-  RagCorpus.backend_config is a union field:
+- Graceful degradation: If RAG Engine is restricted (Spanner mode), skip and exit cleanly.
-    vectorDbConfig: RagVectorDbConfig
+- ensure_serverless_engine_config() sets RAG_ENGINE_AVAILABLE flag.
-      .ragManagedDb: RagManagedDb  <- serverless managed DB, no tier field
+- get_or_create_corpus() returns None if RAG is unavailable.
-      .vertexVectorSearch, .pinecone, .weaviate  <- alternatives
-  RagVectorDbConfig has NO 'ragManagedDbConfig' field.
-  RagManagedDb has NO 'tier' field — empty {} means serverless/basic.
-
-  ragEngineConfig (project-level) controls the DEFAULT backend when
-  vectorDbConfig is omitted. We must verify it is actually in 'basic'
-  state (not 'spanner') before creating a corpus without an explicit
-  vectorDbConfig, OR we can pass vectorDbConfig.ragManagedDb explicitly.
 """
 
 import os
 import json
 import time
 import subprocess
+import sys
 
 import vertexai
 from vertexai import rag
@@ -37,6 +30,9 @@ GCS_SOURCE = os.environ.get(
     f"gs://{PROJECT_ID}-agent-staging/rag-docs/",
 )
 
+# Global status flag
+RAG_ENGINE_AVAILABLE = True
+
 # ragEngineConfig is a project-level control-plane endpoint.
 # It only exists in us-central1 regardless of where the corpus lives.
 _ENGINE_CONFIG_LOCATION = "us-central1"
@@ -61,37 +57,61 @@ def get_engine_config() -> dict:
         ["curl", "-s", "-H", f"Authorization: Bearer {token}", _engine_config_url()],
         capture_output=True, text=True,
     )
+    try:
         return json.loads(r.stdout)
+    except Exception:
+        print(f"Failed to parse engine config: {r.stdout}")
+        return {}
 
 
 def ensure_serverless_engine_config() -> None:
     """
     Set project-level RAG Engine Config to basic (serverless) tier.
-    Always targets us-central1.
+    Only applicable for us-central1, us-east1, us-east4.
-
-    The PATCH is a merge by default. We verify the result with a GET
-    to confirm 'spanner' is no longer the active mode before proceeding.
     """
-    print("Ensuring RAG Engine Config is set to serverless (basic) tier...")
+    global RAG_ENGINE_AVAILABLE
+    
+    restricted_regions = {"us-central1", "us-east1", "us-east4"}
+    if LOCATION not in restricted_regions:
+        print(f"Skipping engine config check for location: {LOCATION} (not restricted)")
+        return
+
+    print(f"Ensuring RAG Engine Config in {_ENGINE_CONFIG_LOCATION} is set to serverless (basic) tier...")
     token = get_token()
 
+    # Use updateMask=ragManagedDbConfig to target the entire configuration block.
+    # Set spanner: null to explicitly clear it.
+    payload = {
+        "ragManagedDbConfig": {
+            "basic": {},
+            "spanner": None
+        }
+    }
+    
+    url = f"{_engine_config_url()}"
+    
     result = subprocess.run(
         ["curl", "-s", "-X", "PATCH",
          "-H", f"Authorization: Bearer {token}",
          "-H", "Content-Type: application/json",
-         _engine_config_url(),
+         url,
-         "-d", json.dumps({"ragManagedDbConfig": {"basic": {}}})],
+         "-d", json.dumps(payload)],
         capture_output=True, text=True,
     )
+    
+    try:
         resp = json.loads(result.stdout)
+    except Exception:
+        print(f"  \u26a0\ufe0f  Failed to parse PATCH response: {result.stdout}")
+        return
 
     if "error" in resp:
         err = resp["error"]
-        print(f"  \u26a0\ufe0f  Engine config PATCH warning ({err.get('code')}): {err.get('message')}")
+        print(f"  \u26a0\ufe0f  Engine config PATCH error ({err.get('code')}): {err.get('message')}")
 
     op_name = resp.get("name", "")
     if "/operations/" in op_name and not resp.get("done"):
-        print(f"  Polling operation...")
+        print(f"  Polling engine config operation...")
         op_url = (
             f"https://{_ENGINE_CONFIG_LOCATION}-aiplatform.googleapis.com/v1beta1/{op_name}"
         )
@@ -106,37 +126,32 @@ def ensure_serverless_engine_config() -> None:
             if op.get("done"):
                 break
         else:
-            print("  \u26a0\ufe0f  Operation timed out.")
+            print("  \u26a0\ufe0f  Engine config operation timed out.")
 
     # Verify actual state
     cfg = get_engine_config()
     db_cfg = cfg.get("ragManagedDbConfig", {})
     print(f"  Current ragManagedDbConfig: {json.dumps(db_cfg)}")
-    if "spanner" in db_cfg and "basic" not in db_cfg:
+    
-        raise RuntimeError(
+    if "spanner" in db_cfg:
-            "Engine config is still in Spanner mode. "
+        print(f"\n\u26a0\ufe0f  WARN: Engine config in {_ENGINE_CONFIG_LOCATION} is still in Spanner mode.")
-            "Cannot create a serverless corpus. "
+        print("    RAG Engine appears to be restricted for this project.")
-            "Run: curl -s -X PATCH -H 'Authorization: Bearer $(gcloud auth print-access-token)' "
+        RAG_ENGINE_AVAILABLE = False
-            f"-H 'Content-Type: application/json' {_engine_config_url()} "
+        return
-            "-d '{\"ragManagedDbConfig\":{\"basic\":{}}}'"
+            
-        )
     print("  \u2713 RAG Engine Config verified as basic tier.")
 
 
-def create_corpus_rest() -> str:
+def create_corpus_rest(loc: str) -> str:
     """
-    Create corpus via REST API directly, bypassing SDK backend_config bug.
+    Create corpus via REST API directly.
-    Returns the corpus resource name.
-
-    Per v1beta1 docs, RagCorpus.backend_config is a union:
-      vectorDbConfig.ragManagedDb: {} -> serverless managed DB (no tier field)
-    This explicitly requests serverless at corpus level regardless of
-    the project-level engine config default.
     """
     url = (
-        f"https://{LOCATION}-aiplatform.googleapis.com/v1beta1"
+        f"https://{loc}-aiplatform.googleapis.com/v1beta1"
-        f"/projects/{PROJECT_ID}/locations/{LOCATION}/ragCorpora"
+        f"/projects/{PROJECT_ID}/locations/{loc}/ragCorpora"
     )
+    # vectorDbConfig.ragManagedDb = serverless RAG Managed DB.
+    # This is the correct v1beta1 field name.
     payload = json.dumps({
         "displayName": CORPUS_DISPLAY_NAME,
         "ragEmbeddingModelConfig": {
@@ -144,14 +159,12 @@ def create_corpus_rest() -> str:
                 "model": "publishers/google/models/text-embedding-004"
             }
         },
-        # vectorDbConfig.ragManagedDb = serverless RAG Managed DB.
-        # RagManagedDb has no fields (no tier, no tier enum).
-        # This is the correct v1beta1 field name — not 'ragManagedDbConfig'.
         "vectorDbConfig": {
             "ragManagedDb": {}
         }
     })
     token = get_token()
+    print(f"  Sending POST to {url}...")
     result = subprocess.run(
         ["curl", "-s", "-X", "POST",
          "-H", f"Authorization: Bearer {token}",
@@ -159,17 +172,22 @@ def create_corpus_rest() -> str:
          url, "-d", payload],
         capture_output=True, text=True,
     )
-    print(f"  API response: {result.stdout[:300]}")
+    
+    try:
         resp = json.loads(result.stdout)
+    except Exception:
+        raise RuntimeError(f"Failed to parse API response: {result.stdout}")
+
     if "error" in resp:
-        raise RuntimeError(f"Failed to create corpus: {resp['error']}")
+        # Pass the error object up so get_or_create_corpus can inspect it
+        raise RuntimeError(json.dumps(resp["error"]))
 
     op_name = resp.get("name", "")
     if "/operations/" not in op_name:
         raise RuntimeError(f"Unexpected response (no operation): {resp}")
 
-    print(f"  Polling corpus creation operation...")
+    print(f"  Polling corpus creation operation {op_name}...")
-    op_url = f"https://{LOCATION}-aiplatform.googleapis.com/v1beta1/{op_name}"
+    op_url = f"https://{loc}-aiplatform.googleapis.com/v1beta1/{op_name}"
     for _ in range(40):
         time.sleep(5)
         token = get_token()
@@ -180,7 +198,7 @@ def create_corpus_rest() -> str:
         op = json.loads(r.stdout)
         if op.get("done"):
             if "error" in op:
-                raise RuntimeError(f"Corpus creation failed: {op['error']}")
+                raise RuntimeError(json.dumps(op["error"]))
             corpus_name = op["response"]["name"]
             return corpus_name
 
@@ -188,16 +206,66 @@ def create_corpus_rest() -> str:
 
 
 def get_or_create_corpus() -> RagCorpus:
-    """Return existing corpus by display name, or create a new serverless one."""
+    """Return existing corpus by display name, or create a new serverless one with fallback."""
+    global LOCATION, RAG_ENGINE_AVAILABLE
+    
+    if not RAG_ENGINE_AVAILABLE:
+        print("\n\u2139  RAG Engine appears to be in restricted Spanner mode for this project. Skipping corpus creation.")
+        return None
+
+    def _find_in_list():
+        try:
             for c in rag.list_corpora():
                 if c.display_name == CORPUS_DISPLAY_NAME:
-            print(f"Corpus '{CORPUS_DISPLAY_NAME}' already exists: {c.name}")
                     return c
+        except Exception:
+            pass
+        return None
 
+    # 1. Try finding in current LOCATION
+    vertexai.init(project=PROJECT_ID, location=LOCATION)
+    corpus = _find_in_list()
+    if corpus:
+        print(f"Corpus '{CORPUS_DISPLAY_NAME}' already exists in {LOCATION}: {corpus.name}")
+        return corpus
+
+    # 2. Try creating in current LOCATION
     print(f"Creating corpus '{CORPUS_DISPLAY_NAME}' in {LOCATION} via REST...")
-    corpus_name = create_corpus_rest()
+    try:
-    print(f"\u2713 Corpus created: {corpus_name}")
+        corpus_name = create_corpus_rest(LOCATION)
+        print(f"\u2713 Corpus created in {LOCATION}: {corpus_name}")
+    except RuntimeError as e:
+        err_str = str(e)
+        # Check for Spanner restriction error message
+        if "using Spanner mode with RAG Engine in us-central1, us-east1, and us-east4 is restricted" in err_str:
+            if LOCATION != "europe-west4":
+                print(f"\u26a0\ufe0f Spanner restriction detected in {LOCATION}. Retrying in europe-west4...")
+                LOCATION = "europe-west4"
+                vertexai.init(project=PROJECT_ID, location=LOCATION)
                 
+                # Check if it already exists in the fallback location
+                corpus = _find_in_list()
+                if corpus:
+                    print(f"Corpus '{CORPUS_DISPLAY_NAME}' found in fallback {LOCATION}: {corpus.name}")
+                    return corpus
+                    
+                try:
+                    corpus_name = create_corpus_rest(LOCATION)
+                    print(f"\u2713 Corpus created in fallback {LOCATION}: {corpus_name}")
+                except RuntimeError as e2:
+                    if "using Spanner mode" in str(e2):
+                        print("\n\u26a0\ufe0f  RAG Engine restriction confirmed in fallback region. RAG is effectively unavailable.")
+                        RAG_ENGINE_AVAILABLE = False
+                        return None
+                    raise
+            else:
+                print("\n\u26a0\ufe0f  RAG Engine restriction confirmed in fallback region. RAG is effectively unavailable.")
+                RAG_ENGINE_AVAILABLE = False
+                return None
+        else:
+            raise RuntimeError(f"Failed to create corpus in {LOCATION}: {err_str}")
+
+    # Final retrieval of the corpus object
     for c in rag.list_corpora():
         if c.name == corpus_name:
             return c
@@ -206,7 +274,7 @@ def get_or_create_corpus() -> RagCorpus:
 
 
 def import_documents(corpus: RagCorpus) -> None:
-    print(f"Importing files from {GCS_SOURCE}...")
+    print(f"Importing files from {GCS_SOURCE} into {corpus.name}...")
     rag.import_files(
         corpus.name,
         paths=[GCS_SOURCE],
@@ -217,32 +285,48 @@ def import_documents(corpus: RagCorpus) -> None:
             )
         ),
     )
-    print("\u2713 Import complete.")
+    print("\u2713 Import job submitted.")
 
 
 def test_retrieval(corpus: RagCorpus) -> None:
     print("Running test retrieval query...")
+    try:
         response = rag.retrieval_query(
             rag_resources=[rag.RagResource(rag_corpus=corpus.name)],
             text="test query",
             rag_retrieval_config=rag.RagRetrievalConfig(top_k=3),
         )
         print(f"\u2713 Test retrieval returned {len(response.contexts.contexts)} chunk(s).")
+    except Exception as e:
+        print(f"  \u26a0\ufe0f  Retrieval test failed (import might still be processing): {e}")
 
 
 def main() -> None:
-    vertexai.init(project=PROJECT_ID, location=LOCATION)
+    print(f"--- RAG Setup Starting (Target Location: {LOCATION}) ---")
     
     ensure_serverless_engine_config()
 
     corpus = get_or_create_corpus()
 
+    if corpus is None:
+        print("\n--- RAG AVAILABILITY SUMMARY ---")
+        print(f"Project   : {PROJECT_ID}")
+        print("Status    : UNAVAILABLE (Platform restriction: Spanner Mode only)")
+        print("Action    : Skipping RAG integration. Downstream tools will operate without a corpus.")
+        print("---------------------------------")
+        sys.exit(0)
+
     import_documents(corpus)
 
-    print(f"\nRAG_CORPUS={corpus.name}")
+    print(f"\nSUCCESS")
-    print("Add this to Secret Manager:")
+    print(f"RAG_LOCATION={LOCATION}")
-    print(f"  gcloud secrets create rag-corpus-name --data-file=- <<<'{corpus.name}'")
+    print(f"RAG_CORPUS={corpus.name}")
+    print("\nNext steps (set in your environment):")
+    print(f"export RAG_LOCATION={LOCATION}")
+    print(f"export RAG_CORPUS_NAME={corpus.name}")
     
+    print("\nWaiting 10s before test retrieval...")
+    time.sleep(10)
     test_retrieval(corpus)
 
 

docs/OSVAUCO_OPAX_SESSION_LOG.md

@@ -98,3 +98,7 @@ PHASE 5 — Cloud Run IAM-autentisering på `osvauco-agent`.
 
 ## NESTE OPPGAVE
 PHASE 6 — Roter OAuth client secret for `Vauco OS Web App` i [Google Auth Platform](https://console.cloud.google.com/auth/clients?project=propane-will-491900-m5). Deretter: vurder om `jason.vauger@vauco.no` skal beholde `roles/run.invoker` på `osvauco-agent`, eller om tilgang skal innsnevres til kun `@vauco.no`-kontoer via IAP load balancer på sikt.
+  12:44:47 ✔  git pull origin main
+  12:47:24 ✔  git push origin main
+  12:48:32 ✔  git push origin main
+  12:49:53 ✔  git push origin main

infrastructure/01-setupenv.sh

@@ -10,6 +10,10 @@ set -euo pipefail
 : "${BILLING_ACCOUNT_ID:?Set BILLING_ACCOUNT_ID in .env}"
 : "${AGENT_SA:?Set AGENT_SA in .env}"
 
+# ── Region Guard ────────────────────────────────────────────────────────
+source "$(dirname "$0")/99-region-guard.sh"
+log_region_context "Storage-Bucket" "$REGION" "SELECTED"
+
 BUCKET_NAME="gs://${PROJECT_ID}-agent-staging"
 BUDGET_PROD_NAME="OSVauco-Agent-Budget-500USD"
 BUDGET_DEV_NAME="OSVauco-Dev-Budget-75USD"

infrastructure/05-cloudrun-deploy.sh

@@ -16,12 +16,17 @@ fi
 : "${CLOUD_RUN_SERVICE:?Set CLOUD_RUN_SERVICE in .env}"
 : "${AGENT_SA:?Set AGENT_SA in .env}"
 
-REPO="${REGION}-docker.pkg.dev/${PROJECT_ID}/osvauco-repo"
+# ── Region Guard ────────────────────────────────────────────────────────
+source "$(dirname "$0")/99-region-guard.sh"
+DEPLOY_REGION="${DEPLOY_REGION:-${REGION}}"
+log_region_context "Cloud-Run" "$DEPLOY_REGION" "SELECTED"
+
+REPO="${DEPLOY_REGION}-docker.pkg.dev/${PROJECT_ID}/osvauco-repo"
 IMAGE="${REPO}/osvauco-agent:latest"
 AGENT_PATH="${SCRIPT_DIR}/../agents/core-logic"
 AGENT_SA_NAME=$(echo "${AGENT_SA}" | cut -d@ -f1)
 
-echo "=== 05: Cloud Run Deploy ==="
+echo "=== 05: Cloud Run Deploy to ${DEPLOY_REGION} ==="
 
 bash "${SCRIPT_DIR}/00-authcheck.sh"
 
@@ -67,10 +72,10 @@ echo "✓ iam.serviceAccountUser granted to ${CALLER}"
 
 # Create Artifact Registry repo if it doesn't exist
 gcloud artifacts repositories describe osvauco-repo \
-  --location="${REGION}" --project="${PROJECT_ID}" &>/dev/null || \
+  --location="${DEPLOY_REGION}" --project="${PROJECT_ID}" &>/dev/null || \
 gcloud artifacts repositories create osvauco-repo \
   --repository-format=docker \
-  --location="${REGION}" \
+  --location="${DEPLOY_REGION}" \
   --project="${PROJECT_ID}" --quiet
 echo "✓ Artifact Registry repo ready"
 
@@ -86,24 +91,24 @@ gcloud projects add-iam-policy-binding "${PROJECT_ID}" \
 echo "✓ Cloud Build IAM bindings applied"
 
 # Configure Docker for Artifact Registry
-gcloud auth configure-docker "${REGION}-docker.pkg.dev" --quiet
+gcloud auth configure-docker "${DEPLOY_REGION}-docker.pkg.dev" --quiet
 
 # Build image via Cloud Build
 echo "Building image via Cloud Build..."
 gcloud builds submit "${AGENT_PATH}" \
   --tag="${IMAGE}" \
   --project="${PROJECT_ID}" \
-  --region="${REGION}"
+  --region="${DEPLOY_REGION}"
 echo "✓ Image built: ${IMAGE}"
 
 # Deploy to Cloud Run
 echo "Deploying to Cloud Run..."
 gcloud run deploy "${CLOUD_RUN_SERVICE}" \
   --image="${IMAGE}" \
-  --region="${REGION}" \
+  --region="${DEPLOY_REGION}" \
   --project="${PROJECT_ID}" \
   --service-account="${AGENT_SA}" \
-  --set-env-vars="GOOGLE_CLOUD_PROJECT=${PROJECT_ID},GOOGLE_CLOUD_LOCATION=${REGION},GOOGLE_GENAI_USE_VERTEXAI=True" \
+  --set-env-vars="GOOGLE_CLOUD_PROJECT=${PROJECT_ID},GOOGLE_CLOUD_LOCATION=${DEPLOY_REGION},GOOGLE_GENAI_USE_VERTEXAI=True" \
   --no-allow-unauthenticated \
   --port=8080 \
   --memory=1Gi \
@@ -115,7 +120,7 @@ gcloud run deploy "${CLOUD_RUN_SERVICE}" \
 echo ""
 echo "=== 05: Cloud Run Deploy COMPLETE ==="
 SERVICE_URL=$(gcloud run services describe "${CLOUD_RUN_SERVICE}" \
-  --region="${REGION}" --project="${PROJECT_ID}" \
+  --region="${DEPLOY_REGION}" --project="${PROJECT_ID}" \
   --format="value(status.url)" 2>/dev/null || echo "(pending)")
 echo "  Service URL: ${SERVICE_URL}"
 echo ""
@@ -125,4 +130,4 @@ echo "    curl -H \"Authorization: Bearer \$TOKEN\" -H 'Content-Type: applicatio
 echo "         -d '{\"message\": \"Hello\"}' \${SERVICE_URL}/run"
 echo ""
 echo "  COST NOTE: Cloud Run scales to 0. No idle cost."
-echo "  Delete with: gcloud run services delete ${CLOUD_RUN_SERVICE} --region=${REGION} --quiet"
+echo "  Delete with: gcloud run services delete ${CLOUD_RUN_SERVICE} --region=${DEPLOY_REGION} --quiet"

infrastructure/07-rag-setup.sh

@@ -16,7 +16,11 @@ set -euo pipefail
 : "${REGION:?Set REGION}"
 : "${RAG_CORPUS_DISPLAY_NAME:?Set RAG_CORPUS_DISPLAY_NAME}"
 
-RAG_REGION="${RAG_REGION:-europe-west4}"
+# ── Region Guard ────────────────────────────────────────────────────────
+source "$(dirname "$0")/99-region-guard.sh"
+RAG_REGION=$(validate_rag_region "${RAG_REGION:-${REGION}}")
+log_region_context "RAG-Engine" "$RAG_REGION" "VALIDATED"
+
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
 
 echo "=== 07: Setting up Vertex AI RAG Engine (Serverless mode) ==="

infrastructure/08-memorybank-setup.sh

@@ -11,8 +11,11 @@ set -euo pipefail
 : "${AGENT_SA:?Set AGENT_SA}"
 
 MEMORY_INSTANCE_DISPLAY_NAME="${MEMORY_INSTANCE_DISPLAY_NAME:-${PROJECT_ID}-memory-bank}"
-# Agent Engine lives in the same region as REGION (us-central1 is fine, eu also works)
+
-MEMORY_REGION="${MEMORY_REGION:-${REGION}}"
+# ── Region Guard ────────────────────────────────────────────────────────
+source "$(dirname "$0")/99-region-guard.sh"
+MEMORY_REGION=$(validate_agent_engine_region "${MEMORY_REGION:-${REGION}}")
+log_region_context "Agent-Engine" "$MEMORY_REGION" "VALIDATED"
 
 echo "=== 08: Setting up Vertex AI Memory Bank (Agent Engine) ==="
 echo "    Project  : ${PROJECT_ID}"