feat: 07-rag-setup.sh — upload alle docs/*.md + corpus-seed/*.md til GCS automatisk

infrastructure/07-rag-setup.sh

@@ -2,6 +2,7 @@
 # 07-rag-setup.sh — Create Vertex AI RAG Engine corpus in Serverless mode
 # Uses REST API directly — no SDK version dependency
 # RAG_REGION defaults to europe-west4 (Serverless available, no allowlist)
+# Uploads ALL docs/*.md + docs/corpus-seed/*.md to GCS on every run
 # Idempotent — safe to run multiple times
 # Source .env before running: source .env
 
@@ -14,6 +15,8 @@ set -euo pipefail
 # europe-west4: Serverless RAG available, no allowlist, gemini-2.0-flash + 2.5-pro present
 RAG_REGION="${RAG_REGION:-europe-west4}"
 
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+
 echo "=== 07: Setting up Vertex AI RAG Engine (Serverless mode) ==="
 echo "    Project  : ${PROJECT_ID}"
 echo "    Region   : ${RAG_REGION}  (override with RAG_REGION= if needed)"
@@ -36,21 +39,57 @@ else
   echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}"
 fi
 
-# ── Upload seed docs ──────────────────────────────────────────────────────────
+# ── Upload docs: corpus-seed/*.md + docs/*.md (top-level only) ───────────────────
-SEED_DIR="$(dirname "$0")/../docs/corpus-seed"
+UPLOAD_COUNT=0
-if [[ -d "${SEED_DIR}" ]] && ls "${SEED_DIR}"/*.md &>/dev/null; then
+
+# 1. corpus-seed/ — kuraterte seed-filer
+SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
+if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then
   gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true
-  echo "✓ Seed documents uploaded to gs://${CORPUS_BUCKET}/seed/"
+  COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
+  echo "✓ corpus-seed: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/seed/"
+  UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
 else
-  echo "  No seed documents in docs/corpus-seed/ — skipping"
+  echo "  Ingen filer i docs/corpus-seed/ — hopper over"
 fi
 
-# ── REST: check if corpus exists ───────────────────────────────────────────
+# 2. docs/*.md — alle toppnivå docs (GCP-kunnskap, arkitektur, regler osv.)
+DOCS_DIR="${REPO_ROOT}/docs"
+if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then
+  gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true
+  COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
+  echo "✓ docs/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/docs/"
+  UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
+else
+  echo "  Ingen .md-filer i docs/ — hopper over"
+fi
+
+# 3. protocols/sessions/*.md — sesjonlogger (hele reisen)
+SESSIONS_DIR="${REPO_ROOT}/protocols/sessions"
+if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then
+  gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true
+  COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
+  echo "✓ protocols/sessions/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/sessions/"
+  UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
+else
+  echo "  Ingen sesjonlogger i protocols/sessions/ — hopper over"
+fi
+
+# 4. MASTERPLAN.md (rot)
+if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
+  gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/" 2>/dev/null || true
+  echo "✓ MASTERPLAN.md lastet opp"
+  UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
+fi
+
+echo "  Totalt ${UPLOAD_COUNT} filer synkronisert til gs://${CORPUS_BUCKET}/"
+
+# ── REST: token + endpoint ────────────────────────────────────────────────────
 TOKEN=$(gcloud auth print-access-token)
 API="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
 PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
 
-echo "  Checking for existing corpus..."
+echo "  Sjekker om corpus finnes..."
 EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
   "${API}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}')
 
@@ -64,9 +103,9 @@ for c in data.get('ragCorpora', []):
 " 2>/dev/null || true)
 
 if [[ -n "${CORPUS_NAME}" ]]; then
-  echo "✓ RAG corpus already exists: ${CORPUS_NAME}"
+  echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}"
 else
-  echo "  Creating corpus '${RAG_CORPUS_DISPLAY_NAME}' in ${RAG_REGION} (Serverless/REST)..."
+  echo "  Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION} (Serverless/REST)..."
 
   RESPONSE=$(curl -sf -X POST \
     -H "Authorization: Bearer ${TOKEN}" \
@@ -84,16 +123,15 @@ else
       }
     }')
 
-  # REST create returns an LRO — poll until done
   OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true)
 
   if [[ -z "${OPERATION}" ]]; then
-    echo "ERROR: No operation returned. Response:"
+    echo "ERROR: Ingen operation returnert. Response:"
     echo "${RESPONSE}"
     exit 1
   fi
 
-  echo "  Waiting for LRO: ${OPERATION}"
+  echo "  Venter på LRO: ${OPERATION}"
   for i in $(seq 1 30); do
     sleep 10
     LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
@@ -103,9 +141,8 @@ else
       CORPUS_NAME=$(echo "${LRO}" | python3 -c "
 import sys, json
 d = json.load(sys.stdin)
-print(d.get('response', {}).get('name', '') or d.get('metadata', {}).get('genericMetadata', {}).get('updateTime', ''))
+print(d.get('response', {}).get('name', ''))
 " 2>/dev/null || true)
-      # Fallback: re-list to get name
       if [[ -z "${CORPUS_NAME}" ]] || [[ "${CORPUS_NAME}" != *ragCorpora* ]]; then
         CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
           "${API}/${PARENT}/ragCorpora" 2>/dev/null \
@@ -118,32 +155,39 @@ for c in data.get('ragCorpora', []):
         break
 " 2>/dev/null || true)
       fi
-      echo "✓ RAG corpus created: ${CORPUS_NAME}"
+      echo "✓ RAG corpus opprettet: ${CORPUS_NAME}"
       break
     fi
-    echo "  ... still waiting (${i}/30)"
+    echo "  ... venter (${i}/30)"
   done
 
   if [[ -z "${CORPUS_NAME}" ]]; then
-    echo "ERROR: Corpus creation timed out or failed."
+    echo "ERROR: Corpus-oppretting timet ut eller feilet."
     exit 1
   fi
 fi
 
-# ── Import seed docs via REST ─────────────────────────────────────────────
+# ── Import alle GCS-mapper til corpus ───────────────────────────────────────────
-GCS_URI="gs://${CORPUS_BUCKET}/seed/"
+for GCS_PATH in "seed/" "docs/" "sessions/" ""; do
-curl -sf -X POST \
+  GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
-  -H "Authorization: Bearer ${TOKEN}" \
+  # Sjekk at det finnes noe der
-  -H "Content-Type: application/json" \
+  if gsutil ls "${GCS_URI}" &>/dev/null; then
-  "${API}/${CORPUS_NAME}:importRagFiles" \
+    curl -sf -X POST \
-  -d '{
+      -H "Authorization: Bearer ${TOKEN}" \
-    "importRagFilesConfig": {
+      -H "Content-Type: application/json" \
-      "gcsSource": { "uris": ["'"${GCS_URI}"'"] },
+      "${API}/${CORPUS_NAME}:importRagFiles" \
-      "ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 }
+      -d '{
-    }
+        "importRagFilesConfig": {
-  }' &>/dev/null && echo "✓ Seed import triggered (async)" || echo "  WARNING: Seed import skipped (non-fatal)"
+          "gcsSource": { "uris": ["'"${GCS_URI}"'"] },
+          "ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 }
+        }
+      }' &>/dev/null \
+      && echo "✓ Import trigget (async): ${GCS_URI}" \
+      || echo "  ADVARSEL: Import hoppet over (ikke fatal): ${GCS_URI}"
+  fi
+done
 
-# ── Write output ──────────────────────────────────────────────────────────────────
+# ── Skriv output ──────────────────────────────────────────────────────────────────
 echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt
 
 echo ""