diff --git a/infrastructure/07-rag-setup.sh b/infrastructure/07-rag-setup.sh index b57b07a..f67f39d 100644 --- a/infrastructure/07-rag-setup.sh +++ b/infrastructure/07-rag-setup.sh @@ -2,6 +2,7 @@ # 07-rag-setup.sh — Create Vertex AI RAG Engine corpus in Serverless mode # Uses REST API directly — no SDK version dependency # RAG_REGION defaults to europe-west4 (Serverless available, no allowlist) +# Uploads ALL docs/*.md + docs/corpus-seed/*.md to GCS on every run # Idempotent — safe to run multiple times # Source .env before running: source .env @@ -14,6 +15,8 @@ set -euo pipefail # europe-west4: Serverless RAG available, no allowlist, gemini-2.0-flash + 2.5-pro present RAG_REGION="${RAG_REGION:-europe-west4}" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + echo "=== 07: Setting up Vertex AI RAG Engine (Serverless mode) ===" echo " Project : ${PROJECT_ID}" echo " Region : ${RAG_REGION} (override with RAG_REGION= if needed)" @@ -36,21 +39,57 @@ else echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}" fi -# ── Upload seed docs ────────────────────────────────────────────────────────── -SEED_DIR="$(dirname "$0")/../docs/corpus-seed" -if [[ -d "${SEED_DIR}" ]] && ls "${SEED_DIR}"/*.md &>/dev/null; then +# ── Upload docs: corpus-seed/*.md + docs/*.md (top-level only) ─────────────────── +UPLOAD_COUNT=0 + +# 1. corpus-seed/ — kuraterte seed-filer +SEED_DIR="${REPO_ROOT}/docs/corpus-seed" +if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true - echo "✓ Seed documents uploaded to gs://${CORPUS_BUCKET}/seed/" + COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') + echo "✓ corpus-seed: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/seed/" + UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) else - echo " No seed documents in docs/corpus-seed/ — skipping" + echo " Ingen filer i docs/corpus-seed/ — hopper over" fi -# ── REST: check if corpus exists ─────────────────────────────────────────── +# 2. docs/*.md — alle toppnivå docs (GCP-kunnskap, arkitektur, regler osv.) +DOCS_DIR="${REPO_ROOT}/docs" +if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then + gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true + COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') + echo "✓ docs/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/docs/" + UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) +else + echo " Ingen .md-filer i docs/ — hopper over" +fi + +# 3. protocols/sessions/*.md — sesjonlogger (hele reisen) +SESSIONS_DIR="${REPO_ROOT}/protocols/sessions" +if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then + gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true + COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') + echo "✓ protocols/sessions/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/sessions/" + UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) +else + echo " Ingen sesjonlogger i protocols/sessions/ — hopper over" +fi + +# 4. MASTERPLAN.md (rot) +if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then + gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/" 2>/dev/null || true + echo "✓ MASTERPLAN.md lastet opp" + UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) +fi + +echo " Totalt ${UPLOAD_COUNT} filer synkronisert til gs://${CORPUS_BUCKET}/" + +# ── REST: token + endpoint ──────────────────────────────────────────────────── TOKEN=$(gcloud auth print-access-token) API="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1" PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}" -echo " Checking for existing corpus..." +echo " Sjekker om corpus finnes..." EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ "${API}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}') @@ -64,9 +103,9 @@ for c in data.get('ragCorpora', []): " 2>/dev/null || true) if [[ -n "${CORPUS_NAME}" ]]; then - echo "✓ RAG corpus already exists: ${CORPUS_NAME}" + echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}" else - echo " Creating corpus '${RAG_CORPUS_DISPLAY_NAME}' in ${RAG_REGION} (Serverless/REST)..." + echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION} (Serverless/REST)..." RESPONSE=$(curl -sf -X POST \ -H "Authorization: Bearer ${TOKEN}" \ @@ -84,16 +123,15 @@ else } }') - # REST create returns an LRO — poll until done OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true) if [[ -z "${OPERATION}" ]]; then - echo "ERROR: No operation returned. Response:" + echo "ERROR: Ingen operation returnert. Response:" echo "${RESPONSE}" exit 1 fi - echo " Waiting for LRO: ${OPERATION}" + echo " Venter på LRO: ${OPERATION}" for i in $(seq 1 30); do sleep 10 LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ @@ -103,9 +141,8 @@ else CORPUS_NAME=$(echo "${LRO}" | python3 -c " import sys, json d = json.load(sys.stdin) -print(d.get('response', {}).get('name', '') or d.get('metadata', {}).get('genericMetadata', {}).get('updateTime', '')) +print(d.get('response', {}).get('name', '')) " 2>/dev/null || true) - # Fallback: re-list to get name if [[ -z "${CORPUS_NAME}" ]] || [[ "${CORPUS_NAME}" != *ragCorpora* ]]; then CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ "${API}/${PARENT}/ragCorpora" 2>/dev/null \ @@ -118,32 +155,39 @@ for c in data.get('ragCorpora', []): break " 2>/dev/null || true) fi - echo "✓ RAG corpus created: ${CORPUS_NAME}" + echo "✓ RAG corpus opprettet: ${CORPUS_NAME}" break fi - echo " ... still waiting (${i}/30)" + echo " ... venter (${i}/30)" done if [[ -z "${CORPUS_NAME}" ]]; then - echo "ERROR: Corpus creation timed out or failed." + echo "ERROR: Corpus-oppretting timet ut eller feilet." exit 1 fi fi -# ── Import seed docs via REST ───────────────────────────────────────────── -GCS_URI="gs://${CORPUS_BUCKET}/seed/" -curl -sf -X POST \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - "${API}/${CORPUS_NAME}:importRagFiles" \ - -d '{ - "importRagFilesConfig": { - "gcsSource": { "uris": ["'"${GCS_URI}"'"] }, - "ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 } - } - }' &>/dev/null && echo "✓ Seed import triggered (async)" || echo " WARNING: Seed import skipped (non-fatal)" +# ── Import alle GCS-mapper til corpus ─────────────────────────────────────────── +for GCS_PATH in "seed/" "docs/" "sessions/" ""; do + GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}" + # Sjekk at det finnes noe der + if gsutil ls "${GCS_URI}" &>/dev/null; then + curl -sf -X POST \ + -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + "${API}/${CORPUS_NAME}:importRagFiles" \ + -d '{ + "importRagFilesConfig": { + "gcsSource": { "uris": ["'"${GCS_URI}"'"] }, + "ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 } + } + }' &>/dev/null \ + && echo "✓ Import trigget (async): ${GCS_URI}" \ + || echo " ADVARSEL: Import hoppet over (ikke fatal): ${GCS_URI}" + fi +done -# ── Write output ────────────────────────────────────────────────────────────────── +# ── Skriv output ────────────────────────────────────────────────────────────────── echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt echo ""