diff --git a/infrastructure/07-rag-setup.sh b/infrastructure/07-rag-setup.sh index f67f39d..54e5613 100644 --- a/infrastructure/07-rag-setup.sh +++ b/infrastructure/07-rag-setup.sh @@ -2,7 +2,7 @@ # 07-rag-setup.sh — Create Vertex AI RAG Engine corpus in Serverless mode # Uses REST API directly — no SDK version dependency # RAG_REGION defaults to europe-west4 (Serverless available, no allowlist) -# Uploads ALL docs/*.md + docs/corpus-seed/*.md to GCS on every run +# Uploads ALL docs/*.md + corpus-seed/*.md + protocols/sessions/*.md to GCS # Idempotent — safe to run multiple times # Source .env before running: source .env @@ -12,9 +12,7 @@ set -euo pipefail : "${REGION:?Set REGION}" : "${RAG_CORPUS_DISPLAY_NAME:?Set RAG_CORPUS_DISPLAY_NAME}" -# europe-west4: Serverless RAG available, no allowlist, gemini-2.0-flash + 2.5-pro present RAG_REGION="${RAG_REGION:-europe-west4}" - REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" echo "=== 07: Setting up Vertex AI RAG Engine (Serverless mode) ===" @@ -30,7 +28,7 @@ gcloud services enable aiplatform.googleapis.com \ --project="${PROJECT_ID}" --quiet echo "✓ APIs enabled" -# ── GCS bucket (idempotent) ────────────────────────────────────────────── +# ── GCS bucket (idempotent) ───────────────────────────────────────────── CORPUS_BUCKET="${PROJECT_ID}-agent-corpus" if ! gsutil ls -b "gs://${CORPUS_BUCKET}" &>/dev/null; then gsutil mb -l "${REGION}" -b on "gs://${CORPUS_BUCKET}" @@ -39,59 +37,49 @@ else echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}" fi -# ── Upload docs: corpus-seed/*.md + docs/*.md (top-level only) ─────────────────── +# ── Upload alle kildedokumenter til GCS ────────────────────────────────────── UPLOAD_COUNT=0 -# 1. corpus-seed/ — kuraterte seed-filer SEED_DIR="${REPO_ROOT}/docs/corpus-seed" if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') - echo "✓ corpus-seed: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/seed/" + echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) -else - echo " Ingen filer i docs/corpus-seed/ — hopper over" fi -# 2. docs/*.md — alle toppnivå docs (GCP-kunnskap, arkitektur, regler osv.) DOCS_DIR="${REPO_ROOT}/docs" if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') - echo "✓ docs/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/docs/" + echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) -else - echo " Ingen .md-filer i docs/ — hopper over" fi -# 3. protocols/sessions/*.md — sesjonlogger (hele reisen) SESSIONS_DIR="${REPO_ROOT}/protocols/sessions" if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') - echo "✓ protocols/sessions/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/sessions/" + echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) -else - echo " Ingen sesjonlogger i protocols/sessions/ — hopper over" fi -# 4. MASTERPLAN.md (rot) if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then - gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/" 2>/dev/null || true - echo "✓ MASTERPLAN.md lastet opp" + gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true + echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md" UPLOAD_COUNT=$((UPLOAD_COUNT + 1)) fi -echo " Totalt ${UPLOAD_COUNT} filer synkronisert til gs://${CORPUS_BUCKET}/" +echo " Totalt ${UPLOAD_COUNT} filer synkronisert" -# ── REST: token + endpoint ──────────────────────────────────────────────────── +# ── REST: hent/opprett corpus ─────────────────────────────────────────────────── TOKEN=$(gcloud auth print-access-token) -API="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1" +BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1" PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}" echo " Sjekker om corpus finnes..." EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "${API}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}') + "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}') CORPUS_NAME=$(echo "${EXISTING}" | python3 -c " import sys, json @@ -105,96 +93,69 @@ for c in data.get('ragCorpora', []): if [[ -n "${CORPUS_NAME}" ]]; then echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}" else - echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION} (Serverless/REST)..." - + echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..." RESPONSE=$(curl -sf -X POST \ -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ - "${API}/${PARENT}/ragCorpora" \ - -d '{ - "displayName": "'"${RAG_CORPUS_DISPLAY_NAME}"'", - "ragEmbeddingModelConfig": { - "vertexPredictionEndpoint": { - "publisherModel": "publishers/google/models/text-embedding-005" - } - }, - "ragVectorDbConfig": { - "ragManagedDb": {} - } - }') + "${BASE_URL}/${PARENT}/ragCorpora" \ + -d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}') OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true) + [[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; } - if [[ -z "${OPERATION}" ]]; then - echo "ERROR: Ingen operation returnert. Response:" - echo "${RESPONSE}" - exit 1 - fi - - echo " Venter på LRO: ${OPERATION}" + echo " Venter på LRO..." for i in $(seq 1 30); do sleep 10 LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ "https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${OPERATION}" 2>/dev/null || echo '{}') - DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done','false'))" 2>/dev/null || echo 'false') - if [[ "${DONE}" == "True" ]] || [[ "${DONE}" == "true" ]]; then - CORPUS_NAME=$(echo "${LRO}" | python3 -c " -import sys, json -d = json.load(sys.stdin) -print(d.get('response', {}).get('name', '')) + DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') + if [[ "${DONE}" == "True" ]]; then + CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null \ + | python3 -c " +import sys,json +data=json.load(sys.stdin) +for c in data.get('ragCorpora',[]): + if c.get('displayName')=='${RAG_CORPUS_DISPLAY_NAME}': print(c['name']); break " 2>/dev/null || true) - if [[ -z "${CORPUS_NAME}" ]] || [[ "${CORPUS_NAME}" != *ragCorpora* ]]; then - CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "${API}/${PARENT}/ragCorpora" 2>/dev/null \ - | python3 -c " -import sys, json -data = json.load(sys.stdin) -for c in data.get('ragCorpora', []): - if c.get('displayName') == '${RAG_CORPUS_DISPLAY_NAME}': - print(c['name']) - break -" 2>/dev/null || true) - fi - echo "✓ RAG corpus opprettet: ${CORPUS_NAME}" + echo "✓ Corpus opprettet: ${CORPUS_NAME}" break fi echo " ... venter (${i}/30)" done - - if [[ -z "${CORPUS_NAME}" ]]; then - echo "ERROR: Corpus-oppretting timet ut eller feilet." - exit 1 - fi + [[ -z "${CORPUS_NAME}" ]] && { echo "ERROR: timed out"; exit 1; } fi -# ── Import alle GCS-mapper til corpus ─────────────────────────────────────────── -for GCS_PATH in "seed/" "docs/" "sessions/" ""; do +# ── Import: CORPUS_NAME er full ressurssti — bruk direkte som URL-path ───────── +# Korrekt URL: https://{region}-aiplatform.googleapis.com/v1beta1/{corpus_name}:importRagFiles +IMPORT_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${CORPUS_NAME}:importRagFiles" + +IMPORT_OK=0 +for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}" - # Sjekk at det finnes noe der if gsutil ls "${GCS_URI}" &>/dev/null; then - curl -sf -X POST \ + HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ - "${API}/${CORPUS_NAME}:importRagFiles" \ - -d '{ - "importRagFilesConfig": { - "gcsSource": { "uris": ["'"${GCS_URI}"'"] }, - "ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 } - } - }' &>/dev/null \ - && echo "✓ Import trigget (async): ${GCS_URI}" \ - || echo " ADVARSEL: Import hoppet over (ikke fatal): ${GCS_URI}" + "${IMPORT_URL}" \ + -d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}') + if [[ "${HTTP_STATUS}" == "200" ]]; then + echo "✓ Import trigget: ${GCS_URI}" + IMPORT_OK=$((IMPORT_OK + 1)) + else + echo " ADVARSEL: Import feilet (HTTP ${HTTP_STATUS}): ${GCS_URI}" + fi fi done +echo " ${IMPORT_OK} import-jobber trigget (async — indeksering tar noen minutter)" -# ── Skriv output ────────────────────────────────────────────────────────────────── +# ── Output ───────────────────────────────────────────────────────────────── echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt - echo "" echo " Corpus resource name : ${CORPUS_NAME}" echo " Region : ${RAG_REGION}" echo "" -echo " ACTION REQUIRED — add to .env:" +echo " ACTION REQUIRED — legg til i .env:" echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\"" echo " export RAG_REGION=\"${RAG_REGION}\"" echo ""