diff --git a/infrastructure/07-rag-setup.sh b/infrastructure/07-rag-setup.sh index 85ae9fb..bbb1190 100644 --- a/infrastructure/07-rag-setup.sh +++ b/infrastructure/07-rag-setup.sh @@ -5,6 +5,9 @@ # Uploads ALL docs/*.md + corpus-seed/*.md + protocols/sessions/*.md to GCS # Idempotent — safe to run multiple times # Source .env before running: source .env +# +# VERIFIED WORKING ENDPOINT: +# POST /v1beta1/projects/{id}/locations/{region}/ragCorpora/{id}/ragFiles:import set -euo pipefail @@ -28,12 +31,7 @@ gcloud services enable aiplatform.googleapis.com \ --project="${PROJECT_ID}" --quiet echo "✓ APIs enabled" -# ── Hent project-number (kræves av importRagFiles-endepunktet) ──────────────── -PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" \ - --format='value(projectNumber)' 2>/dev/null) -echo " Project number: ${PROJECT_NUMBER}" - -# ── GCS bucket (idempotent) ───────────────────────────────────────────── +# ── GCS bucket (idempotent) ───────────────────────────────────────────────── CORPUS_BUCKET="${PROJECT_ID}-agent-corpus" if ! gsutil ls -b "gs://${CORPUS_BUCKET}" &>/dev/null; then gsutil mb -l "${REGION}" -b on "gs://${CORPUS_BUCKET}" @@ -42,7 +40,7 @@ else echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}" fi -# ── Upload alle kildedokumenter til GCS ────────────────────────────────────── +# ── Upload alle kildedokumenter til GCS ───────────────────────────────────── UPLOAD_COUNT=0 SEED_DIR="${REPO_ROOT}/docs/corpus-seed" @@ -76,7 +74,7 @@ if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then fi echo " Totalt ${UPLOAD_COUNT} filer synkronisert" -# ── REST: hent/opprett corpus (bruker project-ID for listing/oppretting) ──────── +# ── REST: hent/opprett corpus ─────────────────────────────────────────────── TOKEN=$(gcloud auth print-access-token) BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1" PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}" @@ -109,7 +107,7 @@ else for i in $(seq 1 30); do sleep 10 LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${OPERATION}" 2>/dev/null || echo '{}') + "${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}') DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') if [[ "${DONE}" == "True" ]]; then CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ @@ -128,34 +126,58 @@ for c in data.get('ragCorpora',[]): [[ -z "${CORPUS_NAME}" ]] && { echo "ERROR: timed out"; exit 1; } fi -# Hent corpus-ID (siste del av ressursstien) -CORPUS_ID=$(echo "${CORPUS_NAME}" | rev | cut -d'/' -f1 | rev) - -# ── Import: bygg URL med project-NUMBER (ikke project-ID) ─────────────────────── -# importRagFiles krever project-number i stien -IMPORT_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/projects/${PROJECT_NUMBER}/locations/${RAG_REGION}/ragCorpora/${CORPUS_ID}:importRagFiles" -echo " Import URL: ${IMPORT_URL}" +# ── Import: riktig endepunkt er /ragFiles:import (bekreftet 2026-05-24) ──────── +# IKKE :importRagFiles på corpus-nivå — det gir alltid 404 i europe-west4 +IMPORT_BASE="${BASE_URL}/${CORPUS_NAME}/ragFiles:import" +echo " Import URL: ${IMPORT_BASE}" IMPORT_OK=0 +OPS=() for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}" if gsutil ls "${GCS_URI}" &>/dev/null; then - HTTP_STATUS=$(curl -s -o /tmp/import_response.json -w "%{http_code}" -X POST \ + RESP=$(curl -s -o /tmp/import_resp.json -w "%{http_code}" -X POST \ -H "Authorization: Bearer ${TOKEN}" \ -H "Content-Type: application/json" \ - "${IMPORT_URL}" \ + "${IMPORT_BASE}" \ -d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}') - if [[ "${HTTP_STATUS}" == "200" ]]; then - echo "✓ Import trigget: ${GCS_URI}" + if [[ "${RESP}" == "200" ]]; then + OP_NAME=$(python3 -c "import sys,json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true) + echo "✓ Import trigget: ${GCS_URI} [op: ${OP_NAME##*/}]" + OPS+=("${OP_NAME}") IMPORT_OK=$((IMPORT_OK + 1)) else - echo " ADVARSEL: Import feilet (HTTP ${HTTP_STATUS}): ${GCS_URI}" - cat /tmp/import_response.json 2>/dev/null || true + echo " ADVARSEL: Import feilet (HTTP ${RESP}): ${GCS_URI}" + cat /tmp/import_resp.json 2>/dev/null || true fi fi done echo " ${IMPORT_OK}/4 import-jobber trigget (async — indeksering tar noen minutter)" +# ── Vent på at alle LRO-er er ferdig (valgfri polling) ────────────────────── +if [[ ${#OPS[@]} -gt 0 ]]; then + echo " Venter på indeksering..." + for OP in "${OPS[@]}"; do + for i in $(seq 1 20); do + sleep 15 + LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "${BASE_URL}/${OP}" 2>/dev/null || echo '{}') + DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') + if [[ "${DONE}" == "True" ]]; then + echo " ✓ Ferdig: ${OP##*/}" + break + fi + echo " ... ${OP##*/} (${i}/20)" + done + done +fi + +# ── Verifiser: tell antall indekserte filer ────────────────────────────────── +RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \ + | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?') +echo " Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}" + # ── Output ───────────────────────────────────────────────────────────────── echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt echo ""