diff --git a/infrastructure/07-rag-setup.sh b/infrastructure/07-rag-setup.sh index c9f8fcc..68629c8 100644 --- a/infrastructure/07-rag-setup.sh +++ b/infrastructure/07-rag-setup.sh @@ -31,8 +31,8 @@ echo "" bash "$(dirname "$0")/00-authcheck.sh" -gcloud services enable aiplatform.googleapis.com \ - storage.googleapis.com \ +gcloud services enable aiplatform.googleapis.com + storage.googleapis.com --project="${PROJECT_ID}" --quiet echo "✓ APIs enabled" @@ -50,7 +50,7 @@ UPLOAD_COUNT=0 SEED_DIR="${REPO_ROOT}/docs/corpus-seed" if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then - gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true + gsutil -m cp "${SEED_DIR}/*.md" "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) @@ -58,7 +58,7 @@ fi DOCS_DIR="${REPO_ROOT}/docs" if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then - gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true + gsutil -m cp "${DOCS_DIR}/*.md" "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) @@ -66,12 +66,20 @@ fi SESSIONS_DIR="${REPO_ROOT}/protocols/sessions" if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then - gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true + gsutil -m cp "${SESSIONS_DIR}/*.md" "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/" UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) fi +DATA_DIR="${REPO_ROOT}/data" +if [[ -d "${DATA_DIR}" ]] && compgen -G "${DATA_DIR}/*.md" > /dev/null 2>&1; then + gsutil -m cp "${DATA_DIR}/*.md" "gs://${CORPUS_BUCKET}/data/" 2>/dev/null || true + COUNT=$(ls "${DATA_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') + echo "✓ data/: ${COUNT} filer → gs://${CORPUS_BUCKET}/data/" + UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) +fi + if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md" @@ -85,7 +93,7 @@ BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1" PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}" echo " Sjekker om corpus finnes..." -EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ +EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}') CORPUS_NAME=$(echo "${EXISTING}" | python3 -c " @@ -101,22 +109,22 @@ if [[ -n "${CORPUS_NAME}" ]]; then echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}" else echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..." - RESPONSE=$(curl -sf -X POST \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - "${BASE_URL}/${PARENT}/ragCorpora" \ + RESPONSE=$(curl -sf -X POST + -H "Authorization: Bearer ${TOKEN}" + -H "Content-Type: application/json" + "${BASE_URL}/${PARENT}/ragCorpora" -d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}') OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true) [[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; } echo " Venter på LRO..." for i in $(seq 1 30); do sleep 10 - LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" "${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}') DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') if [[ "${DONE}" == "True" ]]; then - CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null \ + CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" + "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null | python3 -c " import sys,json data=json.load(sys.stdin) @@ -144,7 +152,7 @@ wait_for_op() { for i in $(seq 1 40); do sleep 15 local LRO DONE - LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" "${BASE_URL}/${OP}" 2>/dev/null || echo '{}') DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') if [[ "${DONE}" == "True" ]]; then @@ -160,10 +168,10 @@ wait_for_op() { for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}" if gsutil ls "${GCS_URI}" &>/dev/null; then - RESP=$(curl -s -o /tmp/import_resp.json -w "%{http_code}" -X POST \ - -H "Authorization: Bearer ${TOKEN}" \ - -H "Content-Type: application/json" \ - "${IMPORT_BASE}" \ + RESP=$(curl -s -o /tmp/import_resp.json -w "%<{http_code}>" -X POST + -H "Authorization: Bearer ${TOKEN}" + -H "Content-Type: application/json" + "${IMPORT_BASE}" -d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}') if [[ "${RESP}" == "200" ]]; then OP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true) @@ -177,11 +185,11 @@ for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do fi fi done -echo " ${IMPORT_OK}/4 import-jobber fullført" +echo " ${IMPORT_OK}/5 import-jobber fullført" # ── Verifiser: tell antall indekserte filer ────────────────────────────────── -RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \ +RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" + "${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?') echo "✓ Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}" @@ -192,18 +200,8 @@ echo " Corpus resource name : ${CORPUS_NAME}" echo " Region : ${RAG_REGION}" echo "" echo " ACTION REQUIRED — legg til i .env:" -echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\"" -echo " export RAG_REGION=\"${RAG_REGION}\"" -echo "" -echo "=== 07: RAG Engine setup COMPLETE ===" -echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}" -echo "" - ${CORPUS_NAME}" -echo " Region : ${RAG_REGION}" -echo "" -echo " ACTION REQUIRED — legg til i .env:" -echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\"" -echo " export RAG_REGION=\"${RAG_REGION}\"" +echo " export RAG_CORPUS_NAME="${CORPUS_NAME}"" +echo " export RAG_REGION="${RAG_REGION}"" echo "" echo "=== 07: RAG Engine setup COMPLETE ===" echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}"