fix: 07-rag-setup.sh — serialiser import-jobber (RAG tillater kun 1 aktiv op om gangen)
This commit is contained in:
parent
b242a0f864
commit
021c84756a
|
|
@ -7,7 +7,8 @@
|
|||
# Source .env before running: source .env
|
||||
#
|
||||
# VERIFIED WORKING ENDPOINT:
|
||||
# POST /v1beta1/projects/{id}/locations/{region}/ragCorpora/{id}/ragFiles:import
|
||||
# POST /v1beta1/.../ragCorpora/{id}/ragFiles:import
|
||||
# NOTE: Only ONE import operation can run at a time per corpus (FAILED_PRECONDITION otherwise)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
|
@ -31,7 +32,7 @@ gcloud services enable aiplatform.googleapis.com \
|
|||
--project="${PROJECT_ID}" --quiet
|
||||
echo "✓ APIs enabled"
|
||||
|
||||
# ── GCS bucket (idempotent) ─────────────────────────────────────────────────
|
||||
# ── GCS bucket (idempotent) ─────────────────────────────────────────────
|
||||
CORPUS_BUCKET="${PROJECT_ID}-agent-corpus"
|
||||
if ! gsutil ls -b "gs://${CORPUS_BUCKET}" &>/dev/null; then
|
||||
gsutil mb -l "${REGION}" -b on "gs://${CORPUS_BUCKET}"
|
||||
|
|
@ -40,7 +41,7 @@ else
|
|||
echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}"
|
||||
fi
|
||||
|
||||
# ── Upload alle kildedokumenter til GCS ─────────────────────────────────────
|
||||
# ── Upload alle kildedokumenter til GCS ──────────────────────────────────
|
||||
UPLOAD_COUNT=0
|
||||
|
||||
SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
|
||||
|
|
@ -74,7 +75,7 @@ if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
|
|||
fi
|
||||
echo " Totalt ${UPLOAD_COUNT} filer synkronisert"
|
||||
|
||||
# ── REST: hent/opprett corpus ───────────────────────────────────────────────
|
||||
# ── REST: hent/opprett corpus ───────────────────────────────────────────
|
||||
TOKEN=$(gcloud auth print-access-token)
|
||||
BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
|
||||
PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
|
||||
|
|
@ -126,13 +127,32 @@ for c in data.get('ragCorpora',[]):
|
|||
[[ -z "${CORPUS_NAME}" ]] && { echo "ERROR: timed out"; exit 1; }
|
||||
fi
|
||||
|
||||
# ── Import: riktig endepunkt er /ragFiles:import (bekreftet 2026-05-24) ────────
|
||||
# IKKE :importRagFiles på corpus-nivå — det gir alltid 404 i europe-west4
|
||||
# ── Hjelpefunksjon: trigge import og VENT til den er ferdig før neste ────────
|
||||
# Vertex AI RAG tillater kun 1 aktiv import-operasjon per corpus om gangen.
|
||||
IMPORT_BASE="${BASE_URL}/${CORPUS_NAME}/ragFiles:import"
|
||||
echo " Import URL: ${IMPORT_BASE}"
|
||||
|
||||
IMPORT_OK=0
|
||||
OPS=()
|
||||
|
||||
wait_for_op() {
|
||||
local OP="$1"
|
||||
local SHORT="${OP##*/}"
|
||||
for i in $(seq 1 40); do
|
||||
sleep 15
|
||||
local LRO DONE
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${OP}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
echo " ✓ Ferdig: ${SHORT}"
|
||||
return 0
|
||||
fi
|
||||
echo " ... ${SHORT} (${i}/40)"
|
||||
done
|
||||
echo " ADVARSEL: Timed out venter på ${SHORT}"
|
||||
return 1
|
||||
}
|
||||
|
||||
for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do
|
||||
GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
|
||||
if gsutil ls "${GCS_URI}" &>/dev/null; then
|
||||
|
|
@ -142,43 +162,26 @@ for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do
|
|||
"${IMPORT_BASE}" \
|
||||
-d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}')
|
||||
if [[ "${RESP}" == "200" ]]; then
|
||||
OP_NAME=$(python3 -c "import sys,json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true)
|
||||
OP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true)
|
||||
echo "✓ Import trigget: ${GCS_URI} [op: ${OP_NAME##*/}]"
|
||||
OPS+=("${OP_NAME}")
|
||||
IMPORT_OK=$((IMPORT_OK + 1))
|
||||
# Vent til denne er ferdig før vi starter neste
|
||||
wait_for_op "${OP_NAME}"
|
||||
else
|
||||
echo " ADVARSEL: Import feilet (HTTP ${RESP}): ${GCS_URI}"
|
||||
cat /tmp/import_resp.json 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo " ${IMPORT_OK}/4 import-jobber trigget (async — indeksering tar noen minutter)"
|
||||
|
||||
# ── Vent på at alle LRO-er er ferdig (valgfri polling) ──────────────────────
|
||||
if [[ ${#OPS[@]} -gt 0 ]]; then
|
||||
echo " Venter på indeksering..."
|
||||
for OP in "${OPS[@]}"; do
|
||||
for i in $(seq 1 20); do
|
||||
sleep 15
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${OP}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
echo " ✓ Ferdig: ${OP##*/}"
|
||||
break
|
||||
fi
|
||||
echo " ... ${OP##*/} (${i}/20)"
|
||||
done
|
||||
done
|
||||
fi
|
||||
echo " ${IMPORT_OK}/4 import-jobber fullført"
|
||||
|
||||
# ── Verifiser: tell antall indekserte filer ──────────────────────────────────
|
||||
RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?')
|
||||
echo " Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}"
|
||||
echo "✓ Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}"
|
||||
|
||||
# ── Output ─────────────────────────────────────────────────────────────────
|
||||
# ── Output ────────────────────────────────────────────────────────────────
|
||||
echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt
|
||||
echo ""
|
||||
echo " Corpus resource name : ${CORPUS_NAME}"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user