fix: 07-rag-setup.sh — riktig import-endepunkt /ragFiles:import (ikke :importRagFiles)
This commit is contained in:
parent
d20a7d5878
commit
b242a0f864
|
|
@ -5,6 +5,9 @@
|
|||
# Uploads ALL docs/*.md + corpus-seed/*.md + protocols/sessions/*.md to GCS
|
||||
# Idempotent — safe to run multiple times
|
||||
# Source .env before running: source .env
|
||||
#
|
||||
# VERIFIED WORKING ENDPOINT:
|
||||
# POST /v1beta1/projects/{id}/locations/{region}/ragCorpora/{id}/ragFiles:import
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
|
@ -28,12 +31,7 @@ gcloud services enable aiplatform.googleapis.com \
|
|||
--project="${PROJECT_ID}" --quiet
|
||||
echo "✓ APIs enabled"
|
||||
|
||||
# ── Hent project-number (kræves av importRagFiles-endepunktet) ────────────────
|
||||
PROJECT_NUMBER=$(gcloud projects describe "${PROJECT_ID}" \
|
||||
--format='value(projectNumber)' 2>/dev/null)
|
||||
echo " Project number: ${PROJECT_NUMBER}"
|
||||
|
||||
# ── GCS bucket (idempotent) ─────────────────────────────────────────────
|
||||
# ── GCS bucket (idempotent) ─────────────────────────────────────────────────
|
||||
CORPUS_BUCKET="${PROJECT_ID}-agent-corpus"
|
||||
if ! gsutil ls -b "gs://${CORPUS_BUCKET}" &>/dev/null; then
|
||||
gsutil mb -l "${REGION}" -b on "gs://${CORPUS_BUCKET}"
|
||||
|
|
@ -42,7 +40,7 @@ else
|
|||
echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}"
|
||||
fi
|
||||
|
||||
# ── Upload alle kildedokumenter til GCS ──────────────────────────────────────
|
||||
# ── Upload alle kildedokumenter til GCS ─────────────────────────────────────
|
||||
UPLOAD_COUNT=0
|
||||
|
||||
SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
|
||||
|
|
@ -76,7 +74,7 @@ if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
|
|||
fi
|
||||
echo " Totalt ${UPLOAD_COUNT} filer synkronisert"
|
||||
|
||||
# ── REST: hent/opprett corpus (bruker project-ID for listing/oppretting) ────────
|
||||
# ── REST: hent/opprett corpus ───────────────────────────────────────────────
|
||||
TOKEN=$(gcloud auth print-access-token)
|
||||
BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
|
||||
PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
|
||||
|
|
@ -109,7 +107,7 @@ else
|
|||
for i in $(seq 1 30); do
|
||||
sleep 10
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${OPERATION}" 2>/dev/null || echo '{}')
|
||||
"${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
|
|
@ -128,34 +126,58 @@ for c in data.get('ragCorpora',[]):
|
|||
[[ -z "${CORPUS_NAME}" ]] && { echo "ERROR: timed out"; exit 1; }
|
||||
fi
|
||||
|
||||
# Hent corpus-ID (siste del av ressursstien)
|
||||
CORPUS_ID=$(echo "${CORPUS_NAME}" | rev | cut -d'/' -f1 | rev)
|
||||
|
||||
# ── Import: bygg URL med project-NUMBER (ikke project-ID) ───────────────────────
|
||||
# importRagFiles krever project-number i stien
|
||||
IMPORT_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/projects/${PROJECT_NUMBER}/locations/${RAG_REGION}/ragCorpora/${CORPUS_ID}:importRagFiles"
|
||||
echo " Import URL: ${IMPORT_URL}"
|
||||
# ── Import: riktig endepunkt er /ragFiles:import (bekreftet 2026-05-24) ────────
|
||||
# IKKE :importRagFiles på corpus-nivå — det gir alltid 404 i europe-west4
|
||||
IMPORT_BASE="${BASE_URL}/${CORPUS_NAME}/ragFiles:import"
|
||||
echo " Import URL: ${IMPORT_BASE}"
|
||||
|
||||
IMPORT_OK=0
|
||||
OPS=()
|
||||
for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do
|
||||
GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
|
||||
if gsutil ls "${GCS_URI}" &>/dev/null; then
|
||||
HTTP_STATUS=$(curl -s -o /tmp/import_response.json -w "%{http_code}" -X POST \
|
||||
RESP=$(curl -s -o /tmp/import_resp.json -w "%{http_code}" -X POST \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${IMPORT_URL}" \
|
||||
"${IMPORT_BASE}" \
|
||||
-d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}')
|
||||
if [[ "${HTTP_STATUS}" == "200" ]]; then
|
||||
echo "✓ Import trigget: ${GCS_URI}"
|
||||
if [[ "${RESP}" == "200" ]]; then
|
||||
OP_NAME=$(python3 -c "import sys,json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true)
|
||||
echo "✓ Import trigget: ${GCS_URI} [op: ${OP_NAME##*/}]"
|
||||
OPS+=("${OP_NAME}")
|
||||
IMPORT_OK=$((IMPORT_OK + 1))
|
||||
else
|
||||
echo " ADVARSEL: Import feilet (HTTP ${HTTP_STATUS}): ${GCS_URI}"
|
||||
cat /tmp/import_response.json 2>/dev/null || true
|
||||
echo " ADVARSEL: Import feilet (HTTP ${RESP}): ${GCS_URI}"
|
||||
cat /tmp/import_resp.json 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo " ${IMPORT_OK}/4 import-jobber trigget (async — indeksering tar noen minutter)"
|
||||
|
||||
# ── Vent på at alle LRO-er er ferdig (valgfri polling) ──────────────────────
|
||||
if [[ ${#OPS[@]} -gt 0 ]]; then
|
||||
echo " Venter på indeksering..."
|
||||
for OP in "${OPS[@]}"; do
|
||||
for i in $(seq 1 20); do
|
||||
sleep 15
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${OP}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
echo " ✓ Ferdig: ${OP##*/}"
|
||||
break
|
||||
fi
|
||||
echo " ... ${OP##*/} (${i}/20)"
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
# ── Verifiser: tell antall indekserte filer ──────────────────────────────────
|
||||
RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?')
|
||||
echo " Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}"
|
||||
|
||||
# ── Output ─────────────────────────────────────────────────────────────────
|
||||
echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt
|
||||
echo ""
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user