fix: 07-rag-setup.sh — rett import-URL (CORPUS_NAME er full ressurssti, ikke relativ)
This commit is contained in:
parent
06767f8806
commit
300ebddd6d
|
|
@ -2,7 +2,7 @@
|
|||
# 07-rag-setup.sh — Create Vertex AI RAG Engine corpus in Serverless mode
|
||||
# Uses REST API directly — no SDK version dependency
|
||||
# RAG_REGION defaults to europe-west4 (Serverless available, no allowlist)
|
||||
# Uploads ALL docs/*.md + docs/corpus-seed/*.md to GCS on every run
|
||||
# Uploads ALL docs/*.md + corpus-seed/*.md + protocols/sessions/*.md to GCS
|
||||
# Idempotent — safe to run multiple times
|
||||
# Source .env before running: source .env
|
||||
|
||||
|
|
@ -12,9 +12,7 @@ set -euo pipefail
|
|||
: "${REGION:?Set REGION}"
|
||||
: "${RAG_CORPUS_DISPLAY_NAME:?Set RAG_CORPUS_DISPLAY_NAME}"
|
||||
|
||||
# europe-west4: Serverless RAG available, no allowlist, gemini-2.0-flash + 2.5-pro present
|
||||
RAG_REGION="${RAG_REGION:-europe-west4}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
echo "=== 07: Setting up Vertex AI RAG Engine (Serverless mode) ==="
|
||||
|
|
@ -30,7 +28,7 @@ gcloud services enable aiplatform.googleapis.com \
|
|||
--project="${PROJECT_ID}" --quiet
|
||||
echo "✓ APIs enabled"
|
||||
|
||||
# ── GCS bucket (idempotent) ──────────────────────────────────────────────
|
||||
# ── GCS bucket (idempotent) ─────────────────────────────────────────────
|
||||
CORPUS_BUCKET="${PROJECT_ID}-agent-corpus"
|
||||
if ! gsutil ls -b "gs://${CORPUS_BUCKET}" &>/dev/null; then
|
||||
gsutil mb -l "${REGION}" -b on "gs://${CORPUS_BUCKET}"
|
||||
|
|
@ -39,59 +37,49 @@ else
|
|||
echo "✓ GCS corpus bucket exists: gs://${CORPUS_BUCKET}"
|
||||
fi
|
||||
|
||||
# ── Upload docs: corpus-seed/*.md + docs/*.md (top-level only) ───────────────────
|
||||
# ── Upload alle kildedokumenter til GCS ──────────────────────────────────────
|
||||
UPLOAD_COUNT=0
|
||||
|
||||
# 1. corpus-seed/ — kuraterte seed-filer
|
||||
SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
|
||||
if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true
|
||||
COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ corpus-seed: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/seed/"
|
||||
echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
else
|
||||
echo " Ingen filer i docs/corpus-seed/ — hopper over"
|
||||
fi
|
||||
|
||||
# 2. docs/*.md — alle toppnivå docs (GCP-kunnskap, arkitektur, regler osv.)
|
||||
DOCS_DIR="${REPO_ROOT}/docs"
|
||||
if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true
|
||||
COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ docs/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/docs/"
|
||||
echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
else
|
||||
echo " Ingen .md-filer i docs/ — hopper over"
|
||||
fi
|
||||
|
||||
# 3. protocols/sessions/*.md — sesjonlogger (hele reisen)
|
||||
SESSIONS_DIR="${REPO_ROOT}/protocols/sessions"
|
||||
if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true
|
||||
COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ protocols/sessions/: ${COUNT} filer lastet opp til gs://${CORPUS_BUCKET}/sessions/"
|
||||
echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
else
|
||||
echo " Ingen sesjonlogger i protocols/sessions/ — hopper over"
|
||||
fi
|
||||
|
||||
# 4. MASTERPLAN.md (rot)
|
||||
if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
|
||||
gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/" 2>/dev/null || true
|
||||
echo "✓ MASTERPLAN.md lastet opp"
|
||||
gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true
|
||||
echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
|
||||
fi
|
||||
|
||||
echo " Totalt ${UPLOAD_COUNT} filer synkronisert til gs://${CORPUS_BUCKET}/"
|
||||
echo " Totalt ${UPLOAD_COUNT} filer synkronisert"
|
||||
|
||||
# ── REST: token + endpoint ────────────────────────────────────────────────────
|
||||
# ── REST: hent/opprett corpus ───────────────────────────────────────────────────
|
||||
TOKEN=$(gcloud auth print-access-token)
|
||||
API="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
|
||||
BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
|
||||
PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
|
||||
|
||||
echo " Sjekker om corpus finnes..."
|
||||
EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${API}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}')
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}')
|
||||
|
||||
CORPUS_NAME=$(echo "${EXISTING}" | python3 -c "
|
||||
import sys, json
|
||||
|
|
@ -105,96 +93,69 @@ for c in data.get('ragCorpora', []):
|
|||
if [[ -n "${CORPUS_NAME}" ]]; then
|
||||
echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}"
|
||||
else
|
||||
echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION} (Serverless/REST)..."
|
||||
|
||||
echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..."
|
||||
RESPONSE=$(curl -sf -X POST \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${API}/${PARENT}/ragCorpora" \
|
||||
-d '{
|
||||
"displayName": "'"${RAG_CORPUS_DISPLAY_NAME}"'",
|
||||
"ragEmbeddingModelConfig": {
|
||||
"vertexPredictionEndpoint": {
|
||||
"publisherModel": "publishers/google/models/text-embedding-005"
|
||||
}
|
||||
},
|
||||
"ragVectorDbConfig": {
|
||||
"ragManagedDb": {}
|
||||
}
|
||||
}')
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" \
|
||||
-d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}')
|
||||
|
||||
OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true)
|
||||
[[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; }
|
||||
|
||||
if [[ -z "${OPERATION}" ]]; then
|
||||
echo "ERROR: Ingen operation returnert. Response:"
|
||||
echo "${RESPONSE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " Venter på LRO: ${OPERATION}"
|
||||
echo " Venter på LRO..."
|
||||
for i in $(seq 1 30); do
|
||||
sleep 10
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${OPERATION}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done','false'))" 2>/dev/null || echo 'false')
|
||||
if [[ "${DONE}" == "True" ]] || [[ "${DONE}" == "true" ]]; then
|
||||
CORPUS_NAME=$(echo "${LRO}" | python3 -c "
|
||||
import sys, json
|
||||
d = json.load(sys.stdin)
|
||||
print(d.get('response', {}).get('name', ''))
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys,json
|
||||
data=json.load(sys.stdin)
|
||||
for c in data.get('ragCorpora',[]):
|
||||
if c.get('displayName')=='${RAG_CORPUS_DISPLAY_NAME}': print(c['name']); break
|
||||
" 2>/dev/null || true)
|
||||
if [[ -z "${CORPUS_NAME}" ]] || [[ "${CORPUS_NAME}" != *ragCorpora* ]]; then
|
||||
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${API}/${PARENT}/ragCorpora" 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for c in data.get('ragCorpora', []):
|
||||
if c.get('displayName') == '${RAG_CORPUS_DISPLAY_NAME}':
|
||||
print(c['name'])
|
||||
break
|
||||
" 2>/dev/null || true)
|
||||
fi
|
||||
echo "✓ RAG corpus opprettet: ${CORPUS_NAME}"
|
||||
echo "✓ Corpus opprettet: ${CORPUS_NAME}"
|
||||
break
|
||||
fi
|
||||
echo " ... venter (${i}/30)"
|
||||
done
|
||||
|
||||
if [[ -z "${CORPUS_NAME}" ]]; then
|
||||
echo "ERROR: Corpus-oppretting timet ut eller feilet."
|
||||
exit 1
|
||||
fi
|
||||
[[ -z "${CORPUS_NAME}" ]] && { echo "ERROR: timed out"; exit 1; }
|
||||
fi
|
||||
|
||||
# ── Import alle GCS-mapper til corpus ───────────────────────────────────────────
|
||||
for GCS_PATH in "seed/" "docs/" "sessions/" ""; do
|
||||
# ── Import: CORPUS_NAME er full ressurssti — bruk direkte som URL-path ─────────
|
||||
# Korrekt URL: https://{region}-aiplatform.googleapis.com/v1beta1/{corpus_name}:importRagFiles
|
||||
IMPORT_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1/${CORPUS_NAME}:importRagFiles"
|
||||
|
||||
IMPORT_OK=0
|
||||
for GCS_PATH in "seed/" "docs/" "sessions/" "MASTERPLAN.md"; do
|
||||
GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
|
||||
# Sjekk at det finnes noe der
|
||||
if gsutil ls "${GCS_URI}" &>/dev/null; then
|
||||
curl -sf -X POST \
|
||||
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${API}/${CORPUS_NAME}:importRagFiles" \
|
||||
-d '{
|
||||
"importRagFilesConfig": {
|
||||
"gcsSource": { "uris": ["'"${GCS_URI}"'"] },
|
||||
"ragFileChunkingConfig": { "chunkSize": 512, "chunkOverlap": 50 }
|
||||
}
|
||||
}' &>/dev/null \
|
||||
&& echo "✓ Import trigget (async): ${GCS_URI}" \
|
||||
|| echo " ADVARSEL: Import hoppet over (ikke fatal): ${GCS_URI}"
|
||||
"${IMPORT_URL}" \
|
||||
-d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}')
|
||||
if [[ "${HTTP_STATUS}" == "200" ]]; then
|
||||
echo "✓ Import trigget: ${GCS_URI}"
|
||||
IMPORT_OK=$((IMPORT_OK + 1))
|
||||
else
|
||||
echo " ADVARSEL: Import feilet (HTTP ${HTTP_STATUS}): ${GCS_URI}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
echo " ${IMPORT_OK} import-jobber trigget (async — indeksering tar noen minutter)"
|
||||
|
||||
# ── Skriv output ──────────────────────────────────────────────────────────────────
|
||||
# ── Output ─────────────────────────────────────────────────────────────────
|
||||
echo "${CORPUS_NAME}" > /tmp/rag_corpus_name.txt
|
||||
|
||||
echo ""
|
||||
echo " Corpus resource name : ${CORPUS_NAME}"
|
||||
echo " Region : ${RAG_REGION}"
|
||||
echo ""
|
||||
echo " ACTION REQUIRED — add to .env:"
|
||||
echo " ACTION REQUIRED — legg til i .env:"
|
||||
echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\""
|
||||
echo " export RAG_REGION=\"${RAG_REGION}\""
|
||||
echo ""
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user