chris/OSVauco
1
0
Fork 0
Code Issues 1 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(rag): add data directory to rag setup

Browse Source
This commit is contained in:
Chris Christiansen 2026-05-30 21:47:35 +00:00
parent ddec009927
commit a2b18d5ca2
1 changed files with 31 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
infrastructure/07-rag-setup.sh
View File

@ -31,8 +31,8 @@ echo ""
bash "$(dirname "$0")/00-authcheck.sh" bash "$(dirname "$0")/00-authcheck.sh"
gcloud services enable aiplatform.googleapis.com \ gcloud services enable aiplatform.googleapis.com
storage.googleapis.com \ storage.googleapis.com
--project="${PROJECT_ID}" --quiet --project="${PROJECT_ID}" --quiet
echo "✓ APIs enabled" echo "✓ APIs enabled"
@ -50,7 +50,7 @@ UPLOAD_COUNT=0
SEED_DIR="${REPO_ROOT}/docs/corpus-seed" SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then
gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true gsutil -m cp "${SEED_DIR}/*.md" "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true
COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/" echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/"
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
@ -58,7 +58,7 @@ fi
DOCS_DIR="${REPO_ROOT}/docs" DOCS_DIR="${REPO_ROOT}/docs"
if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then
gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true gsutil -m cp "${DOCS_DIR}/*.md" "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true
COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/" echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/"
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
@ -66,12 +66,20 @@ fi
SESSIONS_DIR="${REPO_ROOT}/protocols/sessions" SESSIONS_DIR="${REPO_ROOT}/protocols/sessions"
if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then
gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true gsutil -m cp "${SESSIONS_DIR}/*.md" "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true
COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ') COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/" echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/"
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT)) UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
fi fi
DATA_DIR="${REPO_ROOT}/data"
if [[ -d "${DATA_DIR}" ]] && compgen -G "${DATA_DIR}/*.md" > /dev/null 2>&1; then
gsutil -m cp "${DATA_DIR}/*.md" "gs://${CORPUS_BUCKET}/data/" 2>/dev/null || true
COUNT=$(ls "${DATA_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
echo "✓ data/: ${COUNT} filer → gs://${CORPUS_BUCKET}/data/"
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
fi
if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true
echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md" echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md"
@ -85,7 +93,7 @@ BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}" PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
echo " Sjekker om corpus finnes..." echo " Sjekker om corpus finnes..."
EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}') "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}')
CORPUS_NAME=$(echo "${EXISTING}" | python3 -c " CORPUS_NAME=$(echo "${EXISTING}" | python3 -c "
@ -101,22 +109,22 @@ if [[ -n "${CORPUS_NAME}" ]]; then
echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}" echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}"
else else
echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..." echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..."
RESPONSE=$(curl -sf -X POST \ RESPONSE=$(curl -sf -X POST
-H "Authorization: Bearer ${TOKEN}" \ -H "Authorization: Bearer ${TOKEN}"
-H "Content-Type: application/json" \ -H "Content-Type: application/json"
"${BASE_URL}/${PARENT}/ragCorpora" \ "${BASE_URL}/${PARENT}/ragCorpora"
-d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}') -d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}')
OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true) OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true)
[[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; } [[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; }
echo " Venter på LRO..." echo " Venter på LRO..."
for i in $(seq 1 30); do for i in $(seq 1 30); do
sleep 10 sleep 10
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
"${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}') "${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}')
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
if [[ "${DONE}" == "True" ]]; then if [[ "${DONE}" == "True" ]]; then
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null \ "${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null
| python3 -c " | python3 -c "
import sys,json import sys,json
data=json.load(sys.stdin) data=json.load(sys.stdin)
@ -144,7 +152,7 @@ wait_for_op() {
for i in $(seq 1 40); do for i in $(seq 1 40); do
sleep 15 sleep 15
local LRO DONE local LRO DONE
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
"${BASE_URL}/${OP}" 2>/dev/null || echo '{}') "${BASE_URL}/${OP}" 2>/dev/null || echo '{}')
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False') DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
if [[ "${DONE}" == "True" ]]; then if [[ "${DONE}" == "True" ]]; then
@ -160,10 +168,10 @@ wait_for_op() {
for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do
GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}" GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
if gsutil ls "${GCS_URI}" &>/dev/null; then if gsutil ls "${GCS_URI}" &>/dev/null; then
RESP=$(curl -s -o /tmp/import_resp.json -w "%{http_code}" -X POST \ RESP=$(curl -s -o /tmp/import_resp.json -w "%<{http_code}>" -X POST
-H "Authorization: Bearer ${TOKEN}" \ -H "Authorization: Bearer ${TOKEN}"
-H "Content-Type: application/json" \ -H "Content-Type: application/json"
"${IMPORT_BASE}" \ "${IMPORT_BASE}"
-d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}') -d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}')
if [[ "${RESP}" == "200" ]]; then if [[ "${RESP}" == "200" ]]; then
OP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true) OP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true)
@ -177,11 +185,11 @@ for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do
fi fi
fi fi
done done
echo " ${IMPORT_OK}/4 import-jobber fullført" echo " ${IMPORT_OK}/5 import-jobber fullført"
# ── Verifiser: tell antall indekserte filer ────────────────────────────────── # ── Verifiser: tell antall indekserte filer ──────────────────────────────────
RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
"${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \ "${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null
| python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?') | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?')
echo "✓ Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}" echo "✓ Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}"
@ -192,18 +200,8 @@ echo " Corpus resource name : ${CORPUS_NAME}"
echo " Region : ${RAG_REGION}" echo " Region : ${RAG_REGION}"
echo "" echo ""
echo " ACTION REQUIRED — legg til i .env:" echo " ACTION REQUIRED — legg til i .env:"
echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\"" echo " export RAG_CORPUS_NAME="${CORPUS_NAME}""
echo " export RAG_REGION=\"${RAG_REGION}\"" echo " export RAG_REGION="${RAG_REGION}""
echo ""
echo "=== 07: RAG Engine setup COMPLETE ==="
echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}"
echo ""
${CORPUS_NAME}"
echo " Region : ${RAG_REGION}"
echo ""
echo " ACTION REQUIRED — legg til i .env:"
echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\""
echo " export RAG_REGION=\"${RAG_REGION}\""
echo "" echo ""
echo "=== 07: RAG Engine setup COMPLETE ===" echo "=== 07: RAG Engine setup COMPLETE ==="
echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}" echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}"