fix(rag): add data directory to rag setup
This commit is contained in:
Chris Christiansen
parent
ddec009927
commit
a2b18d5ca2
|
|
@ -31,8 +31,8 @@ echo ""
|
|||
|
||||
bash "$(dirname "$0")/00-authcheck.sh"
|
||||
|
||||
gcloud services enable aiplatform.googleapis.com \
|
||||
storage.googleapis.com \
|
||||
gcloud services enable aiplatform.googleapis.com
|
||||
storage.googleapis.com
|
||||
--project="${PROJECT_ID}" --quiet
|
||||
echo "✓ APIs enabled"
|
||||
|
||||
|
|
@ -50,7 +50,7 @@ UPLOAD_COUNT=0
|
|||
|
||||
SEED_DIR="${REPO_ROOT}/docs/corpus-seed"
|
||||
if [[ -d "${SEED_DIR}" ]] && compgen -G "${SEED_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${SEED_DIR}"/*.md "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true
|
||||
gsutil -m cp "${SEED_DIR}/*.md" "gs://${CORPUS_BUCKET}/seed/" 2>/dev/null || true
|
||||
COUNT=$(ls "${SEED_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ corpus-seed/: ${COUNT} filer → gs://${CORPUS_BUCKET}/seed/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
|
|
@ -58,7 +58,7 @@ fi
|
|||
|
||||
DOCS_DIR="${REPO_ROOT}/docs"
|
||||
if compgen -G "${DOCS_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${DOCS_DIR}"/*.md "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true
|
||||
gsutil -m cp "${DOCS_DIR}/*.md" "gs://${CORPUS_BUCKET}/docs/" 2>/dev/null || true
|
||||
COUNT=$(ls "${DOCS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ docs/: ${COUNT} filer → gs://${CORPUS_BUCKET}/docs/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
|
|
@ -66,12 +66,20 @@ fi
|
|||
|
||||
SESSIONS_DIR="${REPO_ROOT}/protocols/sessions"
|
||||
if [[ -d "${SESSIONS_DIR}" ]] && compgen -G "${SESSIONS_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${SESSIONS_DIR}"/*.md "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true
|
||||
gsutil -m cp "${SESSIONS_DIR}/*.md" "gs://${CORPUS_BUCKET}/sessions/" 2>/dev/null || true
|
||||
COUNT=$(ls "${SESSIONS_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ protocols/sessions/: ${COUNT} filer → gs://${CORPUS_BUCKET}/sessions/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
fi
|
||||
|
||||
DATA_DIR="${REPO_ROOT}/data"
|
||||
if [[ -d "${DATA_DIR}" ]] && compgen -G "${DATA_DIR}/*.md" > /dev/null 2>&1; then
|
||||
gsutil -m cp "${DATA_DIR}/*.md" "gs://${CORPUS_BUCKET}/data/" 2>/dev/null || true
|
||||
COUNT=$(ls "${DATA_DIR}"/*.md 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "✓ data/: ${COUNT} filer → gs://${CORPUS_BUCKET}/data/"
|
||||
UPLOAD_COUNT=$((UPLOAD_COUNT + COUNT))
|
||||
fi
|
||||
|
||||
if [[ -f "${REPO_ROOT}/MASTERPLAN.md" ]]; then
|
||||
gsutil cp "${REPO_ROOT}/MASTERPLAN.md" "gs://${CORPUS_BUCKET}/MASTERPLAN.md" 2>/dev/null || true
|
||||
echo "✓ MASTERPLAN.md → gs://${CORPUS_BUCKET}/MASTERPLAN.md"
|
||||
|
|
@ -85,7 +93,7 @@ BASE_URL="https://${RAG_REGION}-aiplatform.googleapis.com/v1beta1"
|
|||
PARENT="projects/${PROJECT_ID}/locations/${RAG_REGION}"
|
||||
|
||||
echo " Sjekker om corpus finnes..."
|
||||
EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
EXISTING=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null || echo '{}')
|
||||
|
||||
CORPUS_NAME=$(echo "${EXISTING}" | python3 -c "
|
||||
|
|
@ -101,22 +109,22 @@ if [[ -n "${CORPUS_NAME}" ]]; then
|
|||
echo "✓ RAG corpus finnes allerede: ${CORPUS_NAME}"
|
||||
else
|
||||
echo " Oppretter corpus '${RAG_CORPUS_DISPLAY_NAME}' i ${RAG_REGION}..."
|
||||
RESPONSE=$(curl -sf -X POST \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" \
|
||||
RESPONSE=$(curl -sf -X POST
|
||||
-H "Authorization: Bearer ${TOKEN}"
|
||||
-H "Content-Type: application/json"
|
||||
"${BASE_URL}/${PARENT}/ragCorpora"
|
||||
-d '{"displayName":"'"${RAG_CORPUS_DISPLAY_NAME}"'","ragEmbeddingModelConfig":{"vertexPredictionEndpoint":{"publisherModel":"publishers/google/models/text-embedding-005"}},"ragVectorDbConfig":{"ragManagedDb":{}}}')
|
||||
OPERATION=$(echo "${RESPONSE}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('name',''))" 2>/dev/null || true)
|
||||
[[ -z "${OPERATION}" ]] && { echo "ERROR: ${RESPONSE}"; exit 1; }
|
||||
echo " Venter på LRO..."
|
||||
for i in $(seq 1 30); do
|
||||
sleep 10
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
|
||||
"${BASE_URL}/${OPERATION}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null \
|
||||
CORPUS_NAME=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
|
||||
"${BASE_URL}/${PARENT}/ragCorpora" 2>/dev/null
|
||||
| python3 -c "
|
||||
import sys,json
|
||||
data=json.load(sys.stdin)
|
||||
|
|
@ -144,7 +152,7 @@ wait_for_op() {
|
|||
for i in $(seq 1 40); do
|
||||
sleep 15
|
||||
local LRO DONE
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
LRO=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
|
||||
"${BASE_URL}/${OP}" 2>/dev/null || echo '{}')
|
||||
DONE=$(echo "${LRO}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('done',False))" 2>/dev/null || echo 'False')
|
||||
if [[ "${DONE}" == "True" ]]; then
|
||||
|
|
@ -160,10 +168,10 @@ wait_for_op() {
|
|||
for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do
|
||||
GCS_URI="gs://${CORPUS_BUCKET}/${GCS_PATH}"
|
||||
if gsutil ls "${GCS_URI}" &>/dev/null; then
|
||||
RESP=$(curl -s -o /tmp/import_resp.json -w "%{http_code}" -X POST \
|
||||
-H "Authorization: Bearer ${TOKEN}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${IMPORT_BASE}" \
|
||||
RESP=$(curl -s -o /tmp/import_resp.json -w "%<{http_code}>" -X POST
|
||||
-H "Authorization: Bearer ${TOKEN}"
|
||||
-H "Content-Type: application/json"
|
||||
"${IMPORT_BASE}"
|
||||
-d '{"importRagFilesConfig":{"gcsSource":{"uris":["'"${GCS_URI}"'"]},"ragFileChunkingConfig":{"chunkSize":512,"chunkOverlap":50}}}')
|
||||
if [[ "${RESP}" == "200" ]]; then
|
||||
OP_NAME=$(python3 -c "import json; print(json.load(open('/tmp/import_resp.json')).get('name',''))" 2>/dev/null || true)
|
||||
|
|
@ -177,11 +185,11 @@ for GCS_PATH in "seed/" "docs/" "sessions/" "data/" "MASTERPLAN.md"; do
|
|||
fi
|
||||
fi
|
||||
done
|
||||
echo " ${IMPORT_OK}/4 import-jobber fullført"
|
||||
echo " ${IMPORT_OK}/5 import-jobber fullført"
|
||||
|
||||
# ── Verifiser: tell antall indekserte filer ──────────────────────────────────
|
||||
RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null \
|
||||
RAG_FILE_COUNT=$(curl -sf -H "Authorization: Bearer ${TOKEN}"
|
||||
"${BASE_URL}/${CORPUS_NAME}/ragFiles" 2>/dev/null
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('ragFiles',[])))" 2>/dev/null || echo '?')
|
||||
echo "✓ Indekserte RAG-filer i corpus: ${RAG_FILE_COUNT}"
|
||||
|
||||
|
|
@ -192,18 +200,8 @@ echo " Corpus resource name : ${CORPUS_NAME}"
|
|||
echo " Region : ${RAG_REGION}"
|
||||
echo ""
|
||||
echo " ACTION REQUIRED — legg til i .env:"
|
||||
echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\""
|
||||
echo " export RAG_REGION=\"${RAG_REGION}\""
|
||||
echo ""
|
||||
echo "=== 07: RAG Engine setup COMPLETE ==="
|
||||
echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}"
|
||||
echo ""
|
||||
${CORPUS_NAME}"
|
||||
echo " Region : ${RAG_REGION}"
|
||||
echo ""
|
||||
echo " ACTION REQUIRED — legg til i .env:"
|
||||
echo " export RAG_CORPUS_NAME=\"${CORPUS_NAME}\""
|
||||
echo " export RAG_REGION=\"${RAG_REGION}\""
|
||||
echo " export RAG_CORPUS_NAME="${CORPUS_NAME}""
|
||||
echo " export RAG_REGION="${RAG_REGION}""
|
||||
echo ""
|
||||
echo "=== 07: RAG Engine setup COMPLETE ==="
|
||||
echo " View: https://console.cloud.google.com/vertex-ai/rag?project=${PROJECT_ID}"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user