mmintel/klz-cables.com
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(qdrant-sync): use rsync with SSH keepalive to prevent broken pipe on large transfers
Some checks failed
Build & Deploy / 🔍 Prepare (push) Successful in 6s
Details
Build & Deploy / 🧪 QA (push) Successful in 1m11s
Details
Build & Deploy / 🏗️ Build (push) Successful in 5m40s
Details
Build & Deploy / 🚀 Deploy (push) Failing after 12s
Details
Build & Deploy / 🧪 Post-Deploy Verification (push) Has been skipped
Details
Build & Deploy / 🔔 Notify (push) Successful in 3s
Details

Browse Source
This commit is contained in:
Marc Mintel
2026-03-17 23:51:02 +01:00
parent d6bdd28b30
commit ef4f0e7ace
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/qdrant-sync.sh
View File

@@ -23,7 +23,8 @@ if [[ -z "$TARGET_ENV" ]]; then
exit 1 exit 1
fi fi
LOCAL_QDRANT_URL=${QDRANT_URL:-"http://localhost:16333"} LOCAL_QDRANT_URL=${QDRANT_URL:-"http://localhost:6337"}
TIMEOUT=300 # 5 minutes for large snapshots
get_target_path() { get_target_path() {
case "$1" in case "$1" in
@@ -52,7 +53,7 @@ echo "🚀 Syncing Qdrant Collection '$COLLECTION' to: $TARGET_ENV"
# 1. Create Snapshot Locally # 1. Create Snapshot Locally
echo "📸 1/5 Creating snapshot on local Qdrant ($LOCAL_QDRANT_URL)..." echo "📸 1/5 Creating snapshot on local Qdrant ($LOCAL_QDRANT_URL)..."
SNAPSHOT_INFO=$(curl -s -X POST "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots") SNAPSHOT_INFO=$(curl --max-time $TIMEOUT -s -X POST "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots")
if ! echo "$SNAPSHOT_INFO" | grep -q '"status":"ok"'; then if ! echo "$SNAPSHOT_INFO" | grep -q '"status":"ok"'; then
echo "❌ Failed to create snapshot." echo "❌ Failed to create snapshot."
@@ -65,13 +66,15 @@ echo " ✅ Snapshot created: $SNAPSHOT_NAME"
# 2. Download Snapshot # 2. Download Snapshot
echo "⬇️ 2/5 Downloading snapshot..." echo "⬇️ 2/5 Downloading snapshot..."
curl -s -o "$WORK_DIR/$SNAPSHOT_NAME" "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots/$SNAPSHOT_NAME" curl --max-time $TIMEOUT -s -o "$WORK_DIR/$SNAPSHOT_NAME" "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots/$SNAPSHOT_NAME"
echo " ✅ Downloaded to $WORK_DIR/$SNAPSHOT_NAME" echo " ✅ Downloaded to $WORK_DIR/$SNAPSHOT_NAME"
# 3. Transfer Snapshot # 3. Transfer Snapshot
echo "📤 3/5 Uploading snapshot to Alpha ($SSH_HOST)..." echo "📤 3/5 Uploading snapshot to Alpha ($SSH_HOST)..."
ssh "$SSH_HOST" "mkdir -p $TGT_PATH/qdrant_tmp" SSH_OPTS="-o ServerAliveInterval=60 -o ServerAliveCountMax=10 -o ConnectTimeout=30"
scp "$WORK_DIR/$SNAPSHOT_NAME" "$SSH_HOST:$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME" ssh $SSH_OPTS "$SSH_HOST" "mkdir -p $TGT_PATH/qdrant_tmp"
rsync --partial --progress --timeout=600 -e "ssh $SSH_OPTS" \
"$WORK_DIR/$SNAPSHOT_NAME" "$SSH_HOST:$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME"
echo " ✅ Upload complete." echo " ✅ Upload complete."
# 4. Restore Snapshot on Remote Server # 4. Restore Snapshot on Remote Server
@@ -82,7 +85,7 @@ echo "🔄 4/5 Restoring snapshot on target container ($QDRANT_CONTAINER)..."
# - Download snapshot to container # - Download snapshot to container
# - Recover from snapshot file # - Recover from snapshot file
ssh "$SSH_HOST" << EOF ssh $SSH_OPTS "$SSH_HOST" << EOF
set -e set -e
# Step A: Copy file into the container # Step A: Copy file into the container
docker cp "$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME" $QDRANT_CONTAINER:/qdrant/$SNAPSHOT_NAME docker cp "$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME" $QDRANT_CONTAINER:/qdrant/$SNAPSHOT_NAME