Compare commits
3 Commits
1dc52da677
...
289e41a040
| Author | SHA1 | Date | |
|---|---|---|---|
| 289e41a040 | |||
| 5144378c7e | |||
| c804b051e0 |
@@ -292,6 +292,7 @@ jobs:
|
|||||||
QDRANT_URL: ${{ secrets.QDRANT_URL || vars.QDRANT_URL || 'http://klz-qdrant:6333' }}
|
QDRANT_URL: ${{ secrets.QDRANT_URL || vars.QDRANT_URL || 'http://klz-qdrant:6333' }}
|
||||||
QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY || vars.QDRANT_API_KEY }}
|
QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY || vars.QDRANT_API_KEY }}
|
||||||
REDIS_URL: ${{ secrets.REDIS_URL || vars.REDIS_URL || 'redis://klz-redis:6379' }}
|
REDIS_URL: ${{ secrets.REDIS_URL || vars.REDIS_URL || 'redis://klz-redis:6379' }}
|
||||||
|
KABELFACHMANN_MCP_URL: ${{ secrets.KABELFACHMANN_MCP_URL || vars.KABELFACHMANN_MCP_URL || 'http://klz-kabelfachmann:3007/sse' }}
|
||||||
# Container Registry (standalone)
|
# Container Registry (standalone)
|
||||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||||
REGISTRY_PASS: ${{ secrets.REGISTRY_PASS }}
|
REGISTRY_PASS: ${{ secrets.REGISTRY_PASS }}
|
||||||
@@ -358,6 +359,7 @@ jobs:
|
|||||||
echo "QDRANT_URL=$QDRANT_URL"
|
echo "QDRANT_URL=$QDRANT_URL"
|
||||||
echo "QDRANT_API_KEY=$QDRANT_API_KEY"
|
echo "QDRANT_API_KEY=$QDRANT_API_KEY"
|
||||||
echo "REDIS_URL=$REDIS_URL"
|
echo "REDIS_URL=$REDIS_URL"
|
||||||
|
echo "KABELFACHMANN_MCP_URL=$KABELFACHMANN_MCP_URL"
|
||||||
echo ""
|
echo ""
|
||||||
echo "TARGET=$TARGET"
|
echo "TARGET=$TARGET"
|
||||||
echo "SENTRY_ENVIRONMENT=$TARGET"
|
echo "SENTRY_ENVIRONMENT=$TARGET"
|
||||||
|
|||||||
@@ -3,6 +3,11 @@ import { searchProducts } from '../../../src/lib/qdrant';
|
|||||||
import redis from '../../../src/lib/redis';
|
import redis from '../../../src/lib/redis';
|
||||||
import { z } from 'zod';
|
import { z } from 'zod';
|
||||||
import * as Sentry from '@sentry/nextjs';
|
import * as Sentry from '@sentry/nextjs';
|
||||||
|
import { generateText } from 'ai';
|
||||||
|
import { createOpenAI } from '@ai-sdk/openai';
|
||||||
|
// @ts-ignore
|
||||||
|
import { createMcpTools } from '@mintel/payload-ai/tools/mcpAdapter';
|
||||||
|
|
||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
export const maxDuration = 60; // Max allowed duration (Vercel)
|
export const maxDuration = 60; // Max allowed duration (Vercel)
|
||||||
|
|
||||||
@@ -108,12 +113,9 @@ Das ECHTE KLZ Team:
|
|||||||
.map((p: any) => p.payload?.content)
|
.map((p: any) => p.payload?.content)
|
||||||
.join('\n\n');
|
.join('\n\n');
|
||||||
|
|
||||||
const knowledgeDescriptions = searchResults
|
if (productDescriptions) {
|
||||||
.filter((p) => p.payload?.type === 'knowledge')
|
contextStr = `KATALOG & PRODUKTE:\n${productDescriptions}`;
|
||||||
.map((p: any) => p.payload?.content)
|
}
|
||||||
.join('\n\n');
|
|
||||||
|
|
||||||
contextStr = `KATALOG & PRODUKTE:\n${productDescriptions}\n\nKABELWISSEN (Handbuch):\n${knowledgeDescriptions}`;
|
|
||||||
|
|
||||||
foundProducts = searchResults
|
foundProducts = searchResults
|
||||||
.filter((p) => (p.payload?.type === 'product' || !p.payload?.type) && p.payload?.data)
|
.filter((p) => (p.payload?.type === 'product' || !p.payload?.type) && p.payload?.data)
|
||||||
@@ -145,12 +147,14 @@ DEINE HAUPTAUFGABE: BERATEN, NICHT AUSFRAGEN!
|
|||||||
- FRAGE NICHT nach abstrakten Dingen wie "Welchen Kabeltyp brauchst du?" -> DAS IST DEIN JOB, IHM DAS ZU SAGEN!
|
- FRAGE NICHT nach abstrakten Dingen wie "Welchen Kabeltyp brauchst du?" -> DAS IST DEIN JOB, IHM DAS ZU SAGEN!
|
||||||
- FRAGE NICHT nach Längen oder genauen Trassen, es sei denn, der Kunde hat schon ganz klar gesagt, was er kaufen will.
|
- FRAGE NICHT nach Längen oder genauen Trassen, es sei denn, der Kunde hat schon ganz klar gesagt, was er kaufen will.
|
||||||
- Biete aktiv Hilfe an: "Ich kann dir die passenden Querschnitte raussuchen, wenn du willst."
|
- Biete aktiv Hilfe an: "Ich kann dir die passenden Querschnitte raussuchen, wenn du willst."
|
||||||
|
- Wenn technisches Wissen aus dem Kabelhandbuch benötigt wird, NUTZE UNBEDINGT eines der "kabelfachmann_*" Tools, anstatt zu raten oder zu behaupten du wüsstest es nicht! Das Tool weiss alles.
|
||||||
|
|
||||||
VORGEHEN:
|
VORGEHEN:
|
||||||
1. Prüfe den KONTEXT auf passende Kabel für das Kundenprojekt.
|
1. Prüfe den KONTEXT auf passende Katalog-Kabel für das Kundenprojekt.
|
||||||
2. Nenne direkt 1-2 passende Produktserien aus dem Kontext, die für diesen Fall Sinn machen.
|
2. Wenn du tiefgehendes Wissen zu einem Kabeltyp brauchst (z.B. Biegeradius, Normen, Querschnitte), rufe das Kabelfachmann-Tool auf.
|
||||||
3. Biete eine konkrete Hilfestellung an (z.B. Leitungsberechnung, Verfügbarkeitsprüfung) ODER stelle EINE einzige fachliche Rückfrage, um das Kabel weiter einzugrenzen (z.B. Alu oder Kupfer?).
|
3. Nenne direkt 1-2 passende Produktserien aus dem Kontext oder der Tool-Abfrage, die für diesen Fall Sinn machen.
|
||||||
4. Wenn das Projekt klar ist und die Kabeltypen besprochen sind, frag nach, ob ein Kollege (z.B. Micha) ein konkretes Angebot machen soll.
|
4. Biete eine konkrete Hilfestellung an (z.B. Leitungsberechnung, Verfügbarkeitsprüfung) ODER stelle EINE einzige fachliche Rückfrage, um das Kabel weiter einzugrenzen (z.B. Alu oder Kupfer?).
|
||||||
|
5. Wenn das Projekt klar ist und die Kabeltypen besprochen sind, frag nach, ob ein Kollege (z.B. Micha) ein konkretes Angebot machen soll.
|
||||||
|
|
||||||
GRENZEN:
|
GRENZEN:
|
||||||
- PRIVAT-ANFRAGEN: B2B only. Private Hausinstallationen lehnen wir freundlich ab.
|
- PRIVAT-ANFRAGEN: B2B only. Private Hausinstallationen lehnen wir freundlich ab.
|
||||||
@@ -161,51 +165,42 @@ ${contextStr || 'Kein Katalogkontext verfügbar.'}
|
|||||||
${teamContextStr}
|
${teamContextStr}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const mistralKey = process.env.MISTRAL_API_KEY;
|
const openrouterApiKey = process.env.OPENROUTER_API_KEY;
|
||||||
if (!mistralKey) {
|
if (!openrouterApiKey) {
|
||||||
throw new Error('MISTRAL_API_KEY is not set');
|
throw new Error('OPENROUTER_API_KEY is not set');
|
||||||
}
|
}
|
||||||
|
|
||||||
// DSGVO: Mistral AI API direkt (EU/Frankreich) statt OpenRouter (US)
|
const openrouter = createOpenAI({
|
||||||
const fetchRes = await fetch('https://api.mistral.ai/v1/chat/completions', {
|
baseURL: 'https://openrouter.ai/api/v1',
|
||||||
method: 'POST',
|
apiKey: openrouterApiKey,
|
||||||
headers: {
|
|
||||||
Authorization: `Bearer ${mistralKey}`,
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
model: 'ministral-8b-latest',
|
|
||||||
temperature: 0.3,
|
|
||||||
max_tokens: MAX_RESPONSE_TOKENS,
|
|
||||||
messages: [
|
|
||||||
{ role: 'system', content: systemPrompt },
|
|
||||||
...cappedMessages.map((m: any) => ({
|
|
||||||
role: m.role,
|
|
||||||
content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content),
|
|
||||||
})),
|
|
||||||
],
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!fetchRes.ok) {
|
let mcpTools: Record<string, any> = {};
|
||||||
const errBody = await fetchRes.text();
|
const mcpUrl = process.env.KABELFACHMANN_MCP_URL || 'http://host.docker.internal:3007/sse';
|
||||||
console.error('Mistral API Error:', errBody);
|
try {
|
||||||
Sentry.captureException(new Error(`Mistral ${fetchRes.status}: ${errBody}`), {
|
const { tools } = await createMcpTools({
|
||||||
tags: { context: 'ai-search-mistral' },
|
name: 'kabelfachmann',
|
||||||
|
url: mcpUrl
|
||||||
});
|
});
|
||||||
|
mcpTools = tools;
|
||||||
// Return user-friendly error based on status
|
} catch (e) {
|
||||||
const userMsg =
|
console.warn('Failed to load MCP tools', e);
|
||||||
fetchRes.status === 429
|
Sentry.captureException(e, { tags: { context: 'ai-search-mcp' } });
|
||||||
? 'Der KI-Service ist gerade überlastet. Bitte versuche es in ein paar Sekunden erneut.'
|
|
||||||
: fetchRes.status >= 500
|
|
||||||
? 'Der KI-Service ist vorübergehend nicht erreichbar. Bitte versuche es gleich nochmal.'
|
|
||||||
: 'Es gab ein Problem mit der KI-Anfrage. Bitte versuche es erneut.';
|
|
||||||
return NextResponse.json({ error: userMsg }, { status: 502 });
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = await fetchRes.json();
|
const { text } = await generateText({
|
||||||
const text = data.choices[0].message.content;
|
model: openrouter('google/gemini-3.0-flash'),
|
||||||
|
system: systemPrompt,
|
||||||
|
messages: cappedMessages.map((m: any) => ({
|
||||||
|
role: m.role,
|
||||||
|
content: typeof m.content === 'string' ? m.content : JSON.stringify(m.content),
|
||||||
|
})),
|
||||||
|
tools: mcpTools,
|
||||||
|
// @ts-ignore
|
||||||
|
maxSteps: 3, // Allow the model to call the tool and then respond
|
||||||
|
temperature: 0.3,
|
||||||
|
maxTokens: MAX_RESPONSE_TOKENS,
|
||||||
|
});
|
||||||
|
|
||||||
// Return the AI's answer along with any found products
|
// Return the AI's answer along with any found products
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
|
|||||||
@@ -112,9 +112,22 @@ services:
|
|||||||
- klz_qdrant_data:/qdrant/storage
|
- klz_qdrant_data:/qdrant/storage
|
||||||
networks:
|
networks:
|
||||||
- default
|
- default
|
||||||
ports:
|
|
||||||
- "16333:6333"
|
- "16333:6333"
|
||||||
|
|
||||||
|
klz-kabelfachmann:
|
||||||
|
image: registry.infra.mintel.me/mintel/kabelfachmann-mcp:${IMAGE_TAG:-latest}
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- default
|
||||||
|
env_file:
|
||||||
|
- ${ENV_FILE:-.env}
|
||||||
|
environment:
|
||||||
|
QDRANT_URL: http://klz-qdrant:6333
|
||||||
|
ports:
|
||||||
|
- "3007:3007"
|
||||||
|
depends_on:
|
||||||
|
- klz-qdrant
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
name: ${PROJECT_NAME:-klz-cables}-internal
|
name: ${PROJECT_NAME:-klz-cables}-internal
|
||||||
|
|||||||
@@ -119,6 +119,20 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
- default
|
- default
|
||||||
|
|
||||||
|
klz-kabelfachmann:
|
||||||
|
image: registry.infra.mintel.me/mintel/kabelfachmann-mcp:${IMAGE_TAG:-latest}
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- default
|
||||||
|
env_file:
|
||||||
|
- ${ENV_FILE:-.env}
|
||||||
|
environment:
|
||||||
|
QDRANT_URL: http://klz-qdrant:6333
|
||||||
|
ports:
|
||||||
|
- "3007:3007"
|
||||||
|
depends_on:
|
||||||
|
- klz-qdrant
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
default:
|
default:
|
||||||
name: ${PROJECT_NAME:-klz-cables}-internal
|
name: ${PROJECT_NAME:-klz-cables}-internal
|
||||||
|
|||||||
@@ -106,7 +106,7 @@
|
|||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "bash -c '[ -f .env ] || (cp .env.example .env && sed -i.bak \"s/TRAEFIK_HOST=klz-cables.com/TRAEFIK_HOST=klz.localhost/\" .env && rm -f .env.bak && echo \"✅ Created .env from .env.example\"); trap \"COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down\" EXIT INT TERM; docker network create infra 2>/dev/null || true && COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down && COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up klz-app klz-db klz-proxy klz-qdrant klz-redis --remove-orphans'",
|
"dev": "bash -c '[ -f .env ] || (cp .env.example .env && sed -i.bak \"s/TRAEFIK_HOST=klz-cables.com/TRAEFIK_HOST=klz.localhost/\" .env && rm -f .env.bak && echo \"✅ Created .env from .env.example\"); trap \"COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down\" EXIT INT TERM; docker network create infra 2>/dev/null || true && COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down && COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up klz-app klz-db klz-proxy klz-qdrant klz-redis --remove-orphans'",
|
||||||
"dev:local": "bash -c 'trap \"COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down\" EXIT INT TERM; COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up -d klz-db klz-proxy klz-qdrant klz-redis && POSTGRES_URI=\"\" NODE_ENV=development next dev --webpack --port 3100 --hostname 0.0.0.0'",
|
"dev:local": "bash -c 'trap \"COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml down\" EXIT INT TERM; COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up -d klz-db klz-proxy klz-qdrant klz-redis && POSTGRES_URI=NODE_ENV=development next dev --webpack --port 3100 --hostname 0.0.0.0'",
|
||||||
"dev:infra": "COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up -d klz-db klz-proxy",
|
"dev:infra": "COMPOSE_PROJECT_NAME=klz-2026 docker-compose -f docker-compose.dev.yml up -d klz-db klz-proxy",
|
||||||
"build": "next build",
|
"build": "next build",
|
||||||
"start": "next start",
|
"start": "next start",
|
||||||
@@ -138,6 +138,10 @@
|
|||||||
"assets:pull:prod": "bash ./scripts/assets-sync.sh prod local",
|
"assets:pull:prod": "bash ./scripts/assets-sync.sh prod local",
|
||||||
"assets:sync:testing-to-staging": "bash ./scripts/assets-sync.sh testing staging",
|
"assets:sync:testing-to-staging": "bash ./scripts/assets-sync.sh testing staging",
|
||||||
"assets:sync:staging-to-prod": "bash ./scripts/assets-sync.sh staging prod",
|
"assets:sync:staging-to-prod": "bash ./scripts/assets-sync.sh staging prod",
|
||||||
|
"qdrant:push:testing": "bash ./scripts/qdrant-sync.sh testing",
|
||||||
|
"qdrant:push:staging": "bash ./scripts/qdrant-sync.sh staging",
|
||||||
|
"qdrant:push:prod": "bash ./scripts/qdrant-sync.sh prod",
|
||||||
|
"qdrant:push:branch": "bash ./scripts/qdrant-sync.sh",
|
||||||
"pagespeed:test": "tsx ./scripts/pagespeed-sitemap.ts",
|
"pagespeed:test": "tsx ./scripts/pagespeed-sitemap.ts",
|
||||||
"pagespeed:audit": "./scripts/audit-local.sh",
|
"pagespeed:audit": "./scripts/audit-local.sh",
|
||||||
"pagespeed:urls": "tsx -e \"import sitemap from './app/sitemap'; sitemap().then(urls => console.log(urls.map(u => u.url).join('\\n')))\"",
|
"pagespeed:urls": "tsx -e \"import sitemap from './app/sitemap'; sitemap().then(urls => console.log(urls.map(u => u.url).join('\\n')))\"",
|
||||||
|
|||||||
120
scripts/qdrant-sync.sh
Executable file
120
scripts/qdrant-sync.sh
Executable file
@@ -0,0 +1,120 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
# Qdrant Snapshot Sync Tool
|
||||||
|
# Syncs a Qdrant collection from the local machine to a remote environment
|
||||||
|
# using the safe Snapshot API to avoid RocksDB corruption.
|
||||||
|
# ────────────────────────────────────────────────────────────────────────────
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Load environment variables
|
||||||
|
if [ -f .env ]; then
|
||||||
|
set -a; source .env; set +a
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── Configuration ──────────────────────────────────────────────────────────
|
||||||
|
TARGET_ENV="${1:-}" # testing | staging | branch_slug | prod
|
||||||
|
COLLECTION="${2:-kabelfachmann}"
|
||||||
|
SSH_HOST="root@alpha.mintel.me"
|
||||||
|
|
||||||
|
if [[ -z "$TARGET_ENV" ]]; then
|
||||||
|
echo "Usage: pnpm run qdrant:push <target_env> [collection]"
|
||||||
|
echo "Example: pnpm run qdrant:push testing kabelfachmann"
|
||||||
|
echo "Example: pnpm run qdrant:push mein-feature-slug kabelfachmann"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
LOCAL_QDRANT_URL=${QDRANT_URL:-"http://localhost:16333"}
|
||||||
|
|
||||||
|
get_target_path() {
|
||||||
|
case "$1" in
|
||||||
|
testing) echo "/home/deploy/sites/testing.klz-cables.com" ;;
|
||||||
|
staging) echo "/home/deploy/sites/staging.klz-cables.com" ;;
|
||||||
|
prod|production) echo "/home/deploy/sites/klz-cables.com" ;;
|
||||||
|
*) echo "/home/deploy/sites/branch.klz-cables.com/$1" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
get_project_name() {
|
||||||
|
case "$1" in
|
||||||
|
testing) echo "klz-testing" ;;
|
||||||
|
staging) echo "klz-staging" ;;
|
||||||
|
prod|production) echo "klz-cablescom" ;;
|
||||||
|
*) echo "klz-branch-$1" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
TGT_PATH=$(get_target_path "$TARGET_ENV")
|
||||||
|
PROJECT_NAME=$(get_project_name "$TARGET_ENV")
|
||||||
|
QDRANT_CONTAINER="${PROJECT_NAME}-klz-qdrant-1"
|
||||||
|
WORK_DIR=$(mktemp -d)
|
||||||
|
|
||||||
|
echo "🚀 Syncing Qdrant Collection '$COLLECTION' to: $TARGET_ENV"
|
||||||
|
|
||||||
|
# 1. Create Snapshot Locally
|
||||||
|
echo "📸 1/5 Creating snapshot on local Qdrant ($LOCAL_QDRANT_URL)..."
|
||||||
|
SNAPSHOT_INFO=$(curl -s -X POST "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots")
|
||||||
|
|
||||||
|
if ! echo "$SNAPSHOT_INFO" | grep -q '"status":"ok"'; then
|
||||||
|
echo "❌ Failed to create snapshot."
|
||||||
|
echo "Response: $SNAPSHOT_INFO"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
SNAPSHOT_NAME=$(echo "$SNAPSHOT_INFO" | grep -o '"name":"[^"]*' | cut -d'"' -f4)
|
||||||
|
echo " ✅ Snapshot created: $SNAPSHOT_NAME"
|
||||||
|
|
||||||
|
# 2. Download Snapshot
|
||||||
|
echo "⬇️ 2/5 Downloading snapshot..."
|
||||||
|
curl -s -o "$WORK_DIR/$SNAPSHOT_NAME" "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots/$SNAPSHOT_NAME"
|
||||||
|
echo " ✅ Downloaded to $WORK_DIR/$SNAPSHOT_NAME"
|
||||||
|
|
||||||
|
# 3. Transfer Snapshot
|
||||||
|
echo "📤 3/5 Uploading snapshot to Alpha ($SSH_HOST)..."
|
||||||
|
ssh "$SSH_HOST" "mkdir -p $TGT_PATH/qdrant_tmp"
|
||||||
|
scp "$WORK_DIR/$SNAPSHOT_NAME" "$SSH_HOST:$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME"
|
||||||
|
echo " ✅ Upload complete."
|
||||||
|
|
||||||
|
# 4. Restore Snapshot on Remote Server
|
||||||
|
echo "🔄 4/5 Restoring snapshot on target container ($QDRANT_CONTAINER)..."
|
||||||
|
|
||||||
|
# Qdrant restore process:
|
||||||
|
# - Recreate collection (so it is clean)
|
||||||
|
# - Download snapshot to container
|
||||||
|
# - Recover from snapshot file
|
||||||
|
|
||||||
|
ssh "$SSH_HOST" << EOF
|
||||||
|
set -e
|
||||||
|
# Step A: Copy file into the container
|
||||||
|
docker cp "$TGT_PATH/qdrant_tmp/$SNAPSHOT_NAME" $QDRANT_CONTAINER:/qdrant/$SNAPSHOT_NAME
|
||||||
|
|
||||||
|
# Step B: Delete existing collection
|
||||||
|
curl -s -X DELETE "http://127.0.0.1:6333/collections/$COLLECTION" > /dev/null
|
||||||
|
|
||||||
|
# Step C: Re-create empty collection (required before recovery)
|
||||||
|
# wir nutzen die standard vector config vom Kabelfachmann (Cosine, 384 dim für all-MiniLM-L6-v2)
|
||||||
|
curl -s -X PUT "http://127.0.0.1:6333/collections/$COLLECTION" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{ "vectors": { "size": 384, "distance": "Cosine" } }' > /dev/null
|
||||||
|
|
||||||
|
# Step D: Recover
|
||||||
|
echo " [Remote] Triggering recover API..."
|
||||||
|
curl -s -X PUT "http://127.0.0.1:6333/collections/$COLLECTION/snapshots/recover" \
|
||||||
|
-H 'Content-Type: application/json' \
|
||||||
|
-d '{ "location": "file:///qdrant/'$SNAPSHOT_NAME'" }' > /dev/null
|
||||||
|
|
||||||
|
# Step E: Cleanup
|
||||||
|
docker exec $QDRANT_CONTAINER rm /qdrant/$SNAPSHOT_NAME
|
||||||
|
rm -rf "$TGT_PATH/qdrant_tmp"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
echo " ✅ Restore complete."
|
||||||
|
|
||||||
|
# 5. Local Cleanup
|
||||||
|
echo "🧹 5/5 Cleaning up..."
|
||||||
|
rm -rf "$WORK_DIR"
|
||||||
|
# Delete snapshot from local Qdrant server to save space
|
||||||
|
curl -s -X DELETE "$LOCAL_QDRANT_URL/collections/$COLLECTION/snapshots/$SNAPSHOT_NAME" > /dev/null
|
||||||
|
echo " ✅ Local cleanup done."
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo "🎉 Successfully synced Qdrant collection '$COLLECTION' to $TARGET_ENV!"
|
||||||
@@ -152,31 +152,6 @@ export async function deleteProductVector(id: string | number) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete knowledge chunks by their source Media ID
|
|
||||||
*/
|
|
||||||
export async function deleteKnowledgeByMediaId(mediaId: string | number) {
|
|
||||||
try {
|
|
||||||
await ensureCollection();
|
|
||||||
await qdrant.delete(COLLECTION_NAME, {
|
|
||||||
wait: true,
|
|
||||||
filter: {
|
|
||||||
must: [
|
|
||||||
{
|
|
||||||
key: 'mediaId',
|
|
||||||
match: {
|
|
||||||
value: mediaId,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
});
|
|
||||||
console.log(`Successfully deleted Qdrant chunks for Media ID: ${mediaId}`);
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error deleting knowledge by Media ID from Qdrant:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Search products in Qdrant.
|
* Search products in Qdrant.
|
||||||
* Results are cached in Redis for 30 minutes keyed by query text.
|
* Results are cached in Redis for 30 minutes keyed by query text.
|
||||||
|
|||||||
@@ -45,81 +45,4 @@ export const Media: CollectionConfig = {
|
|||||||
type: 'text',
|
type: 'text',
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
hooks: {
|
|
||||||
afterChange: [
|
|
||||||
async ({ doc, req }) => {
|
|
||||||
// Only process PDF files
|
|
||||||
if (doc.mimeType === 'application/pdf') {
|
|
||||||
try {
|
|
||||||
const fs = require('fs');
|
|
||||||
const path = require('path');
|
|
||||||
const crypto = require('crypto');
|
|
||||||
const pdfParse = require('pdf-parse');
|
|
||||||
const { upsertProductVector, deleteKnowledgeByMediaId } = require('../../lib/qdrant');
|
|
||||||
|
|
||||||
const filePath = path.join(process.cwd(), 'public/media', doc.filename);
|
|
||||||
|
|
||||||
if (fs.existsSync(filePath)) {
|
|
||||||
req.payload.logger.info(`Extracting text from PDF: ${doc.filename}`);
|
|
||||||
|
|
||||||
const dataBuffer = fs.readFileSync(filePath);
|
|
||||||
const data = await pdfParse(dataBuffer);
|
|
||||||
|
|
||||||
// Clear any previously indexed chunks for this file just in case it's an update
|
|
||||||
await deleteKnowledgeByMediaId(doc.id);
|
|
||||||
|
|
||||||
// Chunk the text like we did in the ingest script
|
|
||||||
const chunks = data.text
|
|
||||||
.split(/\n\s*\n/)
|
|
||||||
.map((c: string) => c.trim())
|
|
||||||
.filter((c: string) => c.length > 50);
|
|
||||||
|
|
||||||
let successCount = 0;
|
|
||||||
for (let i = 0; i < chunks.length; i++) {
|
|
||||||
// Generate a deterministic UUID based on doc ID and chunk index
|
|
||||||
const hash = crypto.createHash('md5').update(`${doc.id}-${i}`).digest('hex');
|
|
||||||
// Qdrant strictly requires UUID: 8-4-4-4-12
|
|
||||||
const uuid = [
|
|
||||||
hash.substring(0, 8),
|
|
||||||
hash.substring(8, 12),
|
|
||||||
hash.substring(12, 16),
|
|
||||||
hash.substring(16, 20),
|
|
||||||
hash.substring(20, 32),
|
|
||||||
].join('-');
|
|
||||||
|
|
||||||
await upsertProductVector(uuid, chunks[i], {
|
|
||||||
type: 'knowledge',
|
|
||||||
title: `${doc.filename} - Teil ${i + 1}`,
|
|
||||||
content: chunks[i],
|
|
||||||
source: doc.filename,
|
|
||||||
mediaId: doc.id,
|
|
||||||
});
|
|
||||||
successCount++;
|
|
||||||
}
|
|
||||||
req.payload.logger.info(
|
|
||||||
`Successfully ingested ${successCount} chunks from ${doc.filename} into Qdrant`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (e: any) {
|
|
||||||
req.payload.logger.error(`Error parsing PDF ${doc.filename}: ${e.message}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
],
|
|
||||||
afterDelete: [
|
|
||||||
async ({ id, doc, req }) => {
|
|
||||||
if (doc.mimeType === 'application/pdf') {
|
|
||||||
try {
|
|
||||||
const { deleteKnowledgeByMediaId } = require('../../lib/qdrant');
|
|
||||||
await deleteKnowledgeByMediaId(id);
|
|
||||||
req.payload.logger.info(`Removed Qdrant chunks for deleted PDF: ${doc.filename}`);
|
|
||||||
} catch (e: any) {
|
|
||||||
req.payload.logger.error(
|
|
||||||
`Error removing Qdrant chunks for ${doc.filename}: ${e.message}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
|
|||||||
BIN
src/scripts/error.png
Normal file
BIN
src/scripts/error.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
@@ -1,58 +0,0 @@
|
|||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
import 'dotenv/config';
|
|
||||||
import { getPayload } from 'payload';
|
|
||||||
import configPromise from '@payload-config';
|
|
||||||
|
|
||||||
async function uploadPDFs() {
|
|
||||||
const payload = await getPayload({ config: configPromise });
|
|
||||||
|
|
||||||
const downloadDir = '/Users/marcmintel/Downloads';
|
|
||||||
const files = fs.readdirSync(downloadDir).filter((f) => f.endsWith('.pdf'));
|
|
||||||
|
|
||||||
console.log(`Found ${files.length} PDFs in Downloads folder.`);
|
|
||||||
|
|
||||||
for (const file of files) {
|
|
||||||
const filePath = path.join(downloadDir, file);
|
|
||||||
try {
|
|
||||||
const stats = fs.statSync(filePath);
|
|
||||||
|
|
||||||
// Check if it already exists
|
|
||||||
const existing = await payload.find({
|
|
||||||
collection: 'media',
|
|
||||||
where: {
|
|
||||||
filename: {
|
|
||||||
equals: file,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (existing.docs.length > 0) {
|
|
||||||
console.log(`Skipping ${file} - already exists in CMS`);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Uploading ${file}...`);
|
|
||||||
await payload.create({
|
|
||||||
collection: 'media',
|
|
||||||
data: {
|
|
||||||
alt: file,
|
|
||||||
},
|
|
||||||
file: {
|
|
||||||
data: fs.readFileSync(filePath),
|
|
||||||
mimetype: 'application/pdf',
|
|
||||||
name: file,
|
|
||||||
size: stats.size,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
console.log(`✅ Uploaded ${file}`);
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`❌ Failed to upload ${file}:`, err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log('Done uploading PDFs to Payload CMS. Payload hooks have synced them to Qdrant.');
|
|
||||||
process.exit(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
uploadPDFs();
|
|
||||||
Reference in New Issue
Block a user