Merge branch 'feature/excel' into feature-ai-search, resolve conflicts

2026-03-21 15:46:12 +01:00
parent 9ba1ddf2a6 b0d66c4192
commit 1ad12a9818
255 changed files with 30803 additions and 1041 deletions
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -14,8 +14,8 @@ on:
        default: 'false'

 env:
-  PUPPETEER_SKIP_DOWNLOAD: 'true'
-  COREPACK_NPM_REGISTRY: 'https://registry.npmmirror.com'
+  PUPPETEER_SKIP_DOWNLOAD: "true"
+  COREPACK_NPM_REGISTRY: "https://registry.npmmirror.com"

 concurrency:
  group: deploy-pipeline
@@ -29,16 +29,16 @@ jobs:
    name: 🔍 Prepare
    runs-on: docker
    outputs:
-      target: ${{ steps.determine.outputs.target }}
-      image_tag: ${{ steps.determine.outputs.image_tag }}
-      env_file: ${{ steps.determine.outputs.env_file }}
-      traefik_host: ${{ steps.determine.outputs.traefik_host }}
-      traefik_rule: ${{ steps.determine.outputs.traefik_rule }}
-      next_public_url: ${{ steps.determine.outputs.next_public_url }}
-      project_name: ${{ steps.determine.outputs.project_name }}
-      short_sha: ${{ steps.determine.outputs.short_sha }}
-      slug: ${{ steps.determine.outputs.slug }}
-      gatekeeper_host: ${{ steps.determine.outputs.gatekeeper_host }}
+      target:           ${{ steps.determine.outputs.target }}
+      image_tag:        ${{ steps.determine.outputs.image_tag }}
+      env_file:         ${{ steps.determine.outputs.env_file }}
+      traefik_host:     ${{ steps.determine.outputs.traefik_host }}
+      traefik_rule:     ${{ steps.determine.outputs.traefik_rule }}
+      next_public_url:  ${{ steps.determine.outputs.next_public_url }}
+      project_name:     ${{ steps.determine.outputs.project_name }}
+      short_sha:        ${{ steps.determine.outputs.short_sha }}
+      slug:             ${{ steps.determine.outputs.slug }}
+      gatekeeper_host:  ${{ steps.determine.outputs.gatekeeper_host }}
    container:
      image: catthehacker/ubuntu:act-latest
    steps:
@@ -96,7 +96,7 @@ jobs:
            TRAEFIK_RULE='Host(`'"$TRAEFIK_HOST"'`)'
            PRIMARY_HOST="$TRAEFIK_HOST"
          fi
-
+          
          GATEKEEPER_HOST="gatekeeper.$PRIMARY_HOST"

          {
@@ -187,13 +187,10 @@ jobs:
      - name: 🔒 Security Audit
        run: pnpm audit --audit-level high || echo "⚠️ Audit found vulnerabilities (non-blocking)"

-      - name: 🧹 Clean Workspace
-        run: rm -rf .next .turbo || true
-
      - name: 🧪 QA Checks
        if: github.event.inputs.skip_checks != 'true'
        env:
-          TURBO_TELEMETRY_DISABLED: '1'
+          TURBO_TELEMETRY_DISABLED: "1"
        run: npx turbo run lint typecheck test --cache-dir=".turbo"
  # ──────────────────────────────────────────────────────────────────────────────
  # JOB 3: Build & Push
@@ -229,21 +226,6 @@ jobs:
          tags: registry.infra.mintel.me/mintel/klz-2026:${{ needs.prepare.outputs.image_tag }}
          secrets: |
            NPM_TOKEN=${{ secrets.NPM_TOKEN }}
-      - name: 🔄 Build and Push Migrator
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          provenance: false
-          platforms: linux/amd64
-          target: migrator
-          build-args: |
-            NEXT_PUBLIC_BASE_URL=${{ needs.prepare.outputs.next_public_url }}
-            NEXT_PUBLIC_TARGET=${{ needs.prepare.outputs.target }}
-            NPM_TOKEN=${{ secrets.NPM_TOKEN }}
-          tags: registry.infra.mintel.me/mintel/klz-2026:migrate-${{ needs.prepare.outputs.image_tag }}
-          secrets: |
-            NPM_TOKEN=${{ secrets.NPM_TOKEN }}

  # ──────────────────────────────────────────────────────────────────────────────
  # JOB 4: Deploy
@@ -255,56 +237,56 @@ jobs:
    container:
      image: catthehacker/ubuntu:act-latest
    env:
-      TARGET: ${{ needs.prepare.outputs.target }}
-      IMAGE_TAG: ${{ needs.prepare.outputs.image_tag }}
-      PROJECT_NAME: ${{ needs.prepare.outputs.project_name }}
+      TARGET:               ${{ needs.prepare.outputs.target }}
+      IMAGE_TAG:            ${{ needs.prepare.outputs.image_tag }}
+      PROJECT_NAME:         ${{ needs.prepare.outputs.project_name }}
      NEXT_PUBLIC_BASE_URL: ${{ needs.prepare.outputs.next_public_url }}
-      TRAEFIK_HOST: ${{ needs.prepare.outputs.traefik_host }}
-      GATEKEEPER_HOST: ${{ needs.prepare.outputs.gatekeeper_host }}
-      SLUG: ${{ needs.prepare.outputs.slug }}
-
+      TRAEFIK_HOST:         ${{ needs.prepare.outputs.traefik_host }}
+      GATEKEEPER_HOST:      ${{ needs.prepare.outputs.gatekeeper_host }}
+      SLUG:                 ${{ needs.prepare.outputs.slug }}
+      
      # Secrets mapping (Payload CMS)
-      PAYLOAD_SECRET: ${{ secrets.PAYLOAD_SECRET || vars.PAYLOAD_SECRET || 'you-need-to-set-a-payload-secret' }}
-      PAYLOAD_DB_NAME: ${{ secrets.PAYLOAD_DB_NAME || vars.PAYLOAD_DB_NAME || 'payload' }}
-      PAYLOAD_DB_USER: ${{ secrets.PAYLOAD_DB_USER || vars.PAYLOAD_DB_USER || 'payload' }}
-      PAYLOAD_DB_PASSWORD: ${{ (needs.prepare.outputs.target == 'testing' && secrets.TESTING_PAYLOAD_DB_PASSWORD) || (needs.prepare.outputs.target == 'staging' && secrets.STAGING_PAYLOAD_DB_PASSWORD) || secrets.PAYLOAD_DB_PASSWORD || vars.PAYLOAD_DB_PASSWORD || 'payload' }}
-
+      PAYLOAD_SECRET:          ${{ secrets.PAYLOAD_SECRET || vars.PAYLOAD_SECRET || 'you-need-to-set-a-payload-secret' }}
+      PAYLOAD_DB_NAME:         ${{ secrets.PAYLOAD_DB_NAME || vars.PAYLOAD_DB_NAME || 'payload' }}
+      PAYLOAD_DB_USER:         ${{ secrets.PAYLOAD_DB_USER || vars.PAYLOAD_DB_USER || 'payload' }}
+      PAYLOAD_DB_PASSWORD:     ${{ (needs.prepare.outputs.target == 'testing' && secrets.TESTING_PAYLOAD_DB_PASSWORD) || (needs.prepare.outputs.target == 'staging' && secrets.STAGING_PAYLOAD_DB_PASSWORD) || secrets.PAYLOAD_DB_PASSWORD || vars.PAYLOAD_DB_PASSWORD || 'payload' }}
+      
      # Secrets mapping (Mail)
-      MAIL_HOST: ${{ secrets.SMTP_HOST || vars.SMTP_HOST }}
-      MAIL_PORT: ${{ secrets.SMTP_PORT || vars.SMTP_PORT || '587' }}
-      MAIL_USERNAME: ${{ secrets.SMTP_USER || vars.SMTP_USER }}
-      MAIL_PASSWORD: ${{ secrets.SMTP_PASS || vars.SMTP_PASS }}
-      MAIL_FROM: ${{ secrets.SMTP_FROM || vars.SMTP_FROM }}
-      MAIL_RECIPIENTS: ${{ secrets.CONTACT_RECIPIENT || vars.CONTACT_RECIPIENT }}
-
+      MAIL_HOST:        ${{ secrets.SMTP_HOST || vars.SMTP_HOST }}
+      MAIL_PORT:        ${{ secrets.SMTP_PORT || vars.SMTP_PORT || '587' }}
+      MAIL_USERNAME:    ${{ secrets.SMTP_USER || vars.SMTP_USER }}
+      MAIL_PASSWORD:    ${{ secrets.SMTP_PASS || vars.SMTP_PASS }}
+      MAIL_FROM:        ${{ secrets.SMTP_FROM || vars.SMTP_FROM || 'noreply@klz-cables.com' }}
+      MAIL_RECIPIENTS:  ${{ secrets.CONTACT_RECIPIENT || vars.CONTACT_RECIPIENT || 'info@klz-cables.com' }}
+      
      # Monitoring
-      SENTRY_DSN: ${{ secrets.SENTRY_DSN || vars.SENTRY_DSN }}
-
+      SENTRY_DSN:       ${{ secrets.SENTRY_DSN || vars.SENTRY_DSN }}
+      
      # Gatekeeper
      GATEKEEPER_PASSWORD: ${{ secrets.GATEKEEPER_PASSWORD || 'klz2026' }}

      # Analytics
-      UMAMI_WEBSITE_ID: ${{ secrets.UMAMI_WEBSITE_ID || vars.UMAMI_WEBSITE_ID }}
-      UMAMI_API_ENDPOINT: ${{ secrets.UMAMI_API_ENDPOINT || vars.UMAMI_API_ENDPOINT || 'https://analytics.infra.mintel.me' }}
-
+      UMAMI_WEBSITE_ID:    ${{ secrets.UMAMI_WEBSITE_ID || vars.UMAMI_WEBSITE_ID }}
+      UMAMI_API_ENDPOINT:  ${{ secrets.UMAMI_API_ENDPOINT || vars.UMAMI_API_ENDPOINT || 'https://analytics.infra.mintel.me' }}
+      
      # Search & AI
-      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY || vars.OPENROUTER_API_KEY }}
-      MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY || vars.MISTRAL_API_KEY }}
-      QDRANT_URL: ${{ secrets.QDRANT_URL || vars.QDRANT_URL || 'http://klz-qdrant:6333' }}
-      QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY || vars.QDRANT_API_KEY }}
-      REDIS_URL: ${{ secrets.REDIS_URL || vars.REDIS_URL || 'redis://klz-redis:6379' }}
+      OPENROUTER_API_KEY:  ${{ secrets.OPENROUTER_API_KEY || vars.OPENROUTER_API_KEY }}
+      MISTRAL_API_KEY:     ${{ secrets.MISTRAL_API_KEY || vars.MISTRAL_API_KEY }}
+      QDRANT_URL:          ${{ secrets.QDRANT_URL || vars.QDRANT_URL || 'http://klz-qdrant:6333' }}
+      QDRANT_API_KEY:      ${{ secrets.QDRANT_API_KEY || vars.QDRANT_API_KEY }}
+      REDIS_URL:           ${{ secrets.REDIS_URL || vars.REDIS_URL || 'redis://klz-redis:6379' }}
      KABELFACHMANN_MCP_URL: ${{ secrets.KABELFACHMANN_MCP_URL || vars.KABELFACHMANN_MCP_URL || 'http://klz-kabelfachmann:3007/sse' }}
      # Container Registry (standalone)
-      REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
-      REGISTRY_PASS: ${{ secrets.REGISTRY_PASS }}
+      REGISTRY_USER:       ${{ secrets.REGISTRY_USER }}
+      REGISTRY_PASS:       ${{ secrets.REGISTRY_PASS }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: 📝 Generate Environment
        shell: bash
        env:
-          TRAEFIK_RULE: ${{ needs.prepare.outputs.traefik_rule }}
-          ENV_FILE: ${{ needs.prepare.outputs.env_file }}
+          TRAEFIK_RULE:     ${{ needs.prepare.outputs.traefik_rule }}
+          ENV_FILE:         ${{ needs.prepare.outputs.env_file }}
        run: |
          # Middleware Selection Logic
          # Regular app routes get auth on non-production
@@ -312,7 +294,7 @@ jobs:
          LOG_LEVEL=$( [[ "$TARGET" == "testing" || "$TARGET" == "development" ]] && echo "debug" || echo "info" )
          COOKIE_DOMAIN=.$(echo $NEXT_PUBLIC_BASE_URL | sed 's|https://||')
          STD_MW="${PROJECT_NAME}-ratelimit,${PROJECT_NAME}-forward,${PROJECT_NAME}-compress"
-
+          
          if [[ "$TARGET" == "production" ]]; then
            AUTH_MIDDLEWARE="$STD_MW"
            COMPOSE_PROFILES=""
@@ -411,7 +393,11 @@ jobs:
      - name: 🚀 SSH Deploy
        shell: bash
        env:
-          ENV_FILE: ${{ needs.prepare.outputs.env_file }}
+          TARGET:           ${{ needs.prepare.outputs.target }}
+          SLUG:             ${{ needs.prepare.outputs.slug }}
+          IMAGE_TAG:        ${{ needs.prepare.outputs.image_tag }}
+          PROJECT_NAME:     ${{ needs.prepare.outputs.project_name }}
+          ENV_FILE:         ${{ needs.prepare.outputs.env_file }}
        run: |
          mkdir -p ~/.ssh
          echo "${{ secrets.ALPHA_SSH_KEY }}" > ~/.ssh/id_ed25519
@@ -435,48 +421,49 @@ jobs:
          ssh root@alpha.mintel.me "mkdir -p $SITE_DIR"
          scp .env.deploy root@alpha.mintel.me:$SITE_DIR/$ENV_FILE
          scp docker-compose.yml root@alpha.mintel.me:$SITE_DIR/docker-compose.yml
-
+          
+          APP_CONTAINER="${{ needs.prepare.outputs.project_name }}-klz-app-1"
+          DB_CONTAINER="${{ needs.prepare.outputs.project_name }}-klz-db-1"
+          
+          # Branch Seeding Logic (Production -> Branch)
+          if [[ "$TARGET" == "branch" ]]; then
+            echo "🌱 Seeding Branch Environment from Production Database..."
+            ssh root@alpha.mintel.me "cd $SITE_DIR && docker compose -p '${{ needs.prepare.outputs.project_name }}' --env-file $ENV_FILE pull && docker compose -p '${{ needs.prepare.outputs.project_name }}' --env-file $ENV_FILE up -d klz-db"
+            
+            # Wait for DB to be healthy with a 60s timeout
+            echo "⏳ Waiting for branch database to be ready..."
+            ssh root@alpha.mintel.me "
+              for i in {1..30}; do
+                if docker exec $DB_CONTAINER pg_isready -U payload >/dev/null 2>&1; then
+                  exit 0
+                fi
+                sleep 2
+              done
+              echo '❌ Database failed to become ready after 60 seconds'
+              exit 1
+            " || exit 1
+            
+            # Copy Production Payload DB to Branch Payload DB & ensure media is copied
+            echo "📦 Syncing Production DB into Branch DB..."
+            ssh root@alpha.mintel.me "
+              set -e -o pipefail
+              docker exec klz-cablescom-klz-db-1 pg_dump -U payload -d payload --clean --if-exists | docker exec -i $DB_CONTAINER psql -U payload -d payload --quiet
+              rsync -a --delete /var/lib/docker/volumes/klz-cablescom_klz_media_data/_data/ /var/lib/docker/volumes/${{ needs.prepare.outputs.project_name }}_klz_media_data/_data/
+            " || exit 1
+            
+            echo "✅ Branch database and media synced successfully."
+          fi
+          
          # Execute remote commands — alpha is pre-logged into registry.infra.mintel.me
          ssh root@alpha.mintel.me "cd $SITE_DIR && docker compose -p '${{ needs.prepare.outputs.project_name }}' --env-file $ENV_FILE pull && docker compose -p '${{ needs.prepare.outputs.project_name }}' --env-file $ENV_FILE up -d --remove-orphans"
-
-          # Sanitize Payload Migrations: Replace 'dev' push entries with proper migration names.
-          # Without this, Payload prompts interactively for confirmation and blocks forever in Docker.
-          DB_CONTAINER="${{ needs.prepare.outputs.project_name }}-klz-db-1"
-          echo "⏳ Waiting for database container to be ready..."
-          for i in $(seq 1 15); do
-            if ssh root@alpha.mintel.me "docker exec $DB_CONTAINER pg_isready -U payload -q 2>/dev/null"; then
-              echo "✅ Database is ready."
-              break
-            fi
-            echo "   Attempt $i/15..."
-            sleep 2
-          done
-
-          echo "🔧 Sanitizing payload_migrations table (if exists)..."
-          REMOTE_DB_USER=$(ssh root@alpha.mintel.me "grep -h '^PAYLOAD_DB_USER=' $SITE_DIR/.env* 2>/dev/null | tail -1 | cut -d= -f2" || echo "payload")
-          REMOTE_DB_NAME=$(ssh root@alpha.mintel.me "grep -h '^PAYLOAD_DB_NAME=' $SITE_DIR/.env* 2>/dev/null | tail -1 | cut -d= -f2" || echo "payload")
-          REMOTE_DB_USER="${REMOTE_DB_USER:-payload}"
-          REMOTE_DB_NAME="${REMOTE_DB_NAME:-payload}"
-
-          # Run Payload migrations via a temporary container before restarting the app.
-          # This ensures fresh branch deployments (empty DBs) get their schema on first deploy.
-          echo "🔄 Running Payload migrations..."
-          MIGRATOR_IMAGE="registry.infra.mintel.me/mintel/klz-2026:migrate-$IMAGE_TAG"
-
-          ssh root@alpha.mintel.me "
-            echo '${{ steps.auth.outputs.token }}' | docker login registry.infra.mintel.me -u '${{ steps.auth.outputs.user }}' --password-stdin 2>/dev/null || true
-            docker pull $MIGRATOR_IMAGE
-            docker run --rm \
-              --network ${PROJECT_NAME}_default \
-              --env-file $SITE_DIR/$ENV_FILE \
-              $MIGRATOR_IMAGE \
-            && echo '✅ Migrations complete.' \
-            || echo '⚠️  Migrations failed or already up-to-date — continuing.'
-          "
-
+          
          # Restart app to pick up clean migration state
-          APP_CONTAINER="${PROJECT_NAME}-klz-app-1"
          ssh root@alpha.mintel.me "docker restart $APP_CONTAINER"
+          
+          # Generate Excel Datasheets
+          echo "📊 Generating Excel Datasheets on live container..."
+          ssh root@alpha.mintel.me "docker exec $APP_CONTAINER pnpm run excel:datasheets" || echo "⚠️ Excel generation failed (non-blocking)"
+          
          ssh root@alpha.mintel.me "docker system prune -f --filter 'until=24h'"

      - name: 🧹 Post-Deploy Cleanup (Runner)
@@ -489,7 +476,7 @@ jobs:
  post_deploy_checks:
    name: 🧪 Post-Deploy Verification
    needs: [prepare, deploy]
-    if: needs.deploy.result == 'success' && needs.prepare.outputs.target != 'branch'
+    if: needs.deploy.result == 'success' && true
    runs-on: docker
    container:
      image: catthehacker/ubuntu:act-latest
@@ -525,7 +512,7 @@ jobs:
        with:
          path: /usr/bin/chromium
          key: ${{ runner.os }}-chromium-native-${{ hashFiles('package.json') }}
-
+          
      - name: 🔍 Install Chromium (Native & ARM64)
        if: steps.cache-chromium.outputs.cache-hit != 'true'
        run: |
@@ -582,6 +569,15 @@ jobs:
          GATEKEEPER_PASSWORD: ${{ secrets.GATEKEEPER_PASSWORD || 'klz2026' }}
          UMAMI_API_ENDPOINT: ${{ secrets.UMAMI_API_ENDPOINT || vars.UMAMI_API_ENDPOINT || 'https://analytics.infra.mintel.me' }}
          SENTRY_DSN: ${{ secrets.SENTRY_DSN || vars.SENTRY_DSN }}
+      - name: 📊 Excel Datasheet Accessibility Check
+        if: always() && steps.deps.outcome == 'success'
+        env:
+          TEST_URL: ${{ needs.prepare.outputs.next_public_url }}
+        run: |
+          echo "Checking if datasheets directory is reachable..."
+          # This checks if the /datasheets/ directory returns a valid response (200, 403, or 404 is technically reachable, but we'd prefer 200/403)
+          # Since the files are in public/datasheets/products/, we check that path.
+          curl -I -L -s -o /dev/null -w "%{http_code}" "$TEST_URL/datasheets/products/" | grep -E "200|301|302|403|404"

      - name: 📝 E2E Form Submission Test
        if: always() && steps.deps.outcome == 'success'
@@ -636,7 +632,14 @@ jobs:
          fi

          TITLE="$EMOJI klz-cables.com $VERSION -> $TARGET"
-          MESSAGE="$STATUS_LINE | Deploy: $DEPLOY | Smoke: $SMOKE | $URL"
+          MESSAGE="$STATUS_LINE
+          Deploy: $DEPLOY | Smoke: $SMOKE | Perf: $PERF
+          $URL"
+
+          if [[ -z "${{ secrets.GOTIFY_URL }}" || -z "${{ secrets.GOTIFY_TOKEN }}" ]]; then
+            echo "⚠️ Gotify credentials missing, skipping notification."
+            exit 0
+          fi

          curl -s -k -X POST "${{ secrets.GOTIFY_URL }}/message?token=${{ secrets.GOTIFY_TOKEN }}" \
            -F "title=$TITLE" \