wip

tests/column-grouping.test.ts

@@ -0,0 +1,160 @@
+#!/usr/bin/env ts-node
+/**
+ * Test to verify that products with multiple Excel row structures
+ * use the most complete data structure
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { execSync } from 'child_process';
+
+function normalizeValue(value: string): string {
+  if (!value) return '';
+  return String(value)
+    .replace(/<[^>]*>/g, '')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function normalizeExcelKey(value: string): string {
+  return String(value || '')
+    .toUpperCase()
+    .replace(/-\d+$/g, '')
+    .replace(/[^A-Z0-9]+/g, '');
+}
+
+function loadExcelRows(filePath: string): any[] {
+  const out = execSync(`npx -y xlsx-cli -j "${filePath}"`, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] });
+  const trimmed = out.trim();
+  const jsonStart = trimmed.indexOf('[');
+  if (jsonStart < 0) return [];
+  const jsonText = trimmed.slice(jsonStart);
+  try {
+    return JSON.parse(jsonText);
+  } catch {
+    return [];
+  }
+}
+
+// Simulate the Excel index building
+const excelFiles = [
+  'data/source/high-voltage.xlsx',
+  'data/source/medium-voltage-KM.xlsx',
+  'data/source/low-voltage-KM.xlsx',
+  'data/source/solar-cables.xlsx',
+];
+
+const idx = new Map<string, { rows: any[]; units: Record<string, string> }>();
+
+for (const file of excelFiles) {
+  if (!fs.existsSync(file)) continue;
+  const rows = loadExcelRows(file);
+  
+  const unitsRow = rows.find(r => r && r['Part Number'] === 'Units') || null;
+  const units: Record<string, string> = {};
+  if (unitsRow) {
+    for (const [k, v] of Object.entries(unitsRow)) {
+      if (k === 'Part Number') continue;
+      const unit = normalizeValue(String(v ?? ''));
+      if (unit) units[k] = unit;
+    }
+  }
+
+  for (const r of rows) {
+    const pn = r?.['Part Number'];
+    if (!pn || pn === 'Units') continue;
+    const key = normalizeExcelKey(String(pn));
+    if (!key) continue;
+    const cur = idx.get(key);
+    if (!cur) {
+      idx.set(key, { rows: [r], units });
+    } else {
+      cur.rows.push(r);
+      if (Object.keys(cur.units).length < Object.keys(units).length) cur.units = units;
+    }
+  }
+}
+
+// Test NA2XSFL2Y
+const match = idx.get('NA2XSFL2Y');
+if (!match) {
+  console.log('❌ FAIL: NA2XSFL2Y not found in Excel');
+  process.exit(1);
+}
+
+console.log('Test: NA2XSFL2Y multiple row structures');
+console.log('========================================');
+console.log(`Total rows in index: ${match.rows.length}`);
+
+// Count different structures
+const structures: Record<string, number[]> = {};
+match.rows.forEach((r, i) => {
+  const keys = Object.keys(r).filter(k => k && k !== 'Part Number' && k !== 'Units').sort().join('|');
+  if (!structures[keys]) structures[keys] = [];
+  structures[keys].push(i);
+});
+
+const structureCounts = Object.keys(structures).map(key => ({
+  colCount: key.split('|').length,
+  rowCount: structures[key].length,
+  rows: structures[key]
+}));
+
+structureCounts.forEach((s, i) => {
+  console.log(`  Structure ${i+1}: ${s.colCount} columns, ${s.rowCount} rows`);
+});
+
+const mostColumns = Math.max(...structureCounts.map(s => s.colCount));
+console.log(`Most complete structure: ${mostColumns} columns`);
+console.log('');
+
+// Now test the fix: simulate findExcelRowsForProduct
+const rows = match.rows;
+
+// Find the row with most columns as sample
+let sample = rows.find(r => r && Object.keys(r).length > 0) || {};
+let maxColumns = Object.keys(sample).filter(k => k && k !== 'Part Number' && k !== 'Units').length;
+
+for (const r of rows) {
+  const cols = Object.keys(r).filter(k => k && k !== 'Part Number' && k !== 'Units').length;
+  if (cols > maxColumns) {
+    sample = r;
+    maxColumns = cols;
+  }
+}
+
+// Filter to only rows with the same column structure as sample
+const sampleKeys = Object.keys(sample).filter(k => k && k !== 'Part Number' && k !== 'Units').sort();
+const compatibleRows = rows.filter(r => {
+  const rKeys = Object.keys(r).filter(k => k && k !== 'Part Number' && k !== 'Units').sort();
+  return JSON.stringify(rKeys) === JSON.stringify(sampleKeys);
+});
+
+console.log('After fix (findExcelRowsForProduct):');
+console.log(`  Filtered rows: ${compatibleRows.length}`);
+console.log(`  Sample columns: ${sampleKeys.length}`);
+console.log(`  All rows have same structure: ${compatibleRows.every(r => {
+  const keys = Object.keys(r).filter(k => k && k !== 'Part Number' && k !== 'Units');
+  return keys.length === sampleKeys.length;
+})}`);
+console.log('');
+
+// Verify the fix
+const firstFilteredRowKeys = Object.keys(compatibleRows[0]).filter(k => k && k !== 'Part Number' && k !== 'Units');
+
+console.log('✅ PASS: Filtered rows use the most complete structure');
+console.log(`   All ${compatibleRows.length} rows have ${mostColumns} columns`);
+console.log(`   First row has ${firstFilteredRowKeys.length} columns (expected ${mostColumns})`);
+
+// Verify all rows have the same structure
+const allSame = compatibleRows.every(r => {
+  const keys = Object.keys(r).filter(k => k && k !== 'Part Number' && k !== 'Units');
+  return keys.length === mostColumns;
+});
+
+if (!allSame || firstFilteredRowKeys.length !== mostColumns) {
+  console.log('❌ FAIL: Verification failed');
+  throw new Error('Verification failed');
+}
+
+console.log('\nAll checks passed!');