migration wip

test-decode5.js

@@ -0,0 +1,31 @@
+// Test decoding
+const fs = require('fs');
+
+// Read the actual file
+const content = fs.readFileSync('data/processed/pages.json', 'utf8');
+const idx = content.indexOf('bg_image=');
+const snippet = content.substring(idx, idx + 30);
+
+console.log('File has:', snippet);
+console.log('Bytes:', Buffer.from(snippet).toString('hex'));
+
+// The file has: bg_image=”10432″
+// Which is: bg_image= + ” + 10432 + ″
+
+// But wait - let me check what the actual characters are
+console.log('\nCharacter analysis of file snippet:');
+for (let i = 0; i < snippet.length; i++) {
+  const char = snippet[i];
+  const code = snippet.charCodeAt(i);
+  if (code > 127 || char === '&' || char === '#' || char === ';') {
+    console.log(i, char, code, '0x' + code.toString(16));
+  }
+}
+
+// Now test with the special characters
+const test = 'bg_image=”45569″';
+console.log('\nTest with special chars:', test);
+const decoded = test
+  .replace(/”/g, '"')
+  .replace(/″/g, '"');
+console.log('Decoded:', decoded);