wip
This commit is contained in:
@@ -55,28 +55,5 @@
|
||||
"https://klz-cables.com/wp-content/uploads/2024/12/1694273920124-copy.webp": "/media/10616-1694273920124-copy.webp",
|
||||
"https://klz-cables.com/wp-content/uploads/2024/12/1694273920124-copy-2.webp": "/media/10615-1694273920124-copy-2.webp",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/02/Still-2025-02-10-104337_1.1.1.webp": "/media/45569-Still-2025-02-10-104337_1.1.1.webp",
|
||||
"https://klz-cables.com/wp-content/uploads/2024/12/DSC08036-Large.webp": "/media/10638-DSC08036-Large.webp",
|
||||
"https://wind-turbine.com/i/53689/68738caa5e58ffdf06031cf2/2/1200/630/68738c85497af_KabelfreinenWindparkpng.png": "/media/salient-1767108351351-68738c85497af_KabelfreinenWindparkpng.png",
|
||||
"https://www.nabu.de/imperia/md/nabu/images/umwelt/energie/energietraeger/windkraft/161125-nabu-windrad-allgaeu-heidrun-burchard.jpeg": "/media/salient-1767108352008-161125-nabu-windrad-allgaeu-heidrun-burchard.jpeg",
|
||||
"https://www.netze-bw.de/logo-seo.png": "/media/salient-1767108352282-logo-seo.png",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/02/og-2.webp": "/media/salient-1767108352570-og-2.webp",
|
||||
"https://images.ctfassets.net/74nz86copyef/1CDlYm1yT02sRPwG1piRUo/dc25523b67f1efc4fae65cc978f900db/hagebau_Ostendorf_Kabelschutz_Headerbild.webp": "/media/salient-1767108352701-hagebau_Ostendorf_Kabelschutz_Headerbild.webp",
|
||||
"https://www.hochspannungsblog.at/201210-netzbau-110kv-wegscheid-mast-kabelanschluss-1723.jpg": "/media/salient-1767108353235-201210-netzbau-110kv-wegscheid-mast-kabelanschluss-1723.jpg",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/01/NA2XSF2Y-3-scaled.webp": "/media/salient-1767108354000-NA2XSF2Y-3-scaled.webp",
|
||||
"https://www.klimareporter.de/images/karo3imgmanager/resized/standard-1/power-line-at-sunset-1100-733-80-ccb.webp": "/media/salient-1767108354097-power-line-at-sunset-1100-733-80-ccb.webp",
|
||||
"https://www.enercity.de/assets/cms/enercity-de/magazin/bedeutung-von-smart-grids-fuer-die-energiewende/306_460751759_1944x822_header.jpg": "/media/salient-1767108354470-306_460751759_1944x822_header.jpg",
|
||||
"https://www.e-werk-mittelbaden.de/sites/default/files/media_image/2024-12/DJI_20231105012629_0029_D-HDR.jpg": "/media/salient-1767108355559-DJI_20231105012629_0029_D-HDR.jpg",
|
||||
"https://i.ytimg.com/vi/YbtdyvQFoVM/maxresdefault.jpg": "/media/salient-1767108356326-maxresdefault.jpg",
|
||||
"https://money-for-future.com/wp-content/uploads/2022/01/Image-153-1.jpg": "/media/salient-1767108356674-Image-153-1.jpg",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/01/power-grid-station-electrical-distribution-statio-2023-11-27-05-25-36-utc-scaled.webp": "/media/salient-1767108357289-power-grid-station-electrical-distribution-statio-2023-11-27-05-25-36-utc-scaled.webp",
|
||||
"https://assets.ratedpower.com/1694509274-aerial-view-solar-panels-top-building-eco-building-factory-solar-photovoltaic-cell.jpg": "/media/salient-1767108357500-1694509274-aerial-view-solar-panels-top-building-eco-building-factory-solar-photovoltaic-cell.jpg",
|
||||
"https://assets.solarwatt.de/Resources/Persistent/e084aa09af5f0cdef386088bc558a52d81509cc0/Regenerative%20Energie-1200x628.jpg": "/media/salient-1767108358972-Regenerative%20Energie-1200x628.jpg",
|
||||
"https://www.konnworld.com/wp-content/uploads/2018/08/konn-b-logo.png": "/media/salient-1767108359266-konn-b-logo.png",
|
||||
"https://insights.regencysupply.com/hubfs/copper%20wire.jpg": "/media/salient-1767108360278-copper%20wire.jpg",
|
||||
"https://www.flukenetworks.com/sites/default/files/blog/fn-dsx-8000_14a_w.jpg": "/media/salient-1767108360607-fn-dsx-8000_14a_w.jpg",
|
||||
"https://sb-web-assets.s3.amazonaws.com/production/46426/conversions/keyart-fbimg.jpg": "/media/salient-1767108361032-keyart-fbimg.jpg",
|
||||
"https://www.erneuerbareenergien.de/sites/default/files/styles/teaser_standard__xs/public/aurora/2024/12/414535.jpeg": "/media/salient-1767108362493-414535.jpeg",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/02/header.mp4": "/media/header.mp4",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/02/header.webm": "/media/header.webm",
|
||||
"https://klz-cables.com/wp-content/uploads/2025/01/agbs.pdf": "/media/agbs.pdf"
|
||||
"https://klz-cables.com/wp-content/uploads/2024/12/DSC08036-Large.webp": "/media/10638-DSC08036-Large.webp"
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
842
scripts/process-data-with-woocommerce.js
Executable file
842
scripts/process-data-with-woocommerce.js
Executable file
@@ -0,0 +1,842 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* WordPress → Next.js Data Processing Pipeline with WooCommerce API Integration
|
||||
* Transforms raw WordPress data into Next.js compatible format with prices and variations
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
const dotenv = require('dotenv');
|
||||
|
||||
// Load environment variables from .env file
|
||||
dotenv.config();
|
||||
|
||||
const DATA_DIR = path.join(__dirname, '..', 'data');
|
||||
const RAW_DIR = path.join(DATA_DIR, 'raw');
|
||||
const PROCESSED_DIR = path.join(DATA_DIR, 'processed');
|
||||
|
||||
// Create processed directory
|
||||
if (!fs.existsSync(PROCESSED_DIR)) {
|
||||
fs.mkdirSync(PROCESSED_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// WooCommerce API Configuration from .env
|
||||
const WOOCOMMERCE_CONFIG = {
|
||||
url: process.env.WOOCOMMERCE_URL || 'https://klz-cables.com',
|
||||
consumerKey: process.env.WOOCOMMERCE_CONSUMER_KEY,
|
||||
consumerSecret: process.env.WOOCOMMERCE_CONSUMER_SECRET,
|
||||
apiVersion: 'wc/v3'
|
||||
};
|
||||
|
||||
// Debug: Check if credentials are loaded
|
||||
if (!WOOCOMMERCE_CONFIG.consumerKey || !WOOCOMMERCE_CONFIG.consumerSecret) {
|
||||
console.error('❌ Missing WooCommerce credentials in environment');
|
||||
console.error('WOOCOMMERCE_CONSUMER_KEY:', WOOCOMMERCE_CONFIG.consumerKey ? '✓ Loaded' : '❌ Missing');
|
||||
console.error('WOOCOMMERCE_CONSUMER_SECRET:', WOOCOMMERCE_CONFIG.consumerSecret ? '✓ Loaded' : '❌ Missing');
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log('✅ WooCommerce credentials loaded successfully');
|
||||
}
|
||||
|
||||
// Rate limiting configuration
|
||||
const RATE_LIMIT = {
|
||||
maxConcurrent: 2, // Max concurrent API calls
|
||||
delayBetweenCalls: 100, // ms between calls
|
||||
timeout: 30000 // 30 second timeout
|
||||
};
|
||||
|
||||
// API call queue and tracking
|
||||
let apiQueue = [];
|
||||
let activeRequests = 0;
|
||||
let apiStats = {
|
||||
total: 0,
|
||||
success: 0,
|
||||
failed: 0,
|
||||
retries: 0
|
||||
};
|
||||
|
||||
/**
|
||||
* WooCommerce API Client
|
||||
*/
|
||||
class WooCommerceAPI {
|
||||
constructor(config) {
|
||||
this.config = config;
|
||||
this.baseURL = `${config.url}/wp-json/${config.apiVersion}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make authenticated API request
|
||||
*/
|
||||
async request(endpoint, method = 'GET', data = null) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const url = new URL(`${this.baseURL}${endpoint}`);
|
||||
|
||||
// Add authentication
|
||||
const auth = Buffer.from(`${this.config.consumerKey}:${this.config.consumerSecret}`).toString('base64');
|
||||
|
||||
const options = {
|
||||
hostname: url.hostname,
|
||||
port: url.port || 443,
|
||||
path: url.pathname + url.search,
|
||||
method,
|
||||
headers: {
|
||||
'Authorization': `Basic ${auth}`,
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'KLZ-Data-Processor/1.0'
|
||||
},
|
||||
timeout: RATE_LIMIT.timeout
|
||||
};
|
||||
|
||||
// Log the request
|
||||
console.log(`🌐 API Request: ${method} ${url.pathname}`);
|
||||
|
||||
const req = https.request(options, (res) => {
|
||||
let data = '';
|
||||
|
||||
res.on('data', (chunk) => {
|
||||
data += chunk;
|
||||
});
|
||||
|
||||
res.on('end', () => {
|
||||
apiStats.total++;
|
||||
|
||||
if (res.statusCode >= 200 && res.statusCode < 300) {
|
||||
try {
|
||||
const parsed = JSON.parse(data);
|
||||
apiStats.success++;
|
||||
resolve(parsed);
|
||||
} catch (e) {
|
||||
apiStats.failed++;
|
||||
reject(new Error(`JSON parse error: ${e.message}`));
|
||||
}
|
||||
} else {
|
||||
apiStats.failed++;
|
||||
reject(new Error(`HTTP ${res.statusCode}: ${data}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', (err) => {
|
||||
apiStats.failed++;
|
||||
reject(err);
|
||||
});
|
||||
|
||||
req.on('timeout', () => {
|
||||
apiStats.failed++;
|
||||
req.destroy();
|
||||
reject(new Error('Request timeout'));
|
||||
});
|
||||
|
||||
// Add request body for POST/PUT
|
||||
if (data && (method === 'POST' || method === 'PUT')) {
|
||||
const body = JSON.stringify(data);
|
||||
req.write(body);
|
||||
}
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product by ID
|
||||
*/
|
||||
async getProduct(productId) {
|
||||
try {
|
||||
return await this.request(`/products/${productId}`);
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to fetch product ${productId}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get product variations
|
||||
*/
|
||||
async getProductVariations(productId) {
|
||||
try {
|
||||
return await this.request(`/products/${productId}/variations?per_page=100`);
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to fetch variations for product ${productId}:`, error.message);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process API queue with rate limiting
|
||||
*/
|
||||
async processQueue(tasks, progressCallback) {
|
||||
const results = [];
|
||||
|
||||
for (let i = 0; i < tasks.length; i++) {
|
||||
// Wait for rate limit
|
||||
if (i > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, RATE_LIMIT.delayBetweenCalls));
|
||||
}
|
||||
|
||||
const task = tasks[i];
|
||||
let attempt = 0;
|
||||
let success = false;
|
||||
let result = null;
|
||||
|
||||
// Retry logic
|
||||
while (attempt < 3 && !success) {
|
||||
try {
|
||||
if (progressCallback) {
|
||||
progressCallback(i + 1, tasks.length, task.label);
|
||||
}
|
||||
|
||||
result = await task.fn();
|
||||
success = true;
|
||||
|
||||
if (attempt > 0) {
|
||||
apiStats.retries++;
|
||||
console.log(`✅ Retry successful for: ${task.label}`);
|
||||
}
|
||||
} catch (error) {
|
||||
attempt++;
|
||||
if (attempt < 3) {
|
||||
console.log(`⚠️ Retry ${attempt}/3 for: ${task.label} - ${error.message}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 1000 * attempt)); // Exponential backoff
|
||||
} else {
|
||||
console.log(`❌ Failed after 3 attempts: ${task.label} - ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode HTML entities in text - comprehensive handling
|
||||
*/
|
||||
function decodeHTMLEntities(text) {
|
||||
if (!text) return '';
|
||||
|
||||
// First, handle numeric entities (decimal and hex)
|
||||
let result = text
|
||||
.replace(/&#(\d+);/g, (match, dec) => {
|
||||
const char = String.fromCharCode(parseInt(dec, 10));
|
||||
return char;
|
||||
})
|
||||
.replace(/&#x([0-9a-fA-F]+);/g, (match, hex) => {
|
||||
const char = String.fromCharCode(parseInt(hex, 16));
|
||||
return char;
|
||||
});
|
||||
|
||||
// Handle common named entities and Unicode characters
|
||||
const entityMap = {
|
||||
' ': ' ',
|
||||
'‘': "'",
|
||||
'’': "'",
|
||||
'“': '"',
|
||||
'”': '"',
|
||||
'″': '"',
|
||||
'–': '-',
|
||||
'—': '—',
|
||||
'…': '…',
|
||||
'•': '•',
|
||||
'€': '€',
|
||||
'©': '©',
|
||||
'®': '®',
|
||||
'™': '™',
|
||||
'°': '°',
|
||||
'±': '±',
|
||||
'×': '×',
|
||||
'÷': '÷',
|
||||
'−': '−',
|
||||
'¢': '¢',
|
||||
'£': '£',
|
||||
'¥': '¥',
|
||||
'§': '§',
|
||||
'¶': '¶',
|
||||
'µ': 'µ',
|
||||
'«': '«',
|
||||
'»': '»',
|
||||
'·': '·'
|
||||
};
|
||||
|
||||
// Replace all named entities
|
||||
for (const [entity, char] of Object.entries(entityMap)) {
|
||||
result = result.replace(new RegExp(entity, 'g'), char);
|
||||
}
|
||||
|
||||
// Clean up any remaining ampersand patterns
|
||||
result = result.replace(/&([a-zA-Z]+);/g, (match, name) => {
|
||||
// If it's not in our map, try to decode it or leave as is
|
||||
return entityMap[`&${name};`] || match;
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* HTML sanitization - preserve content but clean dangerous elements
|
||||
*/
|
||||
function sanitizeHTML(html) {
|
||||
if (!html) return '';
|
||||
|
||||
let sanitized = html;
|
||||
|
||||
// Remove script tags and inline handlers (security)
|
||||
sanitized = sanitized.replace(/<script.*?>.*?<\/script>/gis, '');
|
||||
sanitized = sanitized.replace(/\son\w+=".*?"/gi, '');
|
||||
|
||||
// Remove WPBakery shortcode wrappers but keep their content
|
||||
sanitized = sanitized.replace(/\[vc_row.*?\]/gi, '<div class="vc-row">');
|
||||
sanitized = sanitized.replace(/\[\/vc_row\]/gi, '</div>');
|
||||
sanitized = sanitized.replace(/\[vc_column.*?\]/gi, '<div class="vc-column">');
|
||||
sanitized = sanitized.replace(/\[\/vc_column\]/gi, '</div>');
|
||||
|
||||
// Remove other shortcodes but keep text content
|
||||
sanitized = sanitized.replace(/\[vc_column_text.*?\]/gi, '<div class="vc-text">');
|
||||
sanitized = sanitized.replace(/\[\/vc_column_text\]/gi, '</div>');
|
||||
|
||||
// Handle Nectar shortcodes
|
||||
sanitized = sanitized.replace(/\[nectar_cta.*?\]([\s\S]*?)\[\/nectar_cta\]/gi, '$1');
|
||||
sanitized = sanitized.replace(/\[nectar.*?\]/gi, '');
|
||||
|
||||
// Remove all remaining shortcodes
|
||||
sanitized = sanitized.replace(/\[.*?\]/g, '');
|
||||
|
||||
// Remove empty paragraphs and divs
|
||||
sanitized = sanitized.replace(/<p[^>]*>\s*<\/p>/gi, '');
|
||||
sanitized = sanitized.replace(/<div[^>]*>\s*<\/div>/gi, '');
|
||||
|
||||
// Normalize whitespace
|
||||
sanitized = sanitized.replace(/\s+/g, ' ').trim();
|
||||
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process products with WooCommerce API integration
|
||||
*/
|
||||
async function processProductsWithWooCommerce(productsEN, productsDE, translationMapping) {
|
||||
const api = new WooCommerceAPI(WOOCOMMERCE_CONFIG);
|
||||
const processed = [];
|
||||
|
||||
console.log(`\n🚀 Starting WooCommerce API integration for ${productsEN.length} products...`);
|
||||
|
||||
// Create tasks for price and variation fetching
|
||||
const tasks = [];
|
||||
|
||||
productsEN.forEach(product => {
|
||||
tasks.push({
|
||||
label: `${product.name} (EN) - Prices`,
|
||||
fn: async () => {
|
||||
const wooProduct = await api.getProduct(product.id);
|
||||
if (wooProduct) {
|
||||
return {
|
||||
productId: product.id,
|
||||
locale: 'en',
|
||||
regularPrice: wooProduct.regular_price || '',
|
||||
salePrice: wooProduct.sale_price || '',
|
||||
currency: wooProduct.currency || 'EUR',
|
||||
stockStatus: wooProduct.stock_status || 'instock'
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
tasks.push({
|
||||
label: `${product.name} (EN) - Variations`,
|
||||
fn: async () => {
|
||||
const variations = await api.getProductVariations(product.id);
|
||||
return {
|
||||
productId: product.id,
|
||||
locale: 'en',
|
||||
variations: variations || []
|
||||
};
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
productsDE.forEach(product => {
|
||||
tasks.push({
|
||||
label: `${product.name} (DE) - Prices`,
|
||||
fn: async () => {
|
||||
const wooProduct = await api.getProduct(product.id);
|
||||
if (wooProduct) {
|
||||
return {
|
||||
productId: product.id,
|
||||
locale: 'de',
|
||||
regularPrice: wooProduct.regular_price || '',
|
||||
salePrice: wooProduct.sale_price || '',
|
||||
currency: wooProduct.currency || 'EUR',
|
||||
stockStatus: wooProduct.stock_status || 'instock'
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
tasks.push({
|
||||
label: `${product.name} (DE) - Variations`,
|
||||
fn: async () => {
|
||||
const variations = await api.getProductVariations(product.id);
|
||||
return {
|
||||
productId: product.id,
|
||||
locale: 'de',
|
||||
variations: variations || []
|
||||
};
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Progress callback
|
||||
const progressCallback = (current, total, label) => {
|
||||
const progress = Math.round((current / total) * 100);
|
||||
process.stdout.write(`\r📊 Progress: ${current}/${total} (${progress}%) - ${label}`);
|
||||
};
|
||||
|
||||
// Process all tasks
|
||||
const results = await api.processQueue(tasks, progressCallback);
|
||||
|
||||
// Clear progress line
|
||||
process.stdout.write('\n');
|
||||
|
||||
// Organize results
|
||||
const priceData = {};
|
||||
const variationData = {};
|
||||
|
||||
results.forEach(result => {
|
||||
if (!result) return;
|
||||
|
||||
const key = `${result.productId}_${result.locale}`;
|
||||
|
||||
if (result.variations) {
|
||||
variationData[key] = result.variations;
|
||||
} else {
|
||||
priceData[key] = {
|
||||
regularPrice: result.regularPrice,
|
||||
salePrice: result.salePrice,
|
||||
currency: result.currency,
|
||||
stockStatus: result.stockStatus
|
||||
};
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\n📈 API Statistics:`);
|
||||
console.log(` Total requests: ${apiStats.total}`);
|
||||
console.log(` Successful: ${apiStats.success}`);
|
||||
console.log(` Failed: ${apiStats.failed}`);
|
||||
console.log(` Retries: ${apiStats.retries}`);
|
||||
|
||||
// Process English products
|
||||
productsEN.forEach(product => {
|
||||
const key = `${product.id}_en`;
|
||||
const priceInfo = priceData[key] || {};
|
||||
const variations = variationData[key] || [];
|
||||
|
||||
const translationKey = product.slug;
|
||||
const deMatch = translationMapping.products[translationKey];
|
||||
|
||||
processed.push({
|
||||
id: product.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'en',
|
||||
slug: product.slug,
|
||||
path: `/product/${product.slug}`,
|
||||
name: product.name,
|
||||
shortDescriptionHtml: product.shortDescriptionHtml,
|
||||
descriptionHtml: sanitizeHTML(product.descriptionHtml),
|
||||
images: product.images,
|
||||
featuredImage: product.featuredImage,
|
||||
sku: product.sku,
|
||||
regularPrice: priceInfo.regularPrice || '',
|
||||
salePrice: priceInfo.salePrice || '',
|
||||
currency: priceInfo.currency || 'EUR',
|
||||
stockStatus: priceInfo.stockStatus || 'instock',
|
||||
categories: product.categories,
|
||||
attributes: product.attributes,
|
||||
variations: variations,
|
||||
updatedAt: product.updatedAt,
|
||||
translation: deMatch ? { locale: 'de', id: deMatch.de } : null
|
||||
});
|
||||
});
|
||||
|
||||
// Process German products
|
||||
productsDE.forEach(product => {
|
||||
const key = `${product.id}_de`;
|
||||
const priceInfo = priceData[key] || {};
|
||||
const variations = variationData[key] || [];
|
||||
|
||||
const translationKey = product.slug;
|
||||
const enMatch = translationMapping.products[translationKey];
|
||||
|
||||
processed.push({
|
||||
id: product.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'de',
|
||||
slug: product.slug,
|
||||
path: `/de/product/${product.slug}`,
|
||||
name: product.name,
|
||||
shortDescriptionHtml: product.shortDescriptionHtml,
|
||||
descriptionHtml: sanitizeHTML(product.descriptionHtml),
|
||||
images: product.images,
|
||||
featuredImage: product.featuredImage,
|
||||
sku: product.sku,
|
||||
regularPrice: priceInfo.regularPrice || '',
|
||||
salePrice: priceInfo.salePrice || '',
|
||||
currency: priceInfo.currency || 'EUR',
|
||||
stockStatus: priceInfo.stockStatus || 'instock',
|
||||
categories: product.categories,
|
||||
attributes: product.attributes,
|
||||
variations: variations,
|
||||
updatedAt: product.updatedAt,
|
||||
translation: enMatch ? { locale: 'en', id: enMatch.en } : null
|
||||
});
|
||||
});
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process pages
|
||||
*/
|
||||
function processPages(pagesEN, pagesDE, translationMapping) {
|
||||
const processed = [];
|
||||
|
||||
// Process English pages
|
||||
pagesEN.forEach(page => {
|
||||
const translationKey = page.slug;
|
||||
const deMatch = translationMapping.pages[translationKey];
|
||||
|
||||
const rawTitle = page.titleHtml.replace(/<[^>]*>/g, '');
|
||||
const decodedTitle = decodeHTMLEntities(rawTitle);
|
||||
|
||||
processed.push({
|
||||
id: page.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'en',
|
||||
slug: page.slug,
|
||||
path: `/${page.slug}`,
|
||||
title: decodedTitle,
|
||||
titleHtml: page.titleHtml,
|
||||
contentHtml: sanitizeHTML(page.contentHtml),
|
||||
excerptHtml: page.excerptHtml || '',
|
||||
featuredImage: page.featuredImage,
|
||||
updatedAt: page.updatedAt,
|
||||
translation: deMatch ? { locale: 'de', id: deMatch.de } : null
|
||||
});
|
||||
});
|
||||
|
||||
// Process German pages
|
||||
pagesDE.forEach(page => {
|
||||
const translationKey = page.slug;
|
||||
const enMatch = translationMapping.pages[translationKey];
|
||||
|
||||
const rawTitle = page.titleHtml.replace(/<[^>]*>/g, '');
|
||||
const decodedTitle = decodeHTMLEntities(rawTitle);
|
||||
|
||||
processed.push({
|
||||
id: page.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'de',
|
||||
slug: page.slug,
|
||||
path: `/de/${page.slug}`,
|
||||
title: decodedTitle,
|
||||
titleHtml: page.titleHtml,
|
||||
contentHtml: sanitizeHTML(page.contentHtml),
|
||||
excerptHtml: page.excerptHtml || '',
|
||||
featuredImage: page.featuredImage,
|
||||
updatedAt: page.updatedAt,
|
||||
translation: enMatch ? { locale: 'en', id: enMatch.en } : null
|
||||
});
|
||||
});
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process posts
|
||||
*/
|
||||
function processPosts(postsEN, postsDE, translationMapping) {
|
||||
const processed = [];
|
||||
|
||||
postsEN.forEach(post => {
|
||||
const translationKey = post.slug;
|
||||
const deMatch = translationMapping.posts[translationKey];
|
||||
|
||||
const rawTitle = post.titleHtml.replace(/<[^>]*>/g, '');
|
||||
const decodedTitle = decodeHTMLEntities(rawTitle);
|
||||
|
||||
processed.push({
|
||||
id: post.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'en',
|
||||
slug: post.slug,
|
||||
path: `/blog/${post.slug}`,
|
||||
title: decodedTitle,
|
||||
titleHtml: post.titleHtml,
|
||||
contentHtml: sanitizeHTML(post.contentHtml),
|
||||
excerptHtml: post.excerptHtml || '',
|
||||
featuredImage: post.featuredImage,
|
||||
datePublished: post.datePublished,
|
||||
updatedAt: post.updatedAt,
|
||||
translation: deMatch ? { locale: 'de', id: deMatch.de } : null
|
||||
});
|
||||
});
|
||||
|
||||
postsDE.forEach(post => {
|
||||
const translationKey = post.slug;
|
||||
const enMatch = translationMapping.posts[translationKey];
|
||||
|
||||
const rawTitle = post.titleHtml.replace(/<[^>]*>/g, '');
|
||||
const decodedTitle = decodeHTMLEntities(rawTitle);
|
||||
|
||||
processed.push({
|
||||
id: post.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'de',
|
||||
slug: post.slug,
|
||||
path: `/de/blog/${post.slug}`,
|
||||
title: decodedTitle,
|
||||
titleHtml: post.titleHtml,
|
||||
contentHtml: sanitizeHTML(post.contentHtml),
|
||||
excerptHtml: post.excerptHtml || '',
|
||||
featuredImage: post.featuredImage,
|
||||
datePublished: post.datePublished,
|
||||
updatedAt: post.updatedAt,
|
||||
translation: enMatch ? { locale: 'en', id: enMatch.en } : null
|
||||
});
|
||||
});
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process product categories
|
||||
*/
|
||||
function processProductCategories(categoriesEN, categoriesDE, translationMapping) {
|
||||
const processed = [];
|
||||
|
||||
categoriesEN.forEach(category => {
|
||||
const translationKey = category.slug;
|
||||
const deMatch = translationMapping.productCategories[translationKey];
|
||||
|
||||
processed.push({
|
||||
id: category.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'en',
|
||||
slug: category.slug,
|
||||
name: category.name,
|
||||
path: `/product-category/${category.slug}`,
|
||||
description: category.description,
|
||||
count: category.count,
|
||||
translation: deMatch ? { locale: 'de', id: deMatch.de } : null
|
||||
});
|
||||
});
|
||||
|
||||
categoriesDE.forEach(category => {
|
||||
const translationKey = category.slug;
|
||||
const enMatch = translationMapping.productCategories[translationKey];
|
||||
|
||||
processed.push({
|
||||
id: category.id,
|
||||
translationKey: translationKey,
|
||||
locale: 'de',
|
||||
slug: category.slug,
|
||||
name: category.name,
|
||||
path: `/de/product-category/${category.slug}`,
|
||||
description: category.description,
|
||||
count: category.count,
|
||||
translation: enMatch ? { locale: 'en', id: enMatch.en } : null
|
||||
});
|
||||
});
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process media manifest
|
||||
*/
|
||||
function processMedia(media) {
|
||||
return media.map(item => ({
|
||||
id: item.id,
|
||||
filename: item.filename,
|
||||
url: item.url,
|
||||
localPath: `/media/${item.filename}`,
|
||||
alt: item.alt,
|
||||
width: item.width,
|
||||
height: item.height,
|
||||
mimeType: item.mime_type
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate asset map for URL replacement
|
||||
*/
|
||||
function generateAssetMap(media) {
|
||||
const map = {};
|
||||
media.forEach(item => {
|
||||
if (item.url) {
|
||||
map[item.url] = `/media/${item.filename}`;
|
||||
}
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main processing function
|
||||
*/
|
||||
async function main() {
|
||||
const exportDir = getLatestExportDir();
|
||||
console.log('🔄 Processing WordPress Data for Next.js with WooCommerce Integration');
|
||||
console.log('==========================================================\n');
|
||||
|
||||
// Load raw data
|
||||
const loadJSON = (file) => {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(exportDir, file), 'utf8'));
|
||||
} catch (e) {
|
||||
console.error(`❌ Failed to load ${file}:`, e.message);
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const translationMapping = loadJSON('translation-mapping.json');
|
||||
const pagesEN = loadJSON('pages.en.json');
|
||||
const pagesDE = loadJSON('pages.de.json');
|
||||
const postsEN = loadJSON('posts.en.json');
|
||||
const postsDE = loadJSON('posts.de.json');
|
||||
const productsEN = loadJSON('products.en.json');
|
||||
const productsDE = loadJSON('products.de.json');
|
||||
const categoriesEN = loadJSON('product-categories.en.json');
|
||||
const categoriesDE = loadJSON('product-categories.de.json');
|
||||
const media = loadJSON('media.json');
|
||||
const redirects = loadJSON('redirects.json');
|
||||
const siteInfo = loadJSON('site-info.json');
|
||||
|
||||
console.log('📊 Processing content types...\n');
|
||||
|
||||
// Process each content type
|
||||
const pages = processPages(pagesEN, pagesDE, translationMapping);
|
||||
const posts = processPosts(postsEN, postsDE, translationMapping);
|
||||
const categories = processProductCategories(categoriesEN, categoriesDE, translationMapping);
|
||||
const processedMedia = processMedia(media);
|
||||
const assetMap = generateAssetMap(media);
|
||||
|
||||
// Process products with WooCommerce API
|
||||
const products = await processProductsWithWooCommerce(productsEN, productsDE, translationMapping);
|
||||
|
||||
// Create processed data structure
|
||||
const processedData = {
|
||||
site: {
|
||||
title: siteInfo.siteTitle,
|
||||
description: siteInfo.siteDescription,
|
||||
baseUrl: siteInfo.baseUrl,
|
||||
defaultLocale: siteInfo.defaultLocale || 'en',
|
||||
locales: ['en', 'de']
|
||||
},
|
||||
content: {
|
||||
pages,
|
||||
posts,
|
||||
products,
|
||||
categories
|
||||
},
|
||||
assets: {
|
||||
media: processedMedia,
|
||||
map: assetMap
|
||||
},
|
||||
redirects,
|
||||
exportDate: new Date().toISOString()
|
||||
};
|
||||
|
||||
// Save processed data
|
||||
const outputPath = path.join(PROCESSED_DIR, 'wordpress-data.json');
|
||||
fs.writeFileSync(outputPath, JSON.stringify(processedData, null, 2));
|
||||
|
||||
// Save individual files for easier access
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'pages.json'), JSON.stringify(pages, null, 2));
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'posts.json'), JSON.stringify(posts, null, 2));
|
||||
|
||||
// Always write products.json with the processed data
|
||||
// Even if WooCommerce data is missing, we still want the base product structure
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'products.json'), JSON.stringify(products, null, 2));
|
||||
|
||||
// Report on WooCommerce data quality
|
||||
const productsWithPrices = products.filter(p => p.regularPrice).length;
|
||||
const productsWithVariations = products.filter(p => p.variations && p.variations.length > 0).length;
|
||||
|
||||
console.log('📊 WooCommerce Data Quality:');
|
||||
console.log(` Products with prices: ${productsWithPrices}/${products.length}`);
|
||||
console.log(` Products with variations: ${productsWithVariations}/${products.length}`);
|
||||
|
||||
if (productsWithPrices === 0 && productsWithVariations === 0) {
|
||||
console.log('⚠️ Warning: No WooCommerce pricing or variation data was retrieved');
|
||||
console.log(' Products written with empty price fields\n');
|
||||
} else {
|
||||
console.log('✅ WooCommerce data integrated successfully\n');
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'categories.json'), JSON.stringify(categories, null, 2));
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'media.json'), JSON.stringify(processedMedia, null, 2));
|
||||
fs.writeFileSync(path.join(PROCESSED_DIR, 'asset-map.json'), JSON.stringify(assetMap, null, 2));
|
||||
|
||||
// Summary
|
||||
console.log('\n✅ Data Processing Complete\n');
|
||||
console.log('📦 Processed Content:');
|
||||
console.log(` Pages: ${pages.length} (with translations)`);
|
||||
console.log(` Posts: ${posts.length} (with translations)`);
|
||||
console.log(` Products: ${products.length} (with translations)`);
|
||||
console.log(` Categories: ${categories.length} (with translations)`);
|
||||
console.log(` Media: ${processedMedia.length} files`);
|
||||
console.log(` Redirects: ${redirects.length} rules\n`);
|
||||
|
||||
|
||||
console.log('📁 Output Files:');
|
||||
console.log(` ${outputPath}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'pages.json')}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'posts.json')}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'products.json')}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'categories.json')}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'media.json')}`);
|
||||
console.log(` ${path.join(PROCESSED_DIR, 'asset-map.json')}\n`);
|
||||
|
||||
// Sample data
|
||||
if (products.length > 0) {
|
||||
console.log('📦 Sample Product with WooCommerce Data:');
|
||||
const sampleProduct = products.find(p => p.regularPrice) || products[0];
|
||||
console.log(` Name: ${sampleProduct.name}`);
|
||||
console.log(` SKU: ${sampleProduct.sku}`);
|
||||
console.log(` Price: ${sampleProduct.regularPrice} ${sampleProduct.currency}`);
|
||||
console.log(` Sale Price: ${sampleProduct.salePrice || 'N/A'}`);
|
||||
console.log(` Variations: ${sampleProduct.variations.length}`);
|
||||
console.log(` Locale: ${sampleProduct.locale}\n`);
|
||||
}
|
||||
|
||||
console.log('💡 Next: Ready for Next.js project setup with complete product data!');
|
||||
}
|
||||
|
||||
// Helper function to get latest export directory
|
||||
function getLatestExportDir() {
|
||||
const dirs = fs.readdirSync(RAW_DIR).filter(f => {
|
||||
const stat = fs.statSync(path.join(RAW_DIR, f));
|
||||
return stat.isDirectory();
|
||||
});
|
||||
dirs.sort().reverse();
|
||||
return path.join(RAW_DIR, dirs[0]);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
processPages,
|
||||
processPosts,
|
||||
processProductCategories,
|
||||
processProductsWithWooCommerce,
|
||||
processMedia,
|
||||
generateAssetMap,
|
||||
decodeHTMLEntities,
|
||||
sanitizeHTML
|
||||
};
|
||||
Reference in New Issue
Block a user