clones

scripts/clone-recursive.ts

@@ -0,0 +1,210 @@
+// @ts-ignore
+import scrape from 'website-scraper';
+// @ts-ignore
+import PuppeteerPlugin from 'website-scraper-puppeteer';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import fs from 'node:fs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+class CustomFilenameGeneratorPlugin {
+    apply(registerAction: any) {
+        registerAction('generateFilename', ({ resource }: any) => {
+            const url = new URL(resource.url);
+            const ext = path.extname(url.pathname);
+
+            // Clean the path
+            let safePath = url.pathname;
+            if (safePath.endsWith('/')) {
+                safePath += 'index.html';
+            } else if (!ext && !resource.isHtml()) {
+                // If no extension and not HTML, guess based on content type? 
+                // But usually safe to leave as is or add extension if known.
+            } else if (!ext && resource.isHtml()) {
+                safePath += '.html';
+            }
+
+            // Handle query strings if needed (simplifying by ignoring them for static local files usually better, 
+            // unless they determine content. For a clean clone, we usually ignore unique query params)
+            // But if the site relies on routing via query params (e.g. ?page=2), we might want to encode them.
+            // For now, let's keep it simple and clean.
+
+            // Remove leading slash
+            if (safePath.startsWith('/')) safePath = safePath.substring(1);
+
+            // Sanitization
+            safePath = safePath.replace(/[:*?"<>|]/g, '_');
+
+            // External assets go to a separate folder to avoid collision
+            // We can detect external by checking if the resource parent is different?
+            // Actually, simply using the hostname mapping is safer.
+
+            // However, the USER wants "local cloned pages".
+            // If we just use the path, we merge everything into one root.
+            // If there are collision (e.g. same path on different domains), this is bad.
+            // But typically we clone ONE site.
+
+            return { filename: safePath };
+        });
+    }
+}
+
+async function run() {
+    const targetUrl = process.argv[2];
+    if (!targetUrl) {
+        console.error('Usage: npm run clone-website <URL> [output-dir]');
+        process.exit(1);
+    }
+
+    const urlObj = new URL(targetUrl);
+    const domain = urlObj.hostname;
+    const safeDomain = domain.replace(/[^a-z0-9-]/gi, '_');
+    const outputDir = process.argv[3]
+        ? path.resolve(process.cwd(), process.argv[3])
+        : path.resolve(__dirname, '../cloned-websites', safeDomain);
+
+    if (fs.existsSync(outputDir)) {
+        console.log(`Cleaning existing directory: ${outputDir}`);
+        fs.rmSync(outputDir, { recursive: true, force: true });
+    }
+
+    console.log(`🚀 Starting recursive clone of ${targetUrl}`);
+    console.log(`📂 Output: ${outputDir}`);
+
+    const options = {
+        urls: [targetUrl],
+        directory: outputDir,
+        recursive: true,
+        maxDepth: 10,
+        // Custom filename generation to avoid "https:/" folders
+        // We use 'bySiteStructure' behavior but manually controlled via plugin 
+        // to forcefully strip protocol/domain issues if any.
+        // Actually, let's just use 'bySiteStructure' but strictly configured? 
+        // No, the user saw garbage. Let's use 'byType' combined with preserving structure for HTML.
+
+        // BETTER STRATEGY: 
+        // Use a custom plugin to control filenames EXACTLY how we want.
+        plugins: [
+            new PuppeteerPlugin({
+                launchOptions: {
+                    headless: true,
+                    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
+                },
+                scrollToBottom: { timeout: 10000, viewportN: 10 },
+                blockNavigation: false
+            }),
+            new class FilenamePlugin {
+                apply(registerAction: any) {
+                    registerAction('generateFilename', ({ resource }: any) => {
+                        const u = new URL(resource.url);
+                        let filename = u.pathname;
+
+                        // normalize
+                        if (filename.endsWith('/')) filename += 'index.html';
+                        else if (!path.extname(filename) && resource.url.includes(domain)) filename += '/index.html'; // Assume folder if internal link without ext
+
+                        // If it's an external asset, put it in a separate folder
+                        if (u.hostname !== domain) {
+                            filename = `_external/${u.hostname}${filename}`;
+                        }
+
+                        // Sanitize filename
+                        filename = filename.split('/').map(part => part.replace(/[^a-z0-9._-]/gi, '_')).join('/');
+
+                        // Remove leading slash
+                        if (filename.startsWith('/')) filename = filename.substring(1);
+
+                        // Handle "Unnamed page" by checking if empty
+                        if (!filename || filename === 'index.html') return { filename: 'index.html' };
+
+                        return { filename };
+                    });
+                }
+            }
+        ],
+
+        urlFilter: (url: string) => {
+            const u = new URL(url);
+            const isTargetDomain = u.hostname === domain;
+            const isGoogleFonts = u.hostname.includes('fonts.googleapis.com') || u.hostname.includes('fonts.gstatic.com');
+            // Allow assets from anywhere
+            const isAsset = /\.(css|js|png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot|mp4|webm|ico|json)$/i.test(u.pathname);
+            // Allow fonts/css from common CDNs if standard extension check fails
+            const isCommonAsset = u.pathname.includes('/css/') || u.pathname.includes('/js/') || u.pathname.includes('/static/') || u.pathname.includes('/assets/');
+
+            return isTargetDomain || isAsset || isCommonAsset || isGoogleFonts;
+        },
+
+
+        sources: [
+            { selector: 'img', attr: 'src' },
+            { selector: 'img', attr: 'srcset' },
+            { selector: 'source', attr: 'src' },
+            { selector: 'source', attr: 'srcset' },
+            { selector: 'link[rel="stylesheet"]', attr: 'href' },
+            { selector: 'script', attr: 'src' },
+            { selector: 'video', attr: 'src' },
+            { selector: 'video', attr: 'poster' },
+            { selector: 'iframe', attr: 'src' },
+            { selector: 'link[rel*="icon"]', attr: 'href' },
+            { selector: 'link[rel="manifest"]', attr: 'href' },
+            { selector: 'meta[property="og:image"]', attr: 'content' }
+        ],
+
+        request: {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36'
+            }
+        }
+    };
+
+    try {
+        // @ts-ignore
+        const result = await scrape(options);
+        console.log(`\n✅ Successfully cloned ${result.length} resources to ${outputDir}`);
+
+        // Post-processing: Sanitize HTML to remove Next.js hydration scripts
+        // This prevents the static site from trying to "hydrate" and breaking images/links
+        console.log('🧹 Sanitizing HTML files...');
+        sanitizeHtmlFiles(outputDir);
+
+        console.log(`open "${path.join(outputDir, 'index.html')}"`);
+    } catch (error) {
+        console.error('❌ Error cloning website:', error);
+        process.exit(1);
+    }
+}
+
+function sanitizeHtmlFiles(dir: string) {
+    const files = fs.readdirSync(dir);
+    for (const file of files) {
+        const fullPath = path.join(dir, file);
+        if (fs.statSync(fullPath).isDirectory()) {
+            sanitizeHtmlFiles(fullPath);
+        } else if (file.endsWith('.html')) {
+            let content = fs.readFileSync(fullPath, 'utf8');
+
+            // Remove Next.js data script
+            content = content.replace(/<script id="__NEXT_DATA__"[\s\S]*?<\/script>/gi, '');
+
+            // Remove Next.js chunk scripts (hydration)
+            // match <script src="..._next/static/chunks..." ...
+            content = content.replace(/<script[^>]+src="[^"]*\/_next\/static\/chunks\/[^"]*"[^>]*><\/script>/gi, '');
+            content = content.replace(/<script[^>]+src="[^"]*\/_next\/static\/[^"]*Manifest\.js"[^>]*><\/script>/gi, '');
+
+            // Inject Fonts (Fix for missing dynamic fonts)
+            // We inject Inter as a safe default for modern Next.js/Tailwind sites if strictly missing
+            if (!content.includes('fonts.googleapis.com')) {
+                const fontLink = `<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap">`;
+                const styleBlock = `<style>.body-font{font-family:'Inter',sans-serif;}.title-font{font-family:'Inter',sans-serif;}</style>`;
+                content = content.replace('</head>', `${fontLink}${styleBlock}</head>`);
+            }
+
+            fs.writeFileSync(fullPath, content);
+        }
+    }
+}
+
+run();