crawler

package.json

@@ -11,7 +11,8 @@
     "test": "npm run test:smoke",
     "test:smoke": "tsx ./scripts/smoke-test.ts",
     "test:links": "tsx ./scripts/test-links.ts",
-    "test:file-examples": "tsx ./scripts/test-file-examples-comprehensive.ts"
+    "test:file-examples": "tsx ./scripts/test-file-examples-comprehensive.ts",
+    "clone-website": "tsx ./scripts/clone-website.ts"
   },
   "dependencies": {
     "@react-pdf/renderer": "^4.3.2",
@@ -28,12 +29,15 @@
     "mermaid": "^11.12.2",
     "next": "^16.1.6",
     "prismjs": "^1.30.0",
+    "puppeteer": "^24.36.1",
     "qrcode": "^1.5.4",
     "react": "^19.2.3",
     "react-dom": "^19.2.3",
     "shiki": "^1.24.2",
     "tailwind-merge": "^3.4.0",
-    "tailwindcss": "^3.4.0"
+    "tailwindcss": "^3.4.0",
+    "website-scraper": "^6.0.0",
+    "website-scraper-puppeteer": "^2.0.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.15",