diff --git a/docker-compose.yml b/docker-compose.yml index 48d555b..9b59505 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ services: - "traefik.enable=true" - "traefik.http.routers.sample-website.rule=Host(`${TRAEFIK_HOST:-sample-website.localhost}`)" - "traefik.http.services.sample-website.loadbalancer.server.port=3000" + - "caddy=http://${TRAEFIK_HOST:-acquisition.localhost}" + - "caddy.reverse_proxy={{upstreams 3000}}" directus: image: registry.infra.mintel.me/mintel/directus:${IMAGE_TAG:-latest} @@ -58,6 +60,8 @@ services: - "traefik.enable=true" - "traefik.http.routers.sample-website-directus.rule=Host(`${DIRECTUS_HOST:-cms.sample-website.localhost}`)" - "traefik.http.services.sample-website-directus.loadbalancer.server.port=8055" + - "caddy=http://${DIRECTUS_HOST:-cms.at.localhost}" + - "caddy.reverse_proxy={{upstreams 8055}}" at-mintel-directus-db: image: postgres:15-alpine diff --git a/package.json b/package.json index 081cc85..02c93a0 100644 --- a/package.json +++ b/package.json @@ -10,15 +10,16 @@ "changeset": "changeset", "version-packages": "changeset version", "sync-versions": "tsx scripts/sync-versions.ts --", - "cms:push:infra": "./scripts/sync-directus.sh push infra", - "cms:pull:infra": "./scripts/sync-directus.sh pull infra", + "cms:dev": "pnpm --filter @mintel/cms-infra dev", + "cms:up": "pnpm --filter @mintel/cms-infra up", + "cms:down": "pnpm --filter @mintel/cms-infra down", + "cms:logs": "pnpm --filter @mintel/cms-infra logs", "cms:schema:snapshot": "./scripts/cms-snapshot.sh", "cms:schema:apply": "./scripts/cms-apply.sh local", "cms:schema:apply:infra": "./scripts/cms-apply.sh infra", - "cms:up": "cd packages/cms-infra && npm run up -- --force-recreate", - "cms:down": "cd packages/cms-infra && npm run down", - "cms:logs": "cd packages/cms-infra && npm run logs", - "dev:infra": "docker-compose up -d directus directus-db", + "cms:sync:push": "./scripts/sync-directus.sh push infra", + "cms:sync:pull": "./scripts/sync-directus.sh pull infra", + "build:extensions": "./scripts/sync-extensions.sh", "release": "pnpm build && changeset publish", "release:tag": "pnpm build && pnpm -r publish --no-git-checks --access public", "prepare": "husky" @@ -63,4 +64,4 @@ "@sentry/nextjs": "10.38.0" } } -} +} \ No newline at end of file diff --git a/packages/acquisition-manager/package.json b/packages/acquisition-manager/package.json index 2a21f8d..92d5864 100644 --- a/packages/acquisition-manager/package.json +++ b/packages/acquisition-manager/package.json @@ -20,9 +20,11 @@ "build": "directus-extension build", "dev": "directus-extension build -w" }, + "dependencies": { + "@mintel/directus-extension-toolkit": "workspace:*" + }, "devDependencies": { "@directus/extensions-sdk": "11.0.2", - "@mintel/directus-extension-toolkit": "workspace:*", "vue": "^3.4.0" } -} +} \ No newline at end of file diff --git a/packages/acquisition-manager/src/module.vue b/packages/acquisition-manager/src/module.vue index 5b3c2b3..a046785 100644 --- a/packages/acquisition-manager/src/module.vue +++ b/packages/acquisition-manager/src/module.vue @@ -72,6 +72,15 @@ + + + + Kunde verlinken + import { ref, onMounted, computed } from 'vue'; import { useApi } from '@directus/extensions-sdk'; +import { useRoute, useRouter } from 'vue-router'; import { MintelManagerLayout, MintelSelect, MintelStatCard } from '@mintel/directus-extension-toolkit'; const api = useApi(); +const route = useRoute(); +const router = useRouter(); const leads = ref([]); const selectedLeadId = ref(null); const loadingAudit = ref(false); @@ -204,6 +216,7 @@ const newLead = ref({ const companies = ref([]); const people = ref([]); +const customers = ref([]); const companyOptions = computed(() => companies.value.map(c => ({ @@ -242,7 +255,7 @@ const selectedLead = computed(() => leads.value.find(l => l.id === selectedLeadI async function fetchData() { try { - const [leadsResp, peopleResp, companiesResp] = await Promise.all([ + const [leadsResp, peopleResp, companiesResp, customersResp] = await Promise.all([ api.get('/items/leads', { params: { sort: '-date_created', @@ -250,11 +263,13 @@ async function fetchData() { } }), api.get('/items/people', { params: { sort: 'last_name' } }), - api.get('/items/companies', { params: { sort: 'name' } }) + api.get('/items/companies', { params: { sort: 'name' } }), + api.get('/items/customers', { params: { fields: ['company'] } }) ]); leads.value = leadsResp.data.data; people.value = peopleResp.data.data; companies.value = companiesResp.data.data; + customers.value = customersResp.data.data; if (!selectedLeadId.value && leads.value.length > 0) { selectedLeadId.value = leads.value[0].id; @@ -264,6 +279,33 @@ async function fetchData() { } } +function isCustomer(companyId: string | any) { + if (!companyId) return false; + const id = typeof companyId === 'object' ? companyId.id : companyId; + return customers.value.some(c => (typeof c.company === 'object' ? c.company.id : c.company) === id); +} + +async function linkAsCustomer() { + if (!selectedLead.value) return; + + const companyId = selectedLead.value.company + ? (typeof selectedLead.value.company === 'object' ? selectedLead.value.company.id : selectedLead.value.company) + : null; + + const personId = selectedLead.value.contact_person + ? (typeof selectedLead.value.contact_person === 'object' ? selectedLead.value.contact_person.id : selectedLead.value.contact_person) + : null; + + router.push({ + name: 'module-customer-manager', + query: { + create: 'true', + company: companyId, + contact_person: personId + } + }); +} + async function fetchLeads() { await fetchData(); } @@ -391,7 +433,12 @@ function getStatusColor(status: string) { } } -onMounted(fetchData); +onMounted(async () => { + await fetchData(); + if (route.query.create === 'true') { + openCreateDrawer(); + } +}); `; - finalContent = finalContent.slice(0, headEnd) + stabilityCss + finalContent.slice(headEnd); - } - - const finalPath = path.join(domainDir, htmlFilename); - fs.writeFileSync(finalPath, finalContent); - return finalPath; - } finally { - await browser.close(); + page.on("response", (response) => { + if (response.status() === 200) { + const url = response.url(); + if ( + url.match( + /\.(css|js|png|jpg|jpeg|gif|svg|woff2?|ttf|otf|mp4|webm|webp|ico)/i, + ) + ) { + foundAssets.add(url); } + } + }); + + try { + await page.goto(targetUrl, { waitUntil: "networkidle", timeout: 90000 }); + + // Scroll Wave + await page.evaluate(async () => { + await new Promise((resolve) => { + let totalHeight = 0; + const distance = 400; + const timer = setInterval(() => { + const scrollHeight = document.body.scrollHeight; + window.scrollBy(0, distance); + totalHeight += distance; + if (totalHeight >= scrollHeight) { + clearInterval(timer); + window.scrollTo(0, 0); + resolve(true); + } + }, 100); + }); + }); + + const fullHeight = await page.evaluate(() => document.body.scrollHeight); + await page.setViewportSize({ width: 1920, height: fullHeight + 1000 }); + await page.waitForTimeout(3000); + + // Sanitization + await page.evaluate(() => { + const assetPattern = + /\.(jpg|jpeg|png|gif|svg|webp|mp4|webm|woff2?|ttf|otf)/i; + document.querySelectorAll("*").forEach((el) => { + if ( + ["META", "LINK", "HEAD", "SCRIPT", "STYLE", "SVG", "PATH"].includes( + el.tagName, + ) + ) + return; + const htmlEl = el as HTMLElement; + const style = window.getComputedStyle(htmlEl); + if (style.opacity === "0" || style.visibility === "hidden") { + htmlEl.style.setProperty("opacity", "1", "important"); + htmlEl.style.setProperty("visibility", "visible", "important"); + } + for (const attr of Array.from(el.attributes)) { + const name = attr.name.toLowerCase(); + const val = attr.value; + if ( + assetPattern.test(val) || + name.includes("src") || + name.includes("image") + ) { + if (el.tagName === "IMG") { + const img = el as HTMLImageElement; + if (name.includes("srcset")) img.srcset = val; + else if (!img.src || img.src.includes("data:")) img.src = val; + } + if (el.tagName === "SOURCE") + (el as HTMLSourceElement).srcset = val; + if (el.tagName === "VIDEO" || el.tagName === "AUDIO") + (el as HTMLMediaElement).src = val; + if ( + val.match(/^(https?:\/\/|\/\/|\/)/) && + !name.includes("href") + ) { + const bg = htmlEl.style.backgroundImage; + if (!bg || bg === "none") + htmlEl.style.backgroundImage = `url('${val}')`; + } + } + } + }); + if (document.body) { + document.body.style.setProperty("opacity", "1", "important"); + document.body.style.setProperty("visibility", "visible", "important"); + } + }); + + await page.waitForLoadState("networkidle"); + await page.waitForTimeout(1000); + + const content = await page.content(); + const regexPatterns = [ + /(?:src|href|url|data-[a-z-]+|srcset)=["']([^"'<>\s]+?\.(?:css|js|png|jpg|jpeg|gif|svg|woff2?|ttf|otf|mp4|webm|webp|ico)(?:\?[^"']*)?)["']/gi, + /url\(["']?([^"')]*)["']?\)/gi, + ]; + + for (const pattern of regexPatterns) { + let match; + while ((match = pattern.exec(content)) !== null) { + try { + foundAssets.add(new URL(match[1], targetUrl).href); + } catch { + // Ignore invalid URLs + } + } + } + + for (const url of foundAssets) { + const local = await this.assetManager.downloadFile(url, assetsDir); + if (local) { + urlMap[url] = local; + const clean = url.split("?")[0]; + urlMap[clean] = local; + if (clean.endsWith(".css")) { + try { + const { data } = await axios.get(url, { + headers: { "User-Agent": this.userAgent }, + }); + const processedCss = + await this.assetManager.processCssRecursively( + data, + url, + assetsDir, + urlMap, + ); + const relPath = this.assetManager.sanitizePath( + new URL(url).hostname + new URL(url).pathname, + ); + fs.writeFileSync(path.join(assetsDir, relPath), processedCss); + } catch { + // Ignore stylesheet download/process failures + } + } + } + } + + let finalContent = content; + const sortedUrls = Object.keys(urlMap).sort( + (a, b) => b.length - a.length, + ); + if (sortedUrls.length > 0) { + const escaped = sortedUrls.map((u) => + u.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), + ); + const masterRegex = new RegExp(`(${escaped.join("|")})`, "g"); + finalContent = finalContent.replace( + masterRegex, + (match) => urlMap[match] || match, + ); + } + + const commonDirs = [ + "/wp-content/", + "/wp-includes/", + "/assets/", + "/static/", + "/images/", + ]; + for (const dir of commonDirs) { + const localDir = `./assets/${urlObj.hostname}${dir}`; + finalContent = finalContent + .split(`"${dir}`) + .join(`"${localDir}`) + .split(`'${dir}`) + .join(`'${localDir}`) + .split(`(${dir}`) + .join(`(${localDir}`); + } + + const domainPattern = new RegExp( + `https?://(www\\.)?${urlObj.hostname.replace(/\./g, "\\.")}[^"']*`, + "gi", + ); + finalContent = finalContent.replace(domainPattern, () => "./"); + + finalContent = finalContent.replace( + /]*>([\s\S]*?)<\/script>/gi, + (match, scriptContent) => { + const lower = scriptContent.toLowerCase(); + return lower.includes("google-analytics") || + lower.includes("gtag") || + lower.includes("fbq") || + lower.includes("lazy") || + lower.includes("tracker") + ? "" + : match; + }, + ); + + const headEnd = finalContent.indexOf(""); + if (headEnd > -1) { + const stabilityCss = `\n`; + finalContent = + finalContent.slice(0, headEnd) + + stabilityCss + + finalContent.slice(headEnd); + } + + const finalPath = path.join(domainDir, htmlFilename); + fs.writeFileSync(finalPath, finalContent); + return finalPath; + } finally { + await browser.close(); } + } } diff --git a/packages/cloner-library/src/WebsiteCloner.ts b/packages/cloner-library/src/WebsiteCloner.ts index d39a79b..c274242 100644 --- a/packages/cloner-library/src/WebsiteCloner.ts +++ b/packages/cloner-library/src/WebsiteCloner.ts @@ -1,123 +1,150 @@ -import { PlaywrightCrawler, RequestQueue } from 'crawlee'; -import * as path from 'node:path'; -import * as fs from 'node:fs'; -import { execSync } from 'node:child_process'; +import { PlaywrightCrawler, RequestQueue } from "crawlee"; +import * as path from "node:path"; +import * as fs from "node:fs"; +import { execSync } from "node:child_process"; export interface WebsiteClonerOptions { - baseOutputDir: string; - maxRequestsPerCrawl?: number; - maxConcurrency?: number; + baseOutputDir: string; + maxRequestsPerCrawl?: number; + maxConcurrency?: number; } export class WebsiteCloner { - private options: WebsiteClonerOptions; + private options: WebsiteClonerOptions; - constructor(options: WebsiteClonerOptions) { - this.options = { - maxRequestsPerCrawl: 100, - maxConcurrency: 3, - ...options - }; + constructor(options: WebsiteClonerOptions) { + this.options = { + maxRequestsPerCrawl: 100, + maxConcurrency: 3, + ...options, + }; + } + + public async clone( + targetUrl: string, + outputDirName?: string, + ): Promise { + const urlObj = new URL(targetUrl); + const domain = urlObj.hostname; + const finalOutputDirName = outputDirName || domain.replace(/\./g, "-"); + const baseOutputDir = path.resolve( + this.options.baseOutputDir, + finalOutputDirName, + ); + + if (fs.existsSync(baseOutputDir)) { + fs.rmSync(baseOutputDir, { recursive: true, force: true }); } + fs.mkdirSync(baseOutputDir, { recursive: true }); - public async clone(targetUrl: string, outputDirName?: string): Promise { - const urlObj = new URL(targetUrl); - const domain = urlObj.hostname; - const finalOutputDirName = outputDirName || domain.replace(/\./g, '-'); - const baseOutputDir = path.resolve(this.options.baseOutputDir, finalOutputDirName); + console.log(`🚀 Starting perfect recursive clone of ${targetUrl}...`); + console.log(`📂 Output: ${baseOutputDir}`); - if (fs.existsSync(baseOutputDir)) { - fs.rmSync(baseOutputDir, { recursive: true, force: true }); - } - fs.mkdirSync(baseOutputDir, { recursive: true }); + const requestQueue = await RequestQueue.open(); + await requestQueue.addRequest({ url: targetUrl }); - console.log(`🚀 Starting perfect recursive clone of ${targetUrl}...`); - console.log(`📂 Output: ${baseOutputDir}`); + const crawler = new PlaywrightCrawler({ + requestQueue, + maxRequestsPerCrawl: this.options.maxRequestsPerCrawl, + maxConcurrency: this.options.maxConcurrency, - const requestQueue = await RequestQueue.open(); - await requestQueue.addRequest({ url: targetUrl }); + async requestHandler({ request, enqueueLinks, log }) { + const url = request.url; + log.info(`Capturing ${url}...`); - const crawler = new PlaywrightCrawler({ - requestQueue, - maxRequestsPerCrawl: this.options.maxRequestsPerCrawl, - maxConcurrency: this.options.maxConcurrency, + const u = new URL(url); + let relPath = u.pathname; + if (relPath === "/" || relPath === "") relPath = "/index.html"; + if (!relPath.endsWith(".html") && !path.extname(relPath)) + relPath += "/index.html"; + if (relPath.startsWith("/")) relPath = relPath.substring(1); - async requestHandler({ request, enqueueLinks, log }) { - const url = request.url; - log.info(`Capturing ${url}...`); + const fullPath = path.join(baseOutputDir, relPath); + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - const u = new URL(url); - let relPath = u.pathname; - if (relPath === '/' || relPath === '') relPath = '/index.html'; - if (!relPath.endsWith('.html') && !path.extname(relPath)) relPath += '/index.html'; - if (relPath.startsWith('/')) relPath = relPath.substring(1); - - const fullPath = path.join(baseOutputDir, relPath); - fs.mkdirSync(path.dirname(fullPath), { recursive: true }); - - try { - // Note: This assumes single-file-cli is available in the environment - execSync(`npx single-file-cli "${url}" "${fullPath}" --browser-headless=true --browser-wait-until=networkidle0`, { - stdio: 'inherit' - }); - } catch (e) { - log.error(`Failed to capture ${url} with SingleFile`); - } - - await enqueueLinks({ - strategy: 'same-domain', - transformRequestFunction: (req) => { - if (/\.(download|pdf|zip|gz|exe|png|jpg|jpeg|gif|svg|css|js)$/i.test(req.url)) return false; - return req; - } - }); + try { + // Note: This assumes single-file-cli is available in the environment + execSync( + `npx single-file-cli "${url}" "${fullPath}" --browser-headless=true --browser-wait-until=networkidle0`, + { + stdio: "inherit", }, + ); + } catch (_e) { + log.error(`Failed to capture ${url} with SingleFile`); + } + + await enqueueLinks({ + strategy: "same-domain", + transformRequestFunction: (req) => { + if ( + /\.(download|pdf|zip|gz|exe|png|jpg|jpeg|gif|svg|css|js)$/i.test( + req.url, + ) + ) + return false; + return req; + }, }); + }, + }); - await crawler.run(); + await crawler.run(); - console.log('🔗 Rewriting internal links for offline navigation...'); - const allFiles = this.getFiles(baseOutputDir).filter(f => f.endsWith('.html')); + console.log("🔗 Rewriting internal links for offline navigation..."); + const allFiles = this.getFiles(baseOutputDir).filter((f) => + f.endsWith(".html"), + ); - for (const file of allFiles) { - let content = fs.readFileSync(file, 'utf8'); - const fileRelToRoot = path.relative(baseOutputDir, file); + for (const file of allFiles) { + let content = fs.readFileSync(file, "utf8"); + const fileRelToRoot = path.relative(baseOutputDir, file); - content = content.replace(/href="([^"]+)"/g, (match, href) => { - if (href.startsWith(targetUrl) || href.startsWith('/') || (!href.includes('://') && !href.startsWith('data:'))) { - try { - const linkUrl = new URL(href, targetUrl); - if (linkUrl.hostname === domain) { - let linkPath = linkUrl.pathname; - if (linkPath === '/' || linkPath === '') linkPath = '/index.html'; - if (!linkPath.endsWith('.html') && !path.extname(linkPath)) linkPath += '/index.html'; - if (linkPath.startsWith('/')) linkPath = linkPath.substring(1); + content = content.replace(/href="([^"]+)"/g, (match, href) => { + if ( + href.startsWith(targetUrl) || + href.startsWith("/") || + (!href.includes("://") && !href.startsWith("data:")) + ) { + try { + const linkUrl = new URL(href, targetUrl); + if (linkUrl.hostname === domain) { + let linkPath = linkUrl.pathname; + if (linkPath === "/" || linkPath === "") linkPath = "/index.html"; + if (!linkPath.endsWith(".html") && !path.extname(linkPath)) + linkPath += "/index.html"; + if (linkPath.startsWith("/")) linkPath = linkPath.substring(1); - const relativeLink = path.relative(path.dirname(fileRelToRoot), linkPath); - return `href="${relativeLink}"`; - } - } catch (e) { } - } - return match; - }); - - fs.writeFileSync(file, content); - } - - console.log(`\n✅ Done! Perfect clone complete in: ${baseOutputDir}`); - return baseOutputDir; - } - - private getFiles(dir: string, fileList: string[] = []) { - const files = fs.readdirSync(dir); - for (const file of files) { - const name = path.join(dir, file); - if (fs.statSync(name).isDirectory()) { - this.getFiles(name, fileList); - } else { - fileList.push(name); + const relativeLink = path.relative( + path.dirname(fileRelToRoot), + linkPath, + ); + return `href="${relativeLink}"`; } + } catch (_e) { + // Ignore link rewriting failures + } } - return fileList; + return match; + }); + + fs.writeFileSync(file, content); } + + console.log(`\n✅ Done! Perfect clone complete in: ${baseOutputDir}`); + return baseOutputDir; + } + + private getFiles(dir: string, fileList: string[] = []) { + const files = fs.readdirSync(dir); + for (const file of files) { + const name = path.join(dir, file); + if (fs.statSync(name).isDirectory()) { + this.getFiles(name, fileList); + } else { + fileList.push(name); + } + } + return fileList; + } } diff --git a/packages/cms-infra/database/data.db b/packages/cms-infra/database/data.db index f0b07e7..6e91a91 100644 Binary files a/packages/cms-infra/database/data.db and b/packages/cms-infra/database/data.db differ diff --git a/packages/cms-infra/docker-compose.yml b/packages/cms-infra/docker-compose.yml index d9d2686..78ca640 100644 --- a/packages/cms-infra/docker-compose.yml +++ b/packages/cms-infra/docker-compose.yml @@ -39,9 +39,9 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.at-mintel-infra-cms.rule=Host(`cms.localhost`)" - - "traefik.http.services.at-mintel-infra-cms.loadbalancer.server.port=8055" - - "traefik.http.services.at-mintel-infra-cms.loadbalancer.healthcheck.path=/server/health" - "traefik.docker.network=infra" + - "caddy=cms.localhost" + - "caddy.reverse_proxy={{upstreams 8055}}" networks: default: diff --git a/packages/cms-infra/package.json b/packages/cms-infra/package.json index dfac65d..f003906 100644 --- a/packages/cms-infra/package.json +++ b/packages/cms-infra/package.json @@ -4,7 +4,8 @@ "private": true, "type": "module", "scripts": { - "up": "npm run build:extensions && docker compose up -d", + "dev": "npm run up -- --link", + "up": "../../scripts/cms-up.sh", "down": "docker compose down", "logs": "docker compose logs -f", "build:extensions": "../../scripts/sync-extensions.sh", @@ -14,4 +15,4 @@ "sync:push": "../../scripts/sync-directus.sh push infra", "sync:pull": "../../scripts/sync-directus.sh pull infra" } -} +} \ No newline at end of file diff --git a/packages/company-manager/package.json b/packages/company-manager/package.json index a9fecb3..2f098da 100644 --- a/packages/company-manager/package.json +++ b/packages/company-manager/package.json @@ -20,9 +20,11 @@ "build": "directus-extension build", "dev": "directus-extension build -w" }, + "dependencies": { + "@mintel/directus-extension-toolkit": "workspace:*" + }, "devDependencies": { "@directus/extensions-sdk": "11.0.2", - "@mintel/directus-extension-toolkit": "workspace:*", "vue": "^3.4.0" } -} +} \ No newline at end of file diff --git a/packages/company-manager/src/module.vue b/packages/company-manager/src/module.vue index a067f6b..2dfeeb2 100644 --- a/packages/company-manager/src/module.vue +++ b/packages/company-manager/src/module.vue @@ -104,9 +104,11 @@