fix(pdf): decouple 6 distinct PDFs, fix layout issues and DataForSEO event loop
Some checks failed
Monorepo Pipeline / ⚡ Prioritize Release (push) Successful in 1s
Monorepo Pipeline / 🧹 Lint (push) Failing after 1m1s
Monorepo Pipeline / 🧪 Test (push) Failing after 1m7s
Monorepo Pipeline / 🏗️ Build (push) Failing after 1m10s
Monorepo Pipeline / 🚀 Release (push) Has been skipped
Monorepo Pipeline / 🐳 Build Image Processor (push) Has been skipped
Monorepo Pipeline / 🐳 Build Directus (Base) (push) Has been skipped
Monorepo Pipeline / 🐳 Build Gatekeeper (Product) (push) Has been skipped
Monorepo Pipeline / 🐳 Build Build-Base (push) Has been skipped
Monorepo Pipeline / 🐳 Build Production Runtime (push) Has been skipped

This commit is contained in:
2026-02-27 18:26:00 +01:00
parent 60a2709999
commit a43d96dd0e
12 changed files with 187 additions and 102 deletions

View File

@@ -1,9 +1,8 @@
// ============================================================================ // ============================================================================
// Scraper — Zyte API + Local Persistence // Scraper — Zyte API + Local Persistence
// Crawls all pages of a website, stores them locally for reuse. // Crawls all pages of a website, stores them locally for reuse.
// Crawls all pages of a website, stores them locally for reuse.
// ============================================================================ // ============================================================================
import axios from "axios";
import * as cheerio from "cheerio"; import * as cheerio from "cheerio";
import * as fs from "node:fs/promises"; import * as fs from "node:fs/promises";
import * as path from "node:path"; import * as path from "node:path";
@@ -171,32 +170,39 @@ function extractServices(text: string): string[] {
*/ */
async function fetchWithZyte(url: string, apiKey: string): Promise<string> { async function fetchWithZyte(url: string, apiKey: string): Promise<string> {
try { try {
const resp = await axios.post( const auth = Buffer.from(`${apiKey}:`).toString("base64");
"https://api.zyte.com/v1/extract", const resp = await fetch("https://api.zyte.com/v1/extract", {
{ method: "POST",
headers: {
"Authorization": `Basic ${auth}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
url, url,
browserHtml: true, browserHtml: true,
}, }),
{ signal: AbortSignal.timeout(60000),
auth: { username: apiKey, password: "" }, });
timeout: 60000,
}, if (!resp.ok) {
); const errorText = await resp.text();
const html = resp.data.browserHtml || ""; console.error(` ❌ Zyte API error ${resp.status} for ${url}: ${errorText}`);
// Rate limited — wait and retry once
if (resp.status === 429) {
console.log(" ⏳ Rate limited, waiting 5s and retrying...");
await new Promise((r) => setTimeout(r, 5000));
return fetchWithZyte(url, apiKey);
}
throw new Error(`HTTP ${resp.status}: ${errorText}`);
}
const data = await resp.json();
const html = data.browserHtml || "";
if (!html) { if (!html) {
console.warn(` ⚠️ Zyte returned empty browserHtml for ${url}`); console.warn(` ⚠️ Zyte returned empty browserHtml for ${url}`);
} }
return html; return html;
} catch (err: any) { } catch (err: any) {
if (err.response) {
console.error(` ❌ Zyte API error ${err.response.status} for ${url}: ${err.response.data?.detail || err.response.statusText}`);
// Rate limited — wait and retry once
if (err.response.status === 429) {
console.log(" ⏳ Rate limited, waiting 5s and retrying...");
await new Promise((r) => setTimeout(r, 5000));
return fetchWithZyte(url, apiKey);
}
}
throw err; throw err;
} }
} }
@@ -205,14 +211,19 @@ async function fetchWithZyte(url: string, apiKey: string): Promise<string> {
* Fetch a page via simple HTTP GET (fallback). * Fetch a page via simple HTTP GET (fallback).
*/ */
async function fetchDirect(url: string): Promise<string> { async function fetchDirect(url: string): Promise<string> {
const resp = await axios.get(url, { try {
timeout: 30000, const resp = await fetch(url, {
headers: { headers: {
"User-Agent": "User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
}, },
}); signal: AbortSignal.timeout(30000),
return typeof resp.data === "string" ? resp.data : ""; });
if (!resp.ok) return "";
return await resp.text();
} catch {
return "";
}
} }
/** /**

View File

@@ -83,24 +83,31 @@ export class DataForSeoClient {
let delay = 15_000; let delay = 15_000;
let pollCount = 0; let pollCount = 0;
while (Date.now() - start < timeoutMs) { // Force Node event loop active
await this.sleep(delay); const keepAlive = setInterval(() => { }, 1000);
pollCount++;
const ready = await this.isTaskReady(taskId); try {
const elapsed = Math.round((Date.now() - start) / 1000); while (Date.now() - start < timeoutMs) {
console.log(` 📊 Poll #${pollCount}: ${ready ? "READY ✅" : "not ready"} (${elapsed}s elapsed)`); await this.sleep(delay);
pollCount++;
if (ready) { const ready = await this.isTaskReady(taskId);
// Short grace period so the pages endpoint settles const elapsed = Math.round((Date.now() - start) / 1000);
await this.sleep(5_000); console.log(` 📊 Poll #${pollCount}: ${ready ? "READY ✅" : "not ready"} (${elapsed}s elapsed)`);
return;
if (ready) {
// Short grace period so the pages endpoint settles
await this.sleep(5_000);
return;
}
delay = Math.min(delay * 1.3, 30_000);
} }
delay = Math.min(delay * 1.3, 30_000); throw new Error(`DataForSEO task ${taskId} timed out after ${timeoutMs / 1000}s`);
} finally {
clearInterval(keepAlive);
} }
throw new Error(`DataForSEO task ${taskId} timed out after ${timeoutMs / 1000}s`);
} }
/** /**

View File

@@ -6,6 +6,7 @@ import {
Text as PDFText, Text as PDFText,
View as PDFView, View as PDFView,
StyleSheet as PDFStyleSheet, StyleSheet as PDFStyleSheet,
Document as PDFDocument,
} from "@react-pdf/renderer"; } from "@react-pdf/renderer";
import { import {
pdfStyles, pdfStyles,
@@ -213,30 +214,34 @@ export const AgbsPDF = ({
if (mode === "full") { if (mode === "full") {
return ( return (
<SimpleLayout <PDFDocument title="Allgemeine Geschäftsbedingungen">
companyData={companyData} <SimpleLayout
bankData={bankData} companyData={companyData}
headerIcon={headerIcon} bankData={bankData}
footerLogo={footerLogo} headerIcon={headerIcon}
showPageNumber={false} footerLogo={footerLogo}
> showPageNumber={false}
{content} >
</SimpleLayout> {content}
</SimpleLayout>
</PDFDocument>
); );
} }
return ( return (
<PDFPage size="A4" style={pdfStyles.page}> <PDFDocument title="Allgemeine Geschäftsbedingungen">
<FoldingMarks /> <PDFPage size="A4" style={pdfStyles.page}>
<Header icon={headerIcon} showAddress={false} /> <FoldingMarks />
{content} <Header icon={headerIcon} showAddress={false} />
<Footer {content}
logo={footerLogo} <Footer
companyData={companyData} logo={footerLogo}
_bankData={bankData} companyData={companyData}
showDetails={false} bankData={bankData}
showPageNumber={false} showDetails={false}
/> showPageNumber={false}
</PDFPage> />
</PDFPage>
</PDFDocument>
); );
}; };

View File

@@ -0,0 +1,36 @@
"use client";
import * as React from "react";
import { Document as PDFDocument } from "@react-pdf/renderer";
import { SimpleLayout } from "./pdf/SimpleLayout.js";
import { ClosingModule } from "./pdf/modules/CommonModules.js";
export const ClosingPDF = ({ headerIcon, footerLogo }: any) => {
const date = new Date().toLocaleDateString("de-DE", {
year: "numeric",
month: "long",
day: "numeric",
});
const companyData = {
name: "Marc Mintel",
address1: "Georg-Meistermann-Straße 7",
address2: "54586 Schüller",
ustId: "DE367588065",
};
const commonProps = {
date,
headerIcon,
footerLogo,
companyData,
};
return (
<PDFDocument title="Abschluss">
<SimpleLayout {...commonProps}>
<ClosingModule />
</SimpleLayout>
</PDFDocument>
);
};

View File

@@ -5,17 +5,12 @@ import { Page as PDFPage, Document as PDFDocument } from "@react-pdf/renderer";
import { pdfStyles } from "./pdf/SharedUI.js"; import { pdfStyles } from "./pdf/SharedUI.js";
import { SimpleLayout } from "./pdf/SimpleLayout.js"; import { SimpleLayout } from "./pdf/SimpleLayout.js";
// Modules
import { FrontPageModule } from "./pdf/modules/FrontPageModule.js";
import { BriefingModule } from "./pdf/modules/BriefingModule.js"; import { BriefingModule } from "./pdf/modules/BriefingModule.js";
import { SitemapModule } from "./pdf/modules/SitemapModule.js"; import { SitemapModule } from "./pdf/modules/SitemapModule.js";
import { ClosingModule } from "./pdf/modules/CommonModules.js";
export const ConceptPDF = ({ export const ConceptPDF = (props: any) => {
concept, const { concept, headerIcon, footerLogo } = props;
headerIcon,
footerLogo,
}: any) => {
const date = new Date().toLocaleDateString("de-DE", { const date = new Date().toLocaleDateString("de-DE", {
year: "numeric", year: "numeric",
month: "long", month: "long",
@@ -48,10 +43,6 @@ export const ConceptPDF = ({
return ( return (
<PDFDocument title={`Projektkonzept - ${flatState.companyName || "Projekt"}`}> <PDFDocument title={`Projektkonzept - ${flatState.companyName || "Projekt"}`}>
<PDFPage size="A4" style={pdfStyles.titlePage}>
<FrontPageModule state={flatState} headerIcon={headerIcon} date={date} />
</PDFPage>
<SimpleLayout {...commonProps}> <SimpleLayout {...commonProps}>
<BriefingModule state={flatState} /> <BriefingModule state={flatState} />
</SimpleLayout> </SimpleLayout>
@@ -61,10 +52,6 @@ export const ConceptPDF = ({
<SitemapModule state={flatState} /> <SitemapModule state={flatState} />
</SimpleLayout> </SimpleLayout>
)} )}
<SimpleLayout {...commonProps}>
<ClosingModule />
</SimpleLayout>
</PDFDocument> </PDFDocument>
); );
}; };

View File

@@ -6,10 +6,7 @@ import { pdfStyles } from "./pdf/SharedUI.js";
import { SimpleLayout } from "./pdf/SimpleLayout.js"; import { SimpleLayout } from "./pdf/SimpleLayout.js";
// Modules // Modules
import { FrontPageModule } from "./pdf/modules/FrontPageModule.js";
import { EstimationModule } from "./pdf/modules/EstimationModule.js"; import { EstimationModule } from "./pdf/modules/EstimationModule.js";
import { TransparenzModule } from "./pdf/modules/TransparenzModule.js";
import { ClosingModule } from "./pdf/modules/CommonModules.js";
import { calculatePositions } from "../logic/pricing/calculator.js"; import { calculatePositions } from "../logic/pricing/calculator.js";
@@ -58,10 +55,6 @@ export const EstimationPDF = ({
return ( return (
<PDFDocument title={`Angebot - ${state.companyName || "Projekt"}`}> <PDFDocument title={`Angebot - ${state.companyName || "Projekt"}`}>
<PDFPage size="A4" style={pdfStyles.titlePage}>
<FrontPageModule state={state} headerIcon={headerIcon} date={date} />
</PDFPage>
<SimpleLayout {...commonProps}> <SimpleLayout {...commonProps}>
<EstimationModule <EstimationModule
state={state} state={state}
@@ -70,14 +63,6 @@ export const EstimationPDF = ({
date={date} date={date}
/> />
</SimpleLayout> </SimpleLayout>
<SimpleLayout {...commonProps}>
<TransparenzModule pricing={pricing} />
</SimpleLayout>
<SimpleLayout {...commonProps}>
<ClosingModule />
</SimpleLayout>
</PDFDocument> </PDFDocument>
); );
}; };

View File

@@ -0,0 +1,22 @@
"use client";
import * as React from "react";
import { Page as PDFPage, Document as PDFDocument } from "@react-pdf/renderer";
import { pdfStyles } from "./pdf/SharedUI.js";
import { FrontPageModule } from "./pdf/modules/FrontPageModule.js";
export const FrontPagePDF = ({ state, headerIcon }: any) => {
const date = new Date().toLocaleDateString("de-DE", {
year: "numeric",
month: "long",
day: "numeric",
});
return (
<PDFDocument title={`Deckblatt - ${state?.companyName || "Projekt"}`}>
<PDFPage size="A4" style={pdfStyles.titlePage}>
<FrontPageModule state={state} headerIcon={headerIcon} date={date} />
</PDFPage>
</PDFDocument>
);
};

View File

@@ -16,6 +16,7 @@ import {
Divider, Divider,
} from "./pdf/SharedUI.js"; } from "./pdf/SharedUI.js";
import { SimpleLayout } from "./pdf/SimpleLayout.js"; import { SimpleLayout } from "./pdf/SimpleLayout.js";
import { TransparenzModule } from "./pdf/modules/TransparenzModule.js";
const styles = PDFStyleSheet.create({ const styles = PDFStyleSheet.create({
section: { section: {
@@ -74,7 +75,7 @@ const styles = PDFStyleSheet.create({
}, },
}); });
export const InfoPDF = ({ headerIcon, footerLogo }: { headerIcon?: string; footerLogo?: string }) => { export const InfoPDF = ({ headerIcon, footerLogo, pricing }: { headerIcon?: string; footerLogo?: string; pricing?: any }) => {
const companyData = { const companyData = {
name: "Marc Mintel", name: "Marc Mintel",
address1: "Georg-Meistermann-Straße 7", address1: "Georg-Meistermann-Straße 7",
@@ -153,6 +154,12 @@ export const InfoPDF = ({ headerIcon, footerLogo }: { headerIcon?: string; foote
<PDFText style={[styles.textRegular, { fontSize: FONT_SIZES.SMALL, textAlign: 'center', color: COLORS.TEXT_LIGHT }]}> <PDFText style={[styles.textRegular, { fontSize: FONT_SIZES.SMALL, textAlign: 'center', color: COLORS.TEXT_LIGHT }]}>
Marc Mintel Digital Architect & Senior Software Developer Marc Mintel Digital Architect & Senior Software Developer
</PDFText> </PDFText>
{pricing && (
<PDFView break>
<TransparenzModule pricing={pricing} />
</PDFView>
)}
</PDFView> </PDFView>
); );

View File

@@ -525,13 +525,13 @@ export const FoldingMarks = () => (
export const Footer = ({ export const Footer = ({
logo, logo,
companyData, companyData,
_bankData, bankData,
showDetails = true, showDetails = true,
showPageNumber = true, showPageNumber = true,
}: { }: {
logo?: string; logo?: string;
companyData: any; companyData: any;
_bankData?: any; bankData?: any;
showDetails?: boolean; showDetails?: boolean;
showPageNumber?: boolean; showPageNumber?: boolean;
}) => ( }) => (

View File

@@ -48,7 +48,7 @@ export const SimpleLayout: React.FC<SimpleLayoutProps> = ({
<Footer <Footer
logo={footerLogo} logo={footerLogo}
companyData={companyData} companyData={companyData}
_bankData={bankData} bankData={bankData}
showDetails={showDetails} showDetails={showDetails}
showPageNumber={showPageNumber} showPageNumber={showPageNumber}
/> />

View File

@@ -9,7 +9,7 @@ import {
import { DocumentTitle, COLORS, FONT_SIZES } from "../SharedUI"; import { DocumentTitle, COLORS, FONT_SIZES } from "../SharedUI";
const styles = StyleSheet.create({ const styles = StyleSheet.create({
section: { marginBottom: 32 }, section: { paddingBottom: 16 },
intro: { intro: {
fontSize: FONT_SIZES.BODY, fontSize: FONT_SIZES.BODY,
color: COLORS.TEXT_DIM, color: COLORS.TEXT_DIM,

View File

@@ -22,13 +22,15 @@ import { EstimationPDF } from "../components/EstimationPDF.js";
import { ConceptPDF } from "../components/ConceptPDF.js"; import { ConceptPDF } from "../components/ConceptPDF.js";
import { InfoPDF } from "../components/InfoPDF.js"; import { InfoPDF } from "../components/InfoPDF.js";
import { AgbsPDF } from "../components/AgbsPDF.js"; import { AgbsPDF } from "../components/AgbsPDF.js";
import { FrontPagePDF } from "../components/FrontPagePDF.js";
import { ClosingPDF } from "../components/ClosingPDF.js";
import { PRICING } from "../logic/pricing/constants.js"; import { PRICING } from "../logic/pricing/constants.js";
import { calculateTotals } from "../logic/pricing/calculator.js"; import { calculateTotals } from "../logic/pricing/calculator.js";
export class PdfEngine { export class PdfEngine {
constructor() { } constructor() { }
async generateEstimatePdf(state: any, outputPath: string): Promise<string> { async generateEstimatePdf(state: any, outputPath: string, options: { headerIcon?: string; footerLogo?: string } = {}): Promise<string> {
const totals = calculateTotals(state, PRICING); const totals = calculateTotals(state, PRICING);
await renderToFile( await renderToFile(
@@ -36,6 +38,7 @@ export class PdfEngine {
state, state,
totalPrice: totals.totalPrice, totalPrice: totals.totalPrice,
pricing: PRICING, pricing: PRICING,
...options
} as any) as any, } as any) as any,
outputPath outputPath
); );
@@ -43,10 +46,11 @@ export class PdfEngine {
return outputPath; return outputPath;
} }
async generateConceptPdf(concept: any, outputPath: string): Promise<string> { async generateConceptPdf(concept: any, outputPath: string, options: { headerIcon?: string; footerLogo?: string } = {}): Promise<string> {
await renderToFile( await renderToFile(
createElement(ConceptPDF as any, { createElement(ConceptPDF as any, {
concept, concept,
...options
} as any) as any, } as any) as any,
outputPath outputPath
); );
@@ -56,7 +60,7 @@ export class PdfEngine {
async generateInfoPdf(outputPath: string, options: { headerIcon?: string; footerLogo?: string } = {}): Promise<string> { async generateInfoPdf(outputPath: string, options: { headerIcon?: string; footerLogo?: string } = {}): Promise<string> {
await renderToFile( await renderToFile(
createElement(InfoPDF as any, options as any) as any, createElement(InfoPDF as any, { ...options, pricing: PRICING } as any) as any,
outputPath outputPath
); );
@@ -71,4 +75,25 @@ export class PdfEngine {
return outputPath; return outputPath;
} }
async generateFrontPagePdf(state: any, outputPath: string, options: { headerIcon?: string } = {}): Promise<string> {
await renderToFile(
createElement(FrontPagePDF as any, {
state,
...options
} as any) as any,
outputPath
);
return outputPath;
}
async generateClosingPdf(outputPath: string, options: { headerIcon?: string; footerLogo?: string } = {}): Promise<string> {
await renderToFile(
createElement(ClosingPDF as any, options as any) as any,
outputPath
);
return outputPath;
}
} }