feat(seo-engine): implement competitor scraper, MDX draft editor, and strategy report generator

packages/seo-engine/src/agents/scraper.ts

@@ -0,0 +1,132 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+import { llmJsonRequest } from "../llm-client.js";
+
+export interface ScrapedContext {
+  url: string;
+  wordCount: number;
+  text: string;
+  headings: { level: number; text: string }[];
+}
+
+export interface ReverseEngineeredBriefing {
+  recommendedWordCount: number;
+  coreTopicsToCover: string[];
+  suggestedHeadings: string[];
+  entitiesToInclude: string[];
+  contentFormat: string; // e.g. "Lange Liste mit Fakten", "Kaufberater", "Lexikon-Eintrag"
+}
+
+/**
+ * Fetches the HTML of a URL and extracts the main readable text and headings.
+ */
+export async function scrapeCompetitorUrl(
+  url: string,
+): Promise<ScrapedContext | null> {
+  try {
+    console.log(`[Scraper] Fetching source: ${url}`);
+    const response = await axios.get(url, {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+      },
+      timeout: 10000,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    // Remove junk elements before extracting text
+    $(
+      "script, style, nav, footer, header, aside, .cookie, .banner, iframe",
+    ).remove();
+
+    const headings: { level: number; text: string }[] = [];
+    $(":header").each((_, el) => {
+      const level = parseInt(el.tagName.replace(/h/i, ""), 10);
+      const text = $(el).text().trim().replace(/\s+/g, " ");
+      if (text) headings.push({ level, text });
+    });
+
+    // Extract body text, removing excessive whitespace
+    const text = $("body").text().replace(/\s+/g, " ").trim();
+    const wordCount = text.split(" ").length;
+
+    return {
+      url,
+      text: text.slice(0, 15000), // Cap length to prevent blowing up the LLM token limit
+      wordCount,
+      headings,
+    };
+  } catch (err) {
+    console.error(
+      `[Scraper] Failed to scrape ${url}: ${(err as Error).message}`,
+    );
+    return null;
+  }
+}
+
+const BRIEFING_SYSTEM_PROMPT = `
+You are a Senior Technical SEO Strategist.
+I will give you the scraped text and headings of a competitor's article that currently ranks #1 on Google for our target keyword.
+
+### OBJECTIVE:
+Reverse engineer the content. Tell me EXACTLY what topics, entities, and headings we must include
+in our own article to beat this competitor.
+Do not just copy their headings. Distill the *core intent* and *required knowledge depth*.
+
+### RULES:
+- If the text is very short (e.g., an e-commerce category page), mention that the format is "Category Page" and recommend a word count +50% higher than theirs.
+- Extract hyper-specific entities (e.g. DIN norms, specific materials, specific processes) that prove topic authority.
+- LANGUAGE: Match the language of the provided text.
+
+### OUTPUT FORMAT:
+{
+  "recommendedWordCount": number,
+  "coreTopicsToCover": ["string"],
+  "suggestedHeadings": ["string"],
+  "entitiesToInclude": ["string"],
+  "contentFormat": "string"
+}
+`;
+
+/**
+ * Analyzes the scraped context using an LLM to generate a blueprint to beat the competitor.
+ */
+export async function analyzeCompetitorContent(
+  context: ScrapedContext,
+  targetKeyword: string,
+  config: { openRouterApiKey: string; model?: string },
+): Promise<ReverseEngineeredBriefing | null> {
+  const userPrompt = `
+TARGET KEYWORD TO BEAT: "${targetKeyword}"
+COMPETITOR URL: ${context.url}
+COMPETITOR WORD COUNT: ${context.wordCount}
+
+COMPETITOR HEADINGS:
+${context.headings.map((h) => `H${h.level}: ${h.text}`).join("\n")}
+
+COMPETITOR TEXT (Truncated):
+${context.text}
+`;
+
+  try {
+    const { data } = await llmJsonRequest<ReverseEngineeredBriefing>({
+      model: config.model || "google/gemini-2.5-pro",
+      apiKey: config.openRouterApiKey,
+      systemPrompt: BRIEFING_SYSTEM_PROMPT,
+      userPrompt,
+    });
+
+    // Ensure numbers are numbers
+    data.recommendedWordCount =
+      Number(data.recommendedWordCount) || context.wordCount + 300;
+
+    return data;
+  } catch (err) {
+    console.error(
+      `[Scraper] NLP Analysis failed for ${context.url}:`,
+      (err as Error).message,
+    );
+    return null;
+  }
+}