feat(seo-engine): implement competitor scraper, MDX draft editor, and strategy report generator

packages/seo-engine/src/agents/scraper.ts

@@ -0,0 +1,132 @@
+import axios from "axios";
+import * as cheerio from "cheerio";
+import { llmJsonRequest } from "../llm-client.js";
+
+export interface ScrapedContext {
+  url: string;
+  wordCount: number;
+  text: string;
+  headings: { level: number; text: string }[];
+}
+
+export interface ReverseEngineeredBriefing {
+  recommendedWordCount: number;
+  coreTopicsToCover: string[];
+  suggestedHeadings: string[];
+  entitiesToInclude: string[];
+  contentFormat: string; // e.g. "Lange Liste mit Fakten", "Kaufberater", "Lexikon-Eintrag"
+}
+
+/**
+ * Fetches the HTML of a URL and extracts the main readable text and headings.
+ */
+export async function scrapeCompetitorUrl(
+  url: string,
+): Promise<ScrapedContext | null> {
+  try {
+    console.log(`[Scraper] Fetching source: ${url}`);
+    const response = await axios.get(url, {
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+      },
+      timeout: 10000,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    // Remove junk elements before extracting text
+    $(
+      "script, style, nav, footer, header, aside, .cookie, .banner, iframe",
+    ).remove();
+
+    const headings: { level: number; text: string }[] = [];
+    $(":header").each((_, el) => {
+      const level = parseInt(el.tagName.replace(/h/i, ""), 10);
+      const text = $(el).text().trim().replace(/\s+/g, " ");
+      if (text) headings.push({ level, text });
+    });
+
+    // Extract body text, removing excessive whitespace
+    const text = $("body").text().replace(/\s+/g, " ").trim();
+    const wordCount = text.split(" ").length;
+
+    return {
+      url,
+      text: text.slice(0, 15000), // Cap length to prevent blowing up the LLM token limit
+      wordCount,
+      headings,
+    };
+  } catch (err) {
+    console.error(
+      `[Scraper] Failed to scrape ${url}: ${(err as Error).message}`,
+    );
+    return null;
+  }
+}
+
+const BRIEFING_SYSTEM_PROMPT = `
+You are a Senior Technical SEO Strategist.
+I will give you the scraped text and headings of a competitor's article that currently ranks #1 on Google for our target keyword.
+
+### OBJECTIVE:
+Reverse engineer the content. Tell me EXACTLY what topics, entities, and headings we must include
+in our own article to beat this competitor.
+Do not just copy their headings. Distill the *core intent* and *required knowledge depth*.
+
+### RULES:
+- If the text is very short (e.g., an e-commerce category page), mention that the format is "Category Page" and recommend a word count +50% higher than theirs.
+- Extract hyper-specific entities (e.g. DIN norms, specific materials, specific processes) that prove topic authority.
+- LANGUAGE: Match the language of the provided text.
+
+### OUTPUT FORMAT:
+{
+  "recommendedWordCount": number,
+  "coreTopicsToCover": ["string"],
+  "suggestedHeadings": ["string"],
+  "entitiesToInclude": ["string"],
+  "contentFormat": "string"
+}
+`;
+
+/**
+ * Analyzes the scraped context using an LLM to generate a blueprint to beat the competitor.
+ */
+export async function analyzeCompetitorContent(
+  context: ScrapedContext,
+  targetKeyword: string,
+  config: { openRouterApiKey: string; model?: string },
+): Promise<ReverseEngineeredBriefing | null> {
+  const userPrompt = `
+TARGET KEYWORD TO BEAT: "${targetKeyword}"
+COMPETITOR URL: ${context.url}
+COMPETITOR WORD COUNT: ${context.wordCount}
+
+COMPETITOR HEADINGS:
+${context.headings.map((h) => `H${h.level}: ${h.text}`).join("\n")}
+
+COMPETITOR TEXT (Truncated):
+${context.text}
+`;
+
+  try {
+    const { data } = await llmJsonRequest<ReverseEngineeredBriefing>({
+      model: config.model || "google/gemini-2.5-pro",
+      apiKey: config.openRouterApiKey,
+      systemPrompt: BRIEFING_SYSTEM_PROMPT,
+      userPrompt,
+    });
+
+    // Ensure numbers are numbers
+    data.recommendedWordCount =
+      Number(data.recommendedWordCount) || context.wordCount + 300;
+
+    return data;
+  } catch (err) {
+    console.error(
+      `[Scraper] NLP Analysis failed for ${context.url}:`,
+      (err as Error).message,
+    );
+    return null;
+  }
+}

packages/seo-engine/src/agents/serper-agent.ts

@@ -0,0 +1,64 @@
+import axios from "axios";
+
+export interface SerperResult {
+  relatedSearches: string[];
+  peopleAlsoAsk: string[];
+  organicSnippets: string[];
+  estimatedTotalResults: number;
+}
+
+/**
+ * Fetch Google search data via Serper's /search endpoint.
+ * Extracts related searches, People Also Ask, organic snippets,
+ * and totalResults as a search volume proxy.
+ */
+export async function fetchSerperData(
+  query: string,
+  apiKey: string,
+  locale: { gl: string; hl: string } = { gl: "de", hl: "de" },
+): Promise<SerperResult> {
+  try {
+    const response = await axios.post(
+      "https://google.serper.dev/search",
+      {
+        q: query,
+        gl: locale.gl,
+        hl: locale.hl,
+      },
+      {
+        headers: {
+          "X-API-KEY": apiKey,
+          "Content-Type": "application/json",
+        },
+      },
+    );
+
+    const data = response.data;
+
+    const relatedSearches =
+      data.relatedSearches?.map((r: any) => r.query) || [];
+    const peopleAlsoAsk = data.peopleAlsoAsk?.map((p: any) => p.question) || [];
+    const organicSnippets = data.organic?.map((o: any) => o.snippet) || [];
+    const estimatedTotalResults = data.searchInformation?.totalResults
+      ? parseInt(data.searchInformation.totalResults, 10)
+      : 0;
+
+    return {
+      relatedSearches,
+      peopleAlsoAsk,
+      organicSnippets,
+      estimatedTotalResults,
+    };
+  } catch (error) {
+    console.error(
+      `Serper API error for query "${query}":`,
+      (error as Error).message,
+    );
+    return {
+      relatedSearches: [],
+      peopleAlsoAsk: [],
+      organicSnippets: [],
+      estimatedTotalResults: 0,
+    };
+  }
+}

packages/seo-engine/src/agents/serper-autocomplete.ts

@@ -0,0 +1,43 @@
+import axios from "axios";
+
+export interface AutocompleteResult {
+  suggestions: string[];
+}
+
+/**
+ * Fetch Google Autocomplete suggestions via Serper's /autocomplete endpoint.
+ * These represent real user typing behavior — extremely high-signal for long-tail keywords.
+ */
+export async function fetchAutocompleteSuggestions(
+  query: string,
+  apiKey: string,
+  locale: { gl: string; hl: string } = { gl: "de", hl: "de" },
+): Promise<AutocompleteResult> {
+  try {
+    const response = await axios.post(
+      "https://google.serper.dev/autocomplete",
+      {
+        q: query,
+        gl: locale.gl,
+        hl: locale.hl,
+      },
+      {
+        headers: {
+          "X-API-KEY": apiKey,
+          "Content-Type": "application/json",
+        },
+      },
+    );
+
+    const suggestions =
+      response.data.suggestions?.map((s: any) => s.value || s) || [];
+
+    return { suggestions };
+  } catch (error) {
+    console.error(
+      `Serper Autocomplete error for query "${query}":`,
+      (error as Error).message,
+    );
+    return { suggestions: [] };
+  }
+}

packages/seo-engine/src/agents/serper-competitors.ts

@@ -0,0 +1,75 @@
+import axios from "axios";
+
+export interface CompetitorRanking {
+  keyword: string;
+  domain: string;
+  position: number;
+  title: string;
+  snippet: string;
+  link: string;
+}
+
+/**
+ * For a given keyword, check which competitor domains appear in the top organic results.
+ * Filters results to only include domains in the `competitorDomains` list.
+ */
+export async function fetchCompetitorRankings(
+  keyword: string,
+  competitorDomains: string[],
+  apiKey: string,
+  locale: { gl: string; hl: string } = { gl: "de", hl: "de" },
+): Promise<CompetitorRanking[]> {
+  if (competitorDomains.length === 0) return [];
+
+  try {
+    const response = await axios.post(
+      "https://google.serper.dev/search",
+      {
+        q: keyword,
+        gl: locale.gl,
+        hl: locale.hl,
+        num: 20,
+      },
+      {
+        headers: {
+          "X-API-KEY": apiKey,
+          "Content-Type": "application/json",
+        },
+      },
+    );
+
+    const organic: any[] = response.data.organic || [];
+
+    // Normalize competitor domains for matching
+    const normalizedCompetitors = competitorDomains.map((d) =>
+      d
+        .replace(/^(https?:\/\/)?(www\.)?/, "")
+        .replace(/\/$/, "")
+        .toLowerCase(),
+    );
+
+    return organic
+      .filter((result: any) => {
+        const resultDomain = new URL(result.link).hostname
+          .replace(/^www\./, "")
+          .toLowerCase();
+        return normalizedCompetitors.some(
+          (cd) => resultDomain === cd || resultDomain.endsWith(`.${cd}`),
+        );
+      })
+      .map((result: any) => ({
+        keyword,
+        domain: new URL(result.link).hostname.replace(/^www\./, ""),
+        position: result.position,
+        title: result.title || "",
+        snippet: result.snippet || "",
+        link: result.link,
+      }));
+  } catch (error) {
+    console.error(
+      `Serper Competitor check error for keyword "${keyword}":`,
+      (error as Error).message,
+    );
+    return [];
+  }
+}