at-mintel/packages/seo-engine/src/agents/scraper.ts

import axios from "axios";
import * as cheerio from "cheerio";
import { llmJsonRequest } from "../llm-client.js";

export interface ScrapedContext {
  url: string;
  wordCount: number;
  text: string;
  headings: { level: number; text: string }[];
}

export interface ReverseEngineeredBriefing {
  recommendedWordCount: number;
  coreTopicsToCover: string[];
  suggestedHeadings: string[];
  entitiesToInclude: string[];
  contentFormat: string; // e.g. "Lange Liste mit Fakten", "Kaufberater", "Lexikon-Eintrag"
}

/**
 * Fetches the HTML of a URL and extracts the main readable text and headings.
 */
export async function scrapeCompetitorUrl(
  url: string,
): Promise<ScrapedContext | null> {
  try {
    console.log(`[Scraper] Fetching source: ${url}`);
    const response = await axios.get(url, {
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
      },
      timeout: 10000,
    });

    const $ = cheerio.load(response.data);

    // Remove junk elements before extracting text
    $(
      "script, style, nav, footer, header, aside, .cookie, .banner, iframe",
    ).remove();

    const headings: { level: number; text: string }[] = [];
    $(":header").each((_, el) => {
      const level = parseInt(el.tagName.replace(/h/i, ""), 10);
      const text = $(el).text().trim().replace(/\s+/g, " ");
      if (text) headings.push({ level, text });
    });

    // Extract body text, removing excessive whitespace
    const text = $("body").text().replace(/\s+/g, " ").trim();
    const wordCount = text.split(" ").length;

    return {
      url,
      text: text.slice(0, 15000), // Cap length to prevent blowing up the LLM token limit
      wordCount,
      headings,
    };
  } catch (err) {
    console.error(
      `[Scraper] Failed to scrape ${url}: ${(err as Error).message}`,
    );
    return null;
  }
}

const BRIEFING_SYSTEM_PROMPT = `
You are a Senior Technical SEO Strategist.
I will give you the scraped text and headings of a competitor's article that currently ranks #1 on Google for our target keyword.

### OBJECTIVE:
Reverse engineer the content. Tell me EXACTLY what topics, entities, and headings we must include
in our own article to beat this competitor.
Do not just copy their headings. Distill the *core intent* and *required knowledge depth*.

### RULES:
- If the text is very short (e.g., an e-commerce category page), mention that the format is "Category Page" and recommend a word count +50% higher than theirs.
- Extract hyper-specific entities (e.g. DIN norms, specific materials, specific processes) that prove topic authority.
- LANGUAGE: Match the language of the provided text.

### OUTPUT FORMAT:
{
  "recommendedWordCount": number,
  "coreTopicsToCover": ["string"],
  "suggestedHeadings": ["string"],
  "entitiesToInclude": ["string"],
  "contentFormat": "string"
}
`;

/**
 * Analyzes the scraped context using an LLM to generate a blueprint to beat the competitor.
 */
export async function analyzeCompetitorContent(
  context: ScrapedContext,
  targetKeyword: string,
  config: { openRouterApiKey: string; model?: string },
): Promise<ReverseEngineeredBriefing | null> {
  const userPrompt = `
TARGET KEYWORD TO BEAT: "${targetKeyword}"
COMPETITOR URL: ${context.url}
COMPETITOR WORD COUNT: ${context.wordCount}

COMPETITOR HEADINGS:
${context.headings.map((h) => `H${h.level}: ${h.text}`).join("\n")}

COMPETITOR TEXT (Truncated):
${context.text}
`;

  try {
    const { data } = await llmJsonRequest<ReverseEngineeredBriefing>({
      model: config.model || "google/gemini-2.5-pro",
      apiKey: config.openRouterApiKey,
      systemPrompt: BRIEFING_SYSTEM_PROMPT,
      userPrompt,
    });

    // Ensure numbers are numbers
    data.recommendedWordCount =
      Number(data.recommendedWordCount) || context.wordCount + 300;

    return data;
  } catch (err) {
    console.error(
      `[Scraper] NLP Analysis failed for ${context.url}:`,
      (err as Error).message,
    );
    return null;
  }
}