at-mintel/packages/seo-engine/src/agents/serper-competitors.ts

import axios from "axios";

export interface CompetitorRanking {
  keyword: string;
  domain: string;
  position: number;
  title: string;
  snippet: string;
  link: string;
}

/**
 * For a given keyword, check which competitor domains appear in the top organic results.
 * Filters results to only include domains in the `competitorDomains` list.
 */
export async function fetchCompetitorRankings(
  keyword: string,
  competitorDomains: string[],
  apiKey: string,
  locale: { gl: string; hl: string } = { gl: "de", hl: "de" },
): Promise<CompetitorRanking[]> {
  if (competitorDomains.length === 0) return [];

  try {
    const response = await axios.post(
      "https://google.serper.dev/search",
      {
        q: keyword,
        gl: locale.gl,
        hl: locale.hl,
        num: 20,
      },
      {
        headers: {
          "X-API-KEY": apiKey,
          "Content-Type": "application/json",
        },
      },
    );

    const organic: any[] = response.data.organic || [];

    // Normalize competitor domains for matching
    const normalizedCompetitors = competitorDomains.map((d) =>
      d
        .replace(/^(https?:\/\/)?(www\.)?/, "")
        .replace(/\/$/, "")
        .toLowerCase(),
    );

    return organic
      .filter((result: any) => {
        const resultDomain = new URL(result.link).hostname
          .replace(/^www\./, "")
          .toLowerCase();
        return normalizedCompetitors.some(
          (cd) => resultDomain === cd || resultDomain.endsWith(`.${cd}`),
        );
      })
      .map((result: any) => ({
        keyword,
        domain: new URL(result.link).hostname.replace(/^www\./, ""),
        position: result.position,
        title: result.title || "",
        snippet: result.snippet || "",
        link: result.link,
      }));
  } catch (error) {
    console.error(
      `Serper Competitor check error for keyword "${keyword}":`,
      (error as Error).message,
    );
    return [];
  }
}