at-mintel/packages/concept-engine/src/_test_pipeline.ts

import { config as dotenvConfig } from "dotenv";
import * as path from "node:path";
import * as fs from "node:fs/promises";
import { ConceptPipeline } from "./pipeline.js";

dotenvConfig({ path: path.resolve(process.cwd(), "../../.env") });

const briefing = await fs.readFile(
  path.resolve(process.cwd(), "../../data/briefings/etib.txt"),
  "utf8",
);

console.log(`Briefing loaded: ${briefing.length} chars`);

const pipeline = new ConceptPipeline(
  {
    openrouterKey: process.env.OPENROUTER_API_KEY || "",
    zyteApiKey: process.env.ZYTE_API_KEY,
    outputDir: path.resolve(process.cwd(), "../../out/estimations"),
    crawlDir: path.resolve(process.cwd(), "../../data/crawls"),
  },
  {
    onStepStart: (id, _name) => console.log(`[CB] Starting: ${id}`),
    onStepComplete: (id) => console.log(`[CB] Done: ${id}`),
    onStepError: (id, err) => console.error(`[CB] Error in ${id}: ${err}`),
  },
);

try {
  await pipeline.run({
    briefing,
    url: "https://www.e-tib.com",
  });

  console.log("\n✨ Pipeline complete!");
} catch (err: any) {
  console.error("\n❌ Pipeline failed:", err.message);
  console.error(err.stack);
}