at-mintel/packages/concept-engine/src/pipeline.ts

// ============================================================================
// Pipeline Orchestrator
// Runs all steps sequentially, tracks state, supports re-running individual steps.
// ============================================================================

import * as fs from "node:fs/promises";
import * as path from "node:path";
import { crawlSite, clearCrawlCache } from "./scraper.js";
import { analyzeSite } from "./analyzer.js";
import { executeResearch } from "./steps/00b-research.js";
import { executeExtract } from "./steps/01-extract.js";
import { executeSiteAudit } from "./steps/00a-site-audit.js";
import { executeAudit } from "./steps/02-audit.js";
import { executeStrategize } from "./steps/03-strategize.js";
import { executeArchitect } from "./steps/04-architect.js";
import type {
  PipelineConfig,
  PipelineInput,
  ConceptState,
  ProjectConcept,
  StepResult,
} from "./types.js";

export interface PipelineCallbacks {
  onStepStart?: (stepId: string, stepName: string) => void;
  onStepComplete?: (stepId: string, result: StepResult) => void;
  onStepError?: (stepId: string, error: string) => void;
}

/**
 * The main concept pipeline orchestrator.
 * Runs conceptual steps sequentially and builds the ProjectConcept.
 */
export class ConceptPipeline {
  private config: PipelineConfig;
  private state: ConceptState;
  private callbacks: PipelineCallbacks;

  constructor(config: PipelineConfig, callbacks: PipelineCallbacks = {}) {
    this.config = config;
    this.callbacks = callbacks;
    this.state = this.createInitialState();
  }

  private createInitialState(): ConceptState {
    return {
      briefing: "",
      usage: {
        totalPromptTokens: 0,
        totalCompletionTokens: 0,
        totalCost: 0,
        perStep: [],
      },
    };
  }

  /**
   * Run the full concept pipeline from scratch.
   */
  async run(input: PipelineInput): Promise<ProjectConcept> {
    this.state.briefing = input.briefing;
    this.state.url = input.url;
    this.state.comments = input.comments;

    // Ensure output directories
    await fs.mkdir(this.config.outputDir, { recursive: true });
    await fs.mkdir(this.config.crawlDir, { recursive: true });

    // Step 0: Scrape & Analyze (deterministic)
    if (input.url) {
      if (input.clearCache) {
        const domain = new URL(input.url).hostname;
        await clearCrawlCache(this.config.crawlDir, domain);
      }
      await this.runStep(
        "00-scrape",
        "Scraping & Analyzing Website",
        async () => {
          const pages = await crawlSite(input.url!, {
            zyteApiKey: this.config.zyteApiKey,
            crawlDir: this.config.crawlDir,
          });
          const domain = new URL(input.url!).hostname;
          const siteProfile = analyzeSite(pages, domain);
          this.state.siteProfile = siteProfile;
          this.state.crawlDir = path.join(
            this.config.crawlDir,
            domain.replace(/\./g, "-"),
          );

          // Save site profile
          await fs.writeFile(
            path.join(this.state.crawlDir!, "_site_profile.json"),
            JSON.stringify(siteProfile, null, 2),
          );

          return {
            success: true,
            data: siteProfile,
            usage: {
              step: "00-scrape",
              model: "none",
              promptTokens: 0,
              completionTokens: 0,
              cost: 0,
              durationMs: 0,
            },
          };
        },
      );
    }

    // Step 00a: Site Audit (DataForSEO)
    await this.runStep(
      "00a-site-audit",
      "IST-Analysis (DataForSEO)",
      async () => {
        const result = await executeSiteAudit(this.state, this.config);
        if (result.success && result.data) {
          this.state.siteAudit = result.data;
        }
        return result;
      },
    );

    // Step 00b: Research (real web data via journaling)
    await this.runStep(
      "00b-research",
      "Industry & Company Research",
      async () => {
        const result = await executeResearch(this.state);
        if (result.success && result.data) {
          this.state.researchData = result.data;
        }
        return result;
      },
    );

    // Step 1: Extract facts
    await this.runStep(
      "01-extract",
      "Extracting Facts from Briefing",
      async () => {
        const result = await executeExtract(this.state, this.config);
        if (result.success) this.state.facts = result.data;
        return result;
      },
    );

    // Step 2: Audit features
    await this.runStep(
      "02-audit",
      "Auditing Features (Skeptical Review)",
      async () => {
        const result = await executeAudit(this.state, this.config);
        if (result.success) this.state.auditedFacts = result.data;
        return result;
      },
    );

    // Step 3: Strategic analysis
    await this.runStep("03-strategize", "Strategic Analysis", async () => {
      const result = await executeStrategize(this.state, this.config);
      if (result.success) {
        this.state.briefingSummary = result.data.briefingSummary;
        this.state.designVision = result.data.designVision;
      }
      return result;
    });

    // Step 4: Sitemap architecture
    await this.runStep("04-architect", "Information Architecture", async () => {
      const result = await executeArchitect(this.state, this.config);
      if (result.success) {
        this.state.sitemap = result.data.sitemap;
        this.state.websiteTopic = result.data.websiteTopic;
      }
      return result;
    });

    const projectConcept = this.buildProjectConcept();
    await this.saveState(projectConcept);

    return projectConcept;
  }

  /**
   * Run a single step with callbacks and error handling.
   */
  private async runStep(
    stepId: string,
    stepName: string,
    executor: () => Promise<StepResult>,
  ): Promise<void> {
    this.callbacks.onStepStart?.(stepId, stepName);
    console.log(`\n📍 ${stepName}...`);

    try {
      const result = await executor();
      if (result.usage) {
        this.state.usage.perStep.push(result.usage);
        this.state.usage.totalPromptTokens += result.usage.promptTokens;
        this.state.usage.totalCompletionTokens += result.usage.completionTokens;
        this.state.usage.totalCost += result.usage.cost;
      }

      if (result.success) {
        const cost = result.usage?.cost
          ? ` ($${result.usage.cost.toFixed(4)})`
          : "";
        const duration = result.usage?.durationMs
          ? ` [${(result.usage.durationMs / 1000).toFixed(1)}s]`
          : "";
        console.log(`  ✅ ${stepName} complete${cost}${duration}`);
        this.callbacks.onStepComplete?.(stepId, result);
      } else {
        console.error(`  ❌ ${stepName} failed: ${result.error}`);
        this.callbacks.onStepError?.(stepId, result.error || "Unknown error");
        throw new Error(result.error);
      }
    } catch (err) {
      const errorMsg = (err as Error).message;
      this.callbacks.onStepError?.(stepId, errorMsg);
      throw err;
    }
  }

  /**
   * Build the final Concept object.
   */
  private buildProjectConcept(): ProjectConcept {
    return {
      domain: this.state.siteProfile?.domain || "unknown",
      timestamp: new Date().toISOString(),
      briefing: this.state.briefing,
      auditedFacts: this.state.auditedFacts || {},
      siteProfile: this.state.siteProfile,
      siteAudit: this.state.siteAudit,
      researchData: this.state.researchData,
      strategy: {
        briefingSummary: this.state.briefingSummary || "",
        designVision: this.state.designVision || "",
      },
      architecture: {
        websiteTopic: this.state.websiteTopic || "",
        sitemap: this.state.sitemap || [],
      },
      usage: this.state.usage,
    };
  }

  /**
   * Save the full concept generated state to disk.
   */
  private async saveState(concept: ProjectConcept): Promise<void> {
    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
    const companyName = this.state.auditedFacts?.companyName || "unknown";

    const stateDir = path.join(this.config.outputDir, "concepts");
    await fs.mkdir(stateDir, { recursive: true });

    const statePath = path.join(stateDir, `${companyName}_${timestamp}.json`);
    await fs.writeFile(statePath, JSON.stringify(concept, null, 2));
    console.log(`\n📦 Saved Project Concept to: ${statePath}`);

    // Save debug trace
    const debugPath = path.join(
      stateDir,
      `${companyName}_${timestamp}_debug.json`,
    );
    await fs.writeFile(debugPath, JSON.stringify(this.state, null, 2));

    // Print usage summary
    console.log("\n──────────────────────────────────────────────");
    console.log("📊 PIPELINE USAGE SUMMARY");
    console.log("──────────────────────────────────────────────");
    for (const step of this.state.usage.perStep) {
      if (step.cost > 0) {
        console.log(
          `  ${step.step}: ${step.model} — $${step.cost.toFixed(6)} (${(step.durationMs / 1000).toFixed(1)}s)`,
        );
      }
    }
    console.log("──────────────────────────────────────────────");
    console.log(`  TOTAL: $${this.state.usage.totalCost.toFixed(6)}`);
    console.log(
      `  Tokens: ${(this.state.usage.totalPromptTokens + this.state.usage.totalCompletionTokens).toLocaleString()}`,
    );
    console.log("──────────────────────────────────────────────\n");
  }

  /** Get the current internal state (for CLI inspection). */
  getState(): ConceptState {
    return this.state;
  }
}