at-mintel/packages/concept-engine/src/pipeline.ts

// ============================================================================
// Pipeline Orchestrator
// Runs all steps sequentially, tracks state, supports re-running individual steps.
// ============================================================================

import * as fs from "node:fs/promises";
import * as path from "node:path";
import { existsSync } from "node:fs";
import { crawlSite, clearCrawlCache } from "./scraper.js";
import { analyzeSite } from "./analyzer.js";
import { executeResearch } from "./steps/00b-research.js";
import { executeExtract } from "./steps/01-extract.js";
import { executeSiteAudit } from "./steps/00a-site-audit.js";
import { executeAudit } from "./steps/02-audit.js";
import { executeStrategize } from "./steps/03-strategize.js";
import { executeArchitect } from "./steps/04-architect.js";
import type {
    PipelineConfig,
    PipelineInput,
    ConceptState,
    ProjectConcept,
    StepResult,
    StepUsage,
} from "./types.js";

export interface PipelineCallbacks {
    onStepStart?: (stepId: string, stepName: string) => void;
    onStepComplete?: (stepId: string, result: StepResult) => void;
    onStepError?: (stepId: string, error: string) => void;
}

/**
 * The main concept pipeline orchestrator.
 * Runs conceptual steps sequentially and builds the ProjectConcept.
 */
export class ConceptPipeline {
    private config: PipelineConfig;
    private state: ConceptState;
    private callbacks: PipelineCallbacks;

    constructor(config: PipelineConfig, callbacks: PipelineCallbacks = {}) {
        this.config = config;
        this.callbacks = callbacks;
        this.state = this.createInitialState();
    }

    private createInitialState(): ConceptState {
        return {
            briefing: "",
            usage: {
                totalPromptTokens: 0,
                totalCompletionTokens: 0,
                totalCost: 0,
                perStep: [],
            },
        };
    }

    /**
     * Run the full concept pipeline from scratch.
     */
    async run(input: PipelineInput): Promise<ProjectConcept> {
        this.state.briefing = input.briefing;
        this.state.url = input.url;
        this.state.comments = input.comments;

        // Ensure output directories
        await fs.mkdir(this.config.outputDir, { recursive: true });
        await fs.mkdir(this.config.crawlDir, { recursive: true });

        // Step 0: Scrape & Analyze (deterministic)
        if (input.url) {
            if (input.clearCache) {
                const domain = new URL(input.url).hostname;
                await clearCrawlCache(this.config.crawlDir, domain);
            }
            await this.runStep("00-scrape", "Scraping & Analyzing Website", async () => {
                const pages = await crawlSite(input.url!, {
                    zyteApiKey: this.config.zyteApiKey,
                    crawlDir: this.config.crawlDir,
                });
                const domain = new URL(input.url!).hostname;
                const siteProfile = analyzeSite(pages, domain);
                this.state.siteProfile = siteProfile;
                this.state.crawlDir = path.join(this.config.crawlDir, domain.replace(/\./g, "-"));

                // Save site profile
                await fs.writeFile(
                    path.join(this.state.crawlDir!, "_site_profile.json"),
                    JSON.stringify(siteProfile, null, 2),
                );

                return {
                    success: true,
                    data: siteProfile,
                    usage: { step: "00-scrape", model: "none", promptTokens: 0, completionTokens: 0, cost: 0, durationMs: 0 },
                };
            });
        }

        // Step 00a: Site Audit (DataForSEO)
        await this.runStep("00a-site-audit", "IST-Analysis (DataForSEO)", async () => {
            const result = await executeSiteAudit(this.state, this.config);
            if (result.success && result.data) {
                this.state.siteAudit = result.data;
            }
            return result;
        });

        // Step 00b: Research (real web data via journaling)
        await this.runStep("00b-research", "Industry & Company Research", async () => {
            const result = await executeResearch(this.state);
            if (result.success && result.data) {
                this.state.researchData = result.data;
            }
            return result;
        });

        // Step 1: Extract facts
        await this.runStep("01-extract", "Extracting Facts from Briefing", async () => {
            const result = await executeExtract(this.state, this.config);
            if (result.success) this.state.facts = result.data;
            return result;
        });

        // Step 2: Audit features
        await this.runStep("02-audit", "Auditing Features (Skeptical Review)", async () => {
            const result = await executeAudit(this.state, this.config);
            if (result.success) this.state.auditedFacts = result.data;
            return result;
        });

        // Step 3: Strategic analysis
        await this.runStep("03-strategize", "Strategic Analysis", async () => {
            const result = await executeStrategize(this.state, this.config);
            if (result.success) {
                this.state.briefingSummary = result.data.briefingSummary;
                this.state.designVision = result.data.designVision;
            }
            return result;
        });

        // Step 4: Sitemap architecture
        await this.runStep("04-architect", "Information Architecture", async () => {
            const result = await executeArchitect(this.state, this.config);
            if (result.success) {
                this.state.sitemap = result.data.sitemap;
                this.state.websiteTopic = result.data.websiteTopic;
            }
            return result;
        });

        const projectConcept = this.buildProjectConcept();
        await this.saveState(projectConcept);

        return projectConcept;
    }

    /**
     * Run a single step with callbacks and error handling.
     */
    private async runStep(
        stepId: string,
        stepName: string,
        executor: () => Promise<StepResult>,
    ): Promise<void> {
        this.callbacks.onStepStart?.(stepId, stepName);
        console.log(`\n📍 ${stepName}...`);

        try {
            const result = await executor();
            if (result.usage) {
                this.state.usage.perStep.push(result.usage);
                this.state.usage.totalPromptTokens += result.usage.promptTokens;
                this.state.usage.totalCompletionTokens += result.usage.completionTokens;
                this.state.usage.totalCost += result.usage.cost;
            }

            if (result.success) {
                const cost = result.usage?.cost ? ` ($${result.usage.cost.toFixed(4)})` : "";
                const duration = result.usage?.durationMs ? ` [${(result.usage.durationMs / 1000).toFixed(1)}s]` : "";
                console.log(`  ✅ ${stepName} complete${cost}${duration}`);
                this.callbacks.onStepComplete?.(stepId, result);
            } else {
                console.error(`  ❌ ${stepName} failed: ${result.error}`);
                this.callbacks.onStepError?.(stepId, result.error || "Unknown error");
                throw new Error(result.error);
            }
        } catch (err) {
            const errorMsg = (err as Error).message;
            this.callbacks.onStepError?.(stepId, errorMsg);
            throw err;
        }
    }

    /**
     * Build the final Concept object.
     */
    private buildProjectConcept(): ProjectConcept {
        return {
            domain: this.state.siteProfile?.domain || "unknown",
            timestamp: new Date().toISOString(),
            briefing: this.state.briefing,
            auditedFacts: this.state.auditedFacts || {},
            siteProfile: this.state.siteProfile,
            siteAudit: this.state.siteAudit,
            researchData: this.state.researchData,
            strategy: {
                briefingSummary: this.state.briefingSummary || "",
                designVision: this.state.designVision || "",
            },
            architecture: {
                websiteTopic: this.state.websiteTopic || "",
                sitemap: this.state.sitemap || [],
            },
            usage: this.state.usage,
        };
    }

    /**
     * Save the full concept generated state to disk.
     */
    private async saveState(concept: ProjectConcept): Promise<void> {
        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
        const companyName = this.state.auditedFacts?.companyName || "unknown";

        const stateDir = path.join(this.config.outputDir, "concepts");
        await fs.mkdir(stateDir, { recursive: true });

        const statePath = path.join(stateDir, `${companyName}_${timestamp}.json`);
        await fs.writeFile(statePath, JSON.stringify(concept, null, 2));
        console.log(`\n📦 Saved Project Concept to: ${statePath}`);

        // Save debug trace
        const debugPath = path.join(stateDir, `${companyName}_${timestamp}_debug.json`);
        await fs.writeFile(debugPath, JSON.stringify(this.state, null, 2));

        // Print usage summary
        console.log("\n──────────────────────────────────────────────");
        console.log("📊 PIPELINE USAGE SUMMARY");
        console.log("──────────────────────────────────────────────");
        for (const step of this.state.usage.perStep) {
            if (step.cost > 0) {
                console.log(`  ${step.step}: ${step.model} — $${step.cost.toFixed(6)} (${(step.durationMs / 1000).toFixed(1)}s)`);
            }
        }
        console.log("──────────────────────────────────────────────");
        console.log(`  TOTAL: $${this.state.usage.totalCost.toFixed(6)}`);
        console.log(`  Tokens: ${(this.state.usage.totalPromptTokens + this.state.usage.totalCompletionTokens).toLocaleString()}`);
        console.log("──────────────────────────────────────────────\n");
    }

    /** Get the current internal state (for CLI inspection). */
    getState(): ConceptState {
        return this.state;
    }
}