feat: migrate npm registry from Verdaccio to Gitea Packages

packages/concept-engine/src/pipeline.ts

@@ -0,0 +1,257 @@
+// ============================================================================
+// Pipeline Orchestrator
+// Runs all steps sequentially, tracks state, supports re-running individual steps.
+// ============================================================================
+
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { existsSync } from "node:fs";
+import { crawlSite, clearCrawlCache } from "./scraper.js";
+import { analyzeSite } from "./analyzer.js";
+import { executeResearch } from "./steps/00b-research.js";
+import { executeExtract } from "./steps/01-extract.js";
+import { executeSiteAudit } from "./steps/00a-site-audit.js";
+import { executeAudit } from "./steps/02-audit.js";
+import { executeStrategize } from "./steps/03-strategize.js";
+import { executeArchitect } from "./steps/04-architect.js";
+import type {
+    PipelineConfig,
+    PipelineInput,
+    ConceptState,
+    ProjectConcept,
+    StepResult,
+    StepUsage,
+} from "./types.js";
+
+export interface PipelineCallbacks {
+    onStepStart?: (stepId: string, stepName: string) => void;
+    onStepComplete?: (stepId: string, result: StepResult) => void;
+    onStepError?: (stepId: string, error: string) => void;
+}
+
+/**
+ * The main concept pipeline orchestrator.
+ * Runs conceptual steps sequentially and builds the ProjectConcept.
+ */
+export class ConceptPipeline {
+    private config: PipelineConfig;
+    private state: ConceptState;
+    private callbacks: PipelineCallbacks;
+
+    constructor(config: PipelineConfig, callbacks: PipelineCallbacks = {}) {
+        this.config = config;
+        this.callbacks = callbacks;
+        this.state = this.createInitialState();
+    }
+
+    private createInitialState(): ConceptState {
+        return {
+            briefing: "",
+            usage: {
+                totalPromptTokens: 0,
+                totalCompletionTokens: 0,
+                totalCost: 0,
+                perStep: [],
+            },
+        };
+    }
+
+    /**
+     * Run the full concept pipeline from scratch.
+     */
+    async run(input: PipelineInput): Promise<ProjectConcept> {
+        this.state.briefing = input.briefing;
+        this.state.url = input.url;
+        this.state.comments = input.comments;
+
+        // Ensure output directories
+        await fs.mkdir(this.config.outputDir, { recursive: true });
+        await fs.mkdir(this.config.crawlDir, { recursive: true });
+
+        // Step 0: Scrape & Analyze (deterministic)
+        if (input.url) {
+            if (input.clearCache) {
+                const domain = new URL(input.url).hostname;
+                await clearCrawlCache(this.config.crawlDir, domain);
+            }
+            await this.runStep("00-scrape", "Scraping & Analyzing Website", async () => {
+                const pages = await crawlSite(input.url!, {
+                    zyteApiKey: this.config.zyteApiKey,
+                    crawlDir: this.config.crawlDir,
+                });
+                const domain = new URL(input.url!).hostname;
+                const siteProfile = analyzeSite(pages, domain);
+                this.state.siteProfile = siteProfile;
+                this.state.crawlDir = path.join(this.config.crawlDir, domain.replace(/\./g, "-"));
+
+                // Save site profile
+                await fs.writeFile(
+                    path.join(this.state.crawlDir!, "_site_profile.json"),
+                    JSON.stringify(siteProfile, null, 2),
+                );
+
+                return {
+                    success: true,
+                    data: siteProfile,
+                    usage: { step: "00-scrape", model: "none", promptTokens: 0, completionTokens: 0, cost: 0, durationMs: 0 },
+                };
+            });
+        }
+
+        // Step 00a: Site Audit (DataForSEO)
+        await this.runStep("00a-site-audit", "IST-Analysis (DataForSEO)", async () => {
+            const result = await executeSiteAudit(this.state, this.config);
+            if (result.success && result.data) {
+                this.state.siteAudit = result.data;
+            }
+            return result;
+        });
+
+        // Step 00b: Research (real web data via journaling)
+        await this.runStep("00b-research", "Industry & Company Research", async () => {
+            const result = await executeResearch(this.state);
+            if (result.success && result.data) {
+                this.state.researchData = result.data;
+            }
+            return result;
+        });
+
+        // Step 1: Extract facts
+        await this.runStep("01-extract", "Extracting Facts from Briefing", async () => {
+            const result = await executeExtract(this.state, this.config);
+            if (result.success) this.state.facts = result.data;
+            return result;
+        });
+
+        // Step 2: Audit features
+        await this.runStep("02-audit", "Auditing Features (Skeptical Review)", async () => {
+            const result = await executeAudit(this.state, this.config);
+            if (result.success) this.state.auditedFacts = result.data;
+            return result;
+        });
+
+        // Step 3: Strategic analysis
+        await this.runStep("03-strategize", "Strategic Analysis", async () => {
+            const result = await executeStrategize(this.state, this.config);
+            if (result.success) {
+                this.state.briefingSummary = result.data.briefingSummary;
+                this.state.designVision = result.data.designVision;
+            }
+            return result;
+        });
+
+        // Step 4: Sitemap architecture
+        await this.runStep("04-architect", "Information Architecture", async () => {
+            const result = await executeArchitect(this.state, this.config);
+            if (result.success) {
+                this.state.sitemap = result.data.sitemap;
+                this.state.websiteTopic = result.data.websiteTopic;
+            }
+            return result;
+        });
+
+        const projectConcept = this.buildProjectConcept();
+        await this.saveState(projectConcept);
+
+        return projectConcept;
+    }
+
+    /**
+     * Run a single step with callbacks and error handling.
+     */
+    private async runStep(
+        stepId: string,
+        stepName: string,
+        executor: () => Promise<StepResult>,
+    ): Promise<void> {
+        this.callbacks.onStepStart?.(stepId, stepName);
+        console.log(`\n📍 ${stepName}...`);
+
+        try {
+            const result = await executor();
+            if (result.usage) {
+                this.state.usage.perStep.push(result.usage);
+                this.state.usage.totalPromptTokens += result.usage.promptTokens;
+                this.state.usage.totalCompletionTokens += result.usage.completionTokens;
+                this.state.usage.totalCost += result.usage.cost;
+            }
+
+            if (result.success) {
+                const cost = result.usage?.cost ? ` ($${result.usage.cost.toFixed(4)})` : "";
+                const duration = result.usage?.durationMs ? ` [${(result.usage.durationMs / 1000).toFixed(1)}s]` : "";
+                console.log(`  ✅ ${stepName} complete${cost}${duration}`);
+                this.callbacks.onStepComplete?.(stepId, result);
+            } else {
+                console.error(`  ❌ ${stepName} failed: ${result.error}`);
+                this.callbacks.onStepError?.(stepId, result.error || "Unknown error");
+                throw new Error(result.error);
+            }
+        } catch (err) {
+            const errorMsg = (err as Error).message;
+            this.callbacks.onStepError?.(stepId, errorMsg);
+            throw err;
+        }
+    }
+
+    /**
+     * Build the final Concept object.
+     */
+    private buildProjectConcept(): ProjectConcept {
+        return {
+            domain: this.state.siteProfile?.domain || "unknown",
+            timestamp: new Date().toISOString(),
+            briefing: this.state.briefing,
+            auditedFacts: this.state.auditedFacts || {},
+            siteProfile: this.state.siteProfile,
+            siteAudit: this.state.siteAudit,
+            researchData: this.state.researchData,
+            strategy: {
+                briefingSummary: this.state.briefingSummary || "",
+                designVision: this.state.designVision || "",
+            },
+            architecture: {
+                websiteTopic: this.state.websiteTopic || "",
+                sitemap: this.state.sitemap || [],
+            },
+            usage: this.state.usage,
+        };
+    }
+
+    /**
+     * Save the full concept generated state to disk.
+     */
+    private async saveState(concept: ProjectConcept): Promise<void> {
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const companyName = this.state.auditedFacts?.companyName || "unknown";
+
+        const stateDir = path.join(this.config.outputDir, "concepts");
+        await fs.mkdir(stateDir, { recursive: true });
+
+        const statePath = path.join(stateDir, `${companyName}_${timestamp}.json`);
+        await fs.writeFile(statePath, JSON.stringify(concept, null, 2));
+        console.log(`\n📦 Saved Project Concept to: ${statePath}`);
+
+        // Save debug trace
+        const debugPath = path.join(stateDir, `${companyName}_${timestamp}_debug.json`);
+        await fs.writeFile(debugPath, JSON.stringify(this.state, null, 2));
+
+        // Print usage summary
+        console.log("\n──────────────────────────────────────────────");
+        console.log("📊 PIPELINE USAGE SUMMARY");
+        console.log("──────────────────────────────────────────────");
+        for (const step of this.state.usage.perStep) {
+            if (step.cost > 0) {
+                console.log(`  ${step.step}: ${step.model} — $${step.cost.toFixed(6)} (${(step.durationMs / 1000).toFixed(1)}s)`);
+            }
+        }
+        console.log("──────────────────────────────────────────────");
+        console.log(`  TOTAL: $${this.state.usage.totalCost.toFixed(6)}`);
+        console.log(`  Tokens: ${(this.state.usage.totalPromptTokens + this.state.usage.totalCompletionTokens).toLocaleString()}`);
+        console.log("──────────────────────────────────────────────\n");
+    }
+
+    /** Get the current internal state (for CLI inspection). */
+    getState(): ConceptState {
+        return this.state;
+    }
+}