feat: content engine usw

packages/image-processor/models/tiny_face_detector_model-shard1

@@ -0,0 +1 @@
+404: Not Found

packages/image-processor/models/tiny_face_detector_model-weights_manifest.json

@@ -0,0 +1,30 @@
+[
+  {
+      "weights":
+      [
+          {"name":"conv0/filters","shape":[3,3,3,16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.009007044399485869,"min":-1.2069439495311063}},
+          {"name":"conv0/bias","shape":[16],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005263455241334205,"min":-0.9211046672334858}},
+          {"name":"conv1/depthwise_filter","shape":[3,3,16,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.004001977630690033,"min":-0.5042491814669441}},
+          {"name":"conv1/pointwise_filter","shape":[1,1,16,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013836609615999109,"min":-1.411334180831909}},
+          {"name":"conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0015159862590771096,"min":-0.30926119685173037}},
+          {"name":"conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002666276225856706,"min":-0.317286870876948}},
+          {"name":"conv2/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.015265831292844286,"min":-1.6792414422128714}},
+          {"name":"conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0020280554598453,"min":-0.37113414915168985}},
+          {"name":"conv3/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006100742489683862,"min":-0.8907084034938438}},
+          {"name":"conv3/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.016276211832083907,"min":-2.0508026908425725}},
+          {"name":"conv3/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003394414279975143,"min":-0.7637432129944072}},
+          {"name":"conv4/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006716050119961009,"min":-0.8059260143953211}},
+          {"name":"conv4/pointwise_filter","shape":[1,1,128,256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.021875603993733724,"min":-2.8875797271728514}},
+          {"name":"conv4/bias","shape":[256],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0041141652009066415,"min":-0.8187188749804216}},
+          {"name":"conv5/depthwise_filter","shape":[3,3,256,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008423839597141042,"min":-0.9013508368940915}},
+          {"name":"conv5/pointwise_filter","shape":[1,1,256,512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.030007277283014035,"min":-3.8709387695088107}},
+          {"name":"conv5/bias","shape":[512],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008402082966823203,"min":-1.4871686851277068}},
+          {"name":"conv8/filters","shape":[1,1,512,25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.028336129469030042,"min":-4.675461362389957}},
+          {"name":"conv8/bias","shape":[25],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002268134028303857,"min":-0.41053225912299807}}
+      ],
+      "paths":
+      [
+          "tiny_face_detector_model.bin"
+      ]
+  }
+]

packages/image-processor/package.json

@@ -13,11 +13,14 @@
     }
   },
   "scripts": {
-    "build": "tsup src/index.ts --format esm --dts --clean",
-    "dev": "tsup src/index.ts --format esm --watch --dts",
+    "build": "tsup",
+    "dev": "tsup --watch",
     "lint": "eslint src"
   },
   "dependencies": {
+    "@tensorflow/tfjs": "^4.22.0",
+    "@vladmandic/face-api": "^1.7.15",
+    "canvas": "^3.2.1",
     "sharp": "^0.33.2"
   },
   "devDependencies": {
@@ -27,4 +30,4 @@
     "tsup": "^8.3.5",
     "typescript": "^5.0.0"
   }
-}
+}

packages/image-processor/src/processor.ts

@@ -1,11 +1,40 @@
 import sharp from "sharp";
+import { Canvas, Image, ImageData } from "canvas";
+// Use the ESM no-bundle build to avoid the default Node entrypoint
+// which hardcodes require('@tensorflow/tfjs-node') and crashes in Docker.
+// This build uses pure @tensorflow/tfjs (JS-only, no native C++ bindings).
+// @ts-ignore - direct path import has no type declarations
+import * as faceapi from "@vladmandic/face-api/dist/face-api.esm-nobundle.js";
+import * as tf from "@tensorflow/tfjs";
+import path from "path";
+import { fileURLToPath } from "url";
+
+// Polyfill required by face-api for Node.js
+faceapi.env.monkeyPatch({ Canvas, Image, ImageData } as any);
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const MODEL_URL = path.join(__dirname, "../models");
+
+// State flag to ensure we only load weights once
+let modelsLoaded = false;
+
+async function loadModelsOnce() {
+  if (modelsLoaded) return;
+  // Initialize pure JS CPU backend (no native bindings needed)
+  await tf.setBackend("cpu");
+  await tf.ready();
+
+  // Load the microscopic TinyFaceDetector (~190KB)
+  await faceapi.nets.tinyFaceDetector.loadFromDisk(MODEL_URL);
+  modelsLoaded = true;
+}
 
 export interface ProcessImageOptions {
   width: number;
   height: number;
   format?: "webp" | "jpeg" | "png" | "avif";
   quality?: number;
-  openRouterApiKey?: string;
 }
 
 /**
@@ -80,91 +109,6 @@ export function parseImgproxyOptions(
   return options;
 }
 
-interface FaceDetection {
-  x: number;
-  y: number;
-  width: number;
-  height: number;
-}
-
-/**
- * Detects faces using OpenRouter Vision API.
- * Uses a small preview to save bandwidth and tokens.
- */
-async function detectFacesWithCloud(
-  inputBuffer: Buffer,
-  apiKey: string,
-): Promise<FaceDetection[]> {
-  try {
-    // Generate a small preview for vision API (max 512px)
-    const preview = await sharp(inputBuffer)
-      .resize(512, 512, { fit: "inside" })
-      .jpeg({ quality: 60 })
-      .toBuffer();
-
-    const base64Image = preview.toString("base64");
-
-    const response = await fetch(
-      "https://openrouter.ai/api/v1/chat/completions",
-      {
-        method: "POST",
-        headers: {
-          Authorization: `Bearer ${apiKey}`,
-          "Content-Type": "application/json",
-          "HTTP-Referer": "https://mintel.me",
-          "X-Title": "Mintel Image Service",
-        },
-        body: JSON.stringify({
-          model: "google/gemini-3-flash-preview", // Fast, cheap, and supports vision
-          messages: [
-            {
-              role: "user",
-              content: [
-                {
-                  type: "text",
-                  text: 'Detect all human faces in this image. Return ONLY a JSON array of bounding boxes like: [{"x": 0.1, "y": 0.2, "width": 0.05, "height": 0.05}]. Coordinates must be normalized (0 to 1). If no faces, return [].',
-                },
-                {
-                  type: "image_url",
-                  image_url: {
-                    url: `data:image/jpeg;base64,${base64Image}`,
-                  },
-                },
-              ],
-            },
-          ],
-          response_format: { type: "json_object" },
-        }),
-      },
-    );
-
-    if (!response.ok) {
-      throw new Error(`OpenRouter API error: ${response.statusText}`);
-    }
-
-    const data = (await response.json()) as any;
-    const content = data.choices[0]?.message?.content;
-
-    if (!content) return [];
-
-    // The model might return directly or wrapped in a json field
-    const parsed = typeof content === "string" ? JSON.parse(content) : content;
-    const detections = (parsed.faces || parsed.detections || parsed) as any[];
-
-    if (!Array.isArray(detections)) return [];
-
-    return detections.map((d) => ({
-      x: d.x,
-      y: d.y,
-      width: d.width,
-      height: d.height,
-    }));
-  } catch (error) {
-    console.error("Cloud face detection failed:", error);
-    return [];
-  }
-}
-
 export async function processImageWithSmartCrop(
   inputBuffer: Buffer,
   options: ProcessImageOptions,
@@ -176,32 +120,41 @@ export async function processImageWithSmartCrop(
     throw new Error("Could not read image metadata");
   }
 
-  const detections = options.openRouterApiKey
-    ? await detectFacesWithCloud(inputBuffer, options.openRouterApiKey)
-    : [];
+  // Load ML models (noop if already loaded)
+  await loadModelsOnce();
+
+  // Convert sharp image to a Node-compatible canvas Image for face-api
+  const jpegBuffer = await sharpImage.jpeg().toBuffer();
+  const img = new Image();
+  img.src = jpegBuffer;
+  const canvas = new Canvas(img.width, img.height);
+  const ctx = canvas.getContext("2d");
+  ctx.drawImage(img, 0, 0, img.width, img.height);
+
+  // Detect faces locally using the tiny model
+  // Requires explicit any cast since the types expect HTML elements in browser contexts
+  const detections = await faceapi.detectAllFaces(
+    canvas as any,
+    new faceapi.TinyFaceDetectorOptions(),
+  );
+
+  let cropPosition: "center" | "attention" | number = "attention"; // Fallback to sharp's attention if no faces
 
-  // If faces are found, calculate the bounding box containing all faces
   if (detections.length > 0) {
-    // Map normalized coordinates back to pixels
-    const pixelDetections = detections.map((d) => ({
-      x: d.x * (metadata.width || 0),
-      y: d.y * (metadata.height || 0),
-      width: d.width * (metadata.width || 0),
-      height: d.height * (metadata.height || 0),
-    }));
-
+    // We have faces! Calculate the bounding box that contains all of them
     let minX = metadata.width;
     let minY = metadata.height;
     let maxX = 0;
     let maxY = 0;
 
-    for (const det of pixelDetections) {
-      if (det.x < minX) minX = Math.max(0, det.x);
-      if (det.y < minY) minY = Math.max(0, det.y);
-      if (det.x + det.width > maxX)
-        maxX = Math.min(metadata.width, det.x + det.width);
-      if (det.y + det.height > maxY)
-        maxY = Math.min(metadata.height, det.y + det.height);
+    for (const det of detections) {
+      const box = det.box;
+      if (box.x < minX) minX = Math.max(0, box.x);
+      if (box.y < minY) minY = Math.max(0, box.y);
+      if (box.x + box.width > maxX)
+        maxX = Math.min(metadata.width, box.x + box.width);
+      if (box.y + box.height > maxY)
+        maxY = Math.min(metadata.height, box.y + box.height);
     }
 
     const centerX = Math.floor(minX + (maxX - minX) / 2);
@@ -213,32 +166,39 @@ export async function processImageWithSmartCrop(
     let cropWidth = metadata.width;
     let cropHeight = metadata.height;
 
+    // Determine the maximal crop window that maintains aspect ratio
     if (currentRatio > targetRatio) {
       cropWidth = Math.floor(metadata.height * targetRatio);
     } else {
       cropHeight = Math.floor(metadata.width / targetRatio);
     }
 
+    // Center the crop window over the center of the faces
     let cropX = Math.floor(centerX - cropWidth / 2);
     let cropY = Math.floor(centerY - cropHeight / 2);
 
+    // Keep crop window inside image bounds
     if (cropX < 0) cropX = 0;
     if (cropY < 0) cropY = 0;
     if (cropX + cropWidth > metadata.width) cropX = metadata.width - cropWidth;
     if (cropY + cropHeight > metadata.height)
       cropY = metadata.height - cropHeight;
 
+    // Pre-crop the image to isolate the faces before resizing
     sharpImage.extract({
       left: cropX,
       top: cropY,
       width: cropWidth,
       height: cropHeight,
     });
+
+    // As we manually calculated the exact focal box, we can now just center it
+    cropPosition = "center";
   }
 
   let finalImage = sharpImage.resize(options.width, options.height, {
     fit: "cover",
-    position: detections.length > 0 ? "center" : "attention",
+    position: cropPosition,
   });
 
   const format = options.format || "webp";

packages/image-processor/tsup.config.ts

@@ -0,0 +1,19 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+    entry: ["src/index.ts"],
+    format: ["esm"],
+    dts: true,
+    clean: true,
+    // Bundle face-api and tensorflow inline (they're pure JS).
+    // Keep sharp and canvas external (they have native C++ bindings).
+    noExternal: [
+        "@vladmandic/face-api",
+        "@tensorflow/tfjs",
+        "@tensorflow/tfjs-backend-wasm"
+    ],
+    external: [
+        "sharp",
+        "canvas"
+    ],
+});