feat(automation): add OS-level screen automation foundation services

packages/infrastructure/adapters/automation/NutJsAutomationAdapter.ts

@@ -1,5 +1,5 @@
 import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js';
-import { IBrowserAutomation } from '../../../packages/application/ports/IBrowserAutomation';
+import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation';
 import {
   AutomationResult,
   NavigationResult,
@@ -7,22 +7,33 @@ import {
   ClickResult,
   WaitResult,
   ModalResult,
-} from '../../../packages/application/ports/AutomationResults';
-import { StepId } from '../../../packages/domain/value-objects/StepId';
+} from '../../../application/ports/AutomationResults';
+import { StepId } from '../../../domain/value-objects/StepId';
+import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate';
+import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion';
 import type { ILogger } from '../../../application/ports/ILogger';
 import { NoOpLogAdapter } from '../logging/NoOpLogAdapter';
+import { ScreenRecognitionService } from './ScreenRecognitionService';
+import { TemplateMatchingService } from './TemplateMatchingService';
+import { WindowFocusService } from './WindowFocusService';
+import { getLoginIndicators, getLogoutIndicators } from './templates/IRacingTemplateMap';
 
 export interface NutJsConfig {
   mouseSpeed?: number;
   keyboardDelay?: number;
   screenResolution?: { width: number; height: number };
   defaultTimeout?: number;
+  templatePath?: string;
+  windowTitle?: string;
 }
 
-export class NutJsAutomationAdapter implements IBrowserAutomation {
+export class NutJsAutomationAdapter implements IScreenAutomation {
   private config: Required<NutJsConfig>;
   private connected: boolean = false;
   private logger: ILogger;
+  private screenRecognition: ScreenRecognitionService;
+  private templateMatching: TemplateMatchingService;
+  private windowFocus: WindowFocusService;
 
   constructor(config: NutJsConfig = {}, logger?: ILogger) {
     this.config = {
@@ -30,11 +41,23 @@ export class NutJsAutomationAdapter implements IBrowserAutomation {
       keyboardDelay: config.keyboardDelay ?? 50,
       screenResolution: config.screenResolution ?? { width: 1920, height: 1080 },
       defaultTimeout: config.defaultTimeout ?? 30000,
+      templatePath: config.templatePath ?? './resources/templates/iracing',
+      windowTitle: config.windowTitle ?? 'iRacing',
     };
     this.logger = logger ?? new NoOpLogAdapter();
 
     mouse.config.mouseSpeed = this.config.mouseSpeed;
     keyboard.config.autoDelayMs = this.config.keyboardDelay;
+
+    this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' }));
+    this.templateMatching = new TemplateMatchingService(
+      this.config.templatePath,
+      this.logger.child({ service: 'TemplateMatching' })
+    );
+    this.windowFocus = new WindowFocusService(
+      this.config.windowTitle,
+      this.logger.child({ service: 'WindowFocus' })
+    );
   }
 
   async connect(): Promise<AutomationResult> {
@@ -163,6 +186,106 @@ export class NutJsAutomationAdapter implements IBrowserAutomation {
     return this.connected;
   }
 
+  async detectLoginState(): Promise<LoginDetectionResult> {
+    this.logger.debug('Detecting login state via template matching');
+    
+    try {
+      const detectedIndicators: string[] = [];
+      let isLoggedIn = false;
+      let highestConfidence = 0;
+
+      const loginIndicators = getLoginIndicators();
+      for (const indicator of loginIndicators) {
+        const location = await this.templateMatching.findElement(indicator);
+        if (location) {
+          detectedIndicators.push(indicator.id);
+          isLoggedIn = true;
+          highestConfidence = Math.max(highestConfidence, location.confidence);
+          this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence });
+        }
+      }
+
+      if (!isLoggedIn) {
+        const logoutIndicators = getLogoutIndicators();
+        for (const indicator of logoutIndicators) {
+          const location = await this.templateMatching.findElement(indicator);
+          if (location) {
+            detectedIndicators.push(indicator.id);
+            highestConfidence = Math.max(highestConfidence, location.confidence);
+            this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence });
+          }
+        }
+      }
+
+      this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence });
+
+      return {
+        isLoggedIn,
+        confidence: highestConfidence,
+        detectedIndicators,
+      };
+    } catch (error) {
+      const errorMsg = `Login state detection failed: ${error}`;
+      this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg));
+      
+      return {
+        isLoggedIn: false,
+        confidence: 0,
+        detectedIndicators: [],
+        error: errorMsg,
+      };
+    }
+  }
+
+  async findElement(template: ImageTemplate): Promise<ElementLocation | null> {
+    return this.templateMatching.findElement(template);
+  }
+
+  async focusBrowserWindow(titlePattern?: string): Promise<WindowFocusResult> {
+    return this.windowFocus.focusBrowserWindow(titlePattern);
+  }
+
+  async captureScreen(region?: ScreenRegion): Promise<ScreenCaptureResult> {
+    if (region) {
+      return this.screenRecognition.captureRegion(region);
+    }
+    return this.screenRecognition.captureFullScreen();
+  }
+
+  async clickAtLocation(location: ElementLocation): Promise<ClickResult> {
+    try {
+      const point = new Point(location.center.x, location.center.y);
+      await mouse.move([point]);
+      await mouse.leftClick();
+      
+      this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y });
+      
+      return {
+        success: true,
+        target: `${location.center.x},${location.center.y}`
+      };
+    } catch (error) {
+      return {
+        success: false,
+        target: `${location.center.x},${location.center.y}`,
+        error: String(error)
+      };
+    }
+  }
+
+  async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise<WaitResult> {
+    const timeout = maxWaitMs ?? this.config.defaultTimeout;
+    const result = await this.templateMatching.waitForTemplate(template, timeout);
+    
+    return {
+      success: result.found,
+      target: template.id,
+      waitedMs: result.searchDurationMs,
+      found: result.found,
+      error: result.error,
+    };
+  }
+
   async executeStep(stepId: StepId, config: Record<string, unknown>): Promise<AutomationResult> {
     const stepNumber = stepId.value;
     const startTime = Date.now();