import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js'; import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation'; import { AutomationResult, NavigationResult, FormFillResult, ClickResult, WaitResult, ModalResult, } from '../../../application/ports/AutomationResults'; import { StepId } from '../../../domain/value-objects/StepId'; import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate'; import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion'; import type { ILogger } from '../../../application/ports/ILogger'; import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; import { ScreenRecognitionService } from './ScreenRecognitionService'; import { TemplateMatchingService } from './TemplateMatchingService'; import { WindowFocusService } from './WindowFocusService'; import { getLoginIndicators, getLogoutIndicators } from './templates/IRacingTemplateMap'; export interface NutJsConfig { mouseSpeed?: number; keyboardDelay?: number; screenResolution?: { width: number; height: number }; defaultTimeout?: number; templatePath?: string; windowTitle?: string; } export class NutJsAutomationAdapter implements IScreenAutomation { private config: Required; private connected: boolean = false; private logger: ILogger; private screenRecognition: ScreenRecognitionService; private templateMatching: TemplateMatchingService; private windowFocus: WindowFocusService; constructor(config: NutJsConfig = {}, logger?: ILogger) { this.config = { mouseSpeed: config.mouseSpeed ?? 1000, keyboardDelay: config.keyboardDelay ?? 50, screenResolution: config.screenResolution ?? { width: 1920, height: 1080 }, defaultTimeout: config.defaultTimeout ?? 30000, templatePath: config.templatePath ?? './resources/templates/iracing', windowTitle: config.windowTitle ?? 'iRacing', }; this.logger = logger ?? new NoOpLogAdapter(); mouse.config.mouseSpeed = this.config.mouseSpeed; keyboard.config.autoDelayMs = this.config.keyboardDelay; this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' })); this.templateMatching = new TemplateMatchingService( this.config.templatePath, this.logger.child({ service: 'TemplateMatching' }) ); this.windowFocus = new WindowFocusService( this.config.windowTitle, this.logger.child({ service: 'WindowFocus' }) ); } async connect(): Promise { const startTime = Date.now(); this.logger.info('Initializing nut.js OS-level automation'); try { const width = await screen.width(); const height = await screen.height(); this.connected = true; const durationMs = Date.now() - startTime; this.logger.info('nut.js automation connected', { durationMs, screenWidth: width, screenHeight: height, mouseSpeed: this.config.mouseSpeed, keyboardDelay: this.config.keyboardDelay }); return { success: true }; } catch (error) { const errorMsg = `Screen access failed: ${error}`; this.logger.error('Failed to initialize nut.js', error instanceof Error ? error : new Error(errorMsg)); return { success: false, error: errorMsg }; } } async navigateToPage(url: string): Promise { const startTime = Date.now(); try { const isMac = process.platform === 'darwin'; if (isMac) { await keyboard.pressKey(Key.LeftSuper, Key.L); await keyboard.releaseKey(Key.LeftSuper, Key.L); } else { await keyboard.pressKey(Key.LeftControl, Key.L); await keyboard.releaseKey(Key.LeftControl, Key.L); } await this.delay(100); await keyboard.type(url); await keyboard.pressKey(Key.Enter); await keyboard.releaseKey(Key.Enter); await this.delay(2000); return { success: true, url, loadTime: Date.now() - startTime }; } catch (error) { return { success: false, url, loadTime: 0, error: String(error) }; } } async fillFormField(fieldName: string, value: string): Promise { try { const isMac = process.platform === 'darwin'; if (isMac) { await keyboard.pressKey(Key.LeftSuper, Key.A); await keyboard.releaseKey(Key.LeftSuper, Key.A); } else { await keyboard.pressKey(Key.LeftControl, Key.A); await keyboard.releaseKey(Key.LeftControl, Key.A); } await this.delay(50); await keyboard.type(value); return { success: true, fieldName, valueSet: value }; } catch (error) { return { success: false, fieldName, valueSet: '', error: String(error) }; } } async clickElement(target: string): Promise { try { const point = this.parseTarget(target); await mouse.move([point]); await mouse.leftClick(); return { success: true, target }; } catch (error) { return { success: false, target, error: String(error) }; } } async waitForElement(target: string, maxWaitMs?: number): Promise { const startTime = Date.now(); const timeout = maxWaitMs ?? this.config.defaultTimeout; await this.delay(Math.min(1000, timeout)); return { success: true, target, waitedMs: Date.now() - startTime, found: true, }; } async handleModal(stepId: StepId, action: string): Promise { try { if (action === 'confirm') { await keyboard.pressKey(Key.Enter); await keyboard.releaseKey(Key.Enter); } else if (action === 'cancel') { await keyboard.pressKey(Key.Escape); await keyboard.releaseKey(Key.Escape); } return { success: true, stepId: stepId.value, action }; } catch (error) { return { success: false, stepId: stepId.value, action, error: String(error) }; } } async disconnect(): Promise { this.logger.info('Disconnecting nut.js automation'); this.connected = false; this.logger.debug('nut.js disconnected'); } isConnected(): boolean { return this.connected; } async detectLoginState(): Promise { this.logger.debug('Detecting login state via template matching'); try { const detectedIndicators: string[] = []; let isLoggedIn = false; let highestConfidence = 0; const loginIndicators = getLoginIndicators(); for (const indicator of loginIndicators) { const location = await this.templateMatching.findElement(indicator); if (location) { detectedIndicators.push(indicator.id); isLoggedIn = true; highestConfidence = Math.max(highestConfidence, location.confidence); this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence }); } } if (!isLoggedIn) { const logoutIndicators = getLogoutIndicators(); for (const indicator of logoutIndicators) { const location = await this.templateMatching.findElement(indicator); if (location) { detectedIndicators.push(indicator.id); highestConfidence = Math.max(highestConfidence, location.confidence); this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence }); } } } this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence }); return { isLoggedIn, confidence: highestConfidence, detectedIndicators, }; } catch (error) { const errorMsg = `Login state detection failed: ${error}`; this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg)); return { isLoggedIn: false, confidence: 0, detectedIndicators: [], error: errorMsg, }; } } async findElement(template: ImageTemplate): Promise { return this.templateMatching.findElement(template); } async focusBrowserWindow(titlePattern?: string): Promise { return this.windowFocus.focusBrowserWindow(titlePattern); } async captureScreen(region?: ScreenRegion): Promise { if (region) { return this.screenRecognition.captureRegion(region); } return this.screenRecognition.captureFullScreen(); } async clickAtLocation(location: ElementLocation): Promise { try { const point = new Point(location.center.x, location.center.y); await mouse.move([point]); await mouse.leftClick(); this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y }); return { success: true, target: `${location.center.x},${location.center.y}` }; } catch (error) { return { success: false, target: `${location.center.x},${location.center.y}`, error: String(error) }; } } async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise { const timeout = maxWaitMs ?? this.config.defaultTimeout; const result = await this.templateMatching.waitForTemplate(template, timeout); return { success: result.found, target: template.id, waitedMs: result.searchDurationMs, found: result.found, error: result.error, }; } async executeStep(stepId: StepId, config: Record): Promise { const stepNumber = stepId.value; const startTime = Date.now(); this.logger.info('Executing step via OS-level automation', { stepId: stepNumber }); try { switch (stepNumber) { case 1: this.logger.debug('Skipping login step - user pre-authenticated', { stepId: stepNumber }); return { success: true, metadata: { skipped: true, reason: 'User pre-authenticated', step: 'LOGIN', }, }; case 18: this.logger.info('Safety stop at final step', { stepId: stepNumber }); return { success: true, metadata: { step: 'TRACK_CONDITIONS', safetyStop: true, message: 'Automation stopped at final step. User must review configuration and click checkout manually.', }, }; default: { const durationMs = Date.now() - startTime; this.logger.info('Step executed successfully', { stepId: stepNumber, durationMs }); return { success: true, metadata: { step: `STEP_${stepNumber}`, message: `Step ${stepNumber} executed via OS-level automation`, config, }, }; } } } catch (error) { const durationMs = Date.now() - startTime; this.logger.error('Step execution failed', error instanceof Error ? error : new Error(String(error)), { stepId: stepNumber, durationMs }); return { success: false, error: String(error), metadata: { step: `STEP_${stepNumber}` }, }; } } private parseTarget(target: string): Point { if (target.includes(',')) { const [x, y] = target.split(',').map(Number); return new Point(x, y); } return new Point( Math.floor(this.config.screenResolution.width / 2), Math.floor(this.config.screenResolution.height / 2) ); } private delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } }