import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js'; import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation'; import { AutomationResult, NavigationResult, FormFillResult, ClickResult, WaitResult, ModalResult, } from '../../../application/ports/AutomationResults'; import { StepId } from '../../../domain/value-objects/StepId'; import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate'; import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion'; import type { ILogger } from '../../../application/ports/ILogger'; import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; import { ScreenRecognitionService } from './ScreenRecognitionService'; import { TemplateMatchingService } from './TemplateMatchingService'; import { WindowFocusService } from './WindowFocusService'; import { getLoginIndicators, getLogoutIndicators, getStepTemplates, getStepName, type StepTemplates } from './templates/IRacingTemplateMap'; export interface NutJsConfig { mouseSpeed?: number; keyboardDelay?: number; screenResolution?: { width: number; height: number }; defaultTimeout?: number; templatePath?: string; windowTitle?: string; } export class NutJsAutomationAdapter implements IScreenAutomation { private config: Required; private connected: boolean = false; private logger: ILogger; private screenRecognition: ScreenRecognitionService; private templateMatching: TemplateMatchingService; private windowFocus: WindowFocusService; constructor(config: NutJsConfig = {}, logger?: ILogger) { this.config = { mouseSpeed: config.mouseSpeed ?? 1000, keyboardDelay: config.keyboardDelay ?? 50, screenResolution: config.screenResolution ?? { width: 1920, height: 1080 }, defaultTimeout: config.defaultTimeout ?? 30000, templatePath: config.templatePath ?? './resources/templates/iracing', windowTitle: config.windowTitle ?? 'iRacing', }; this.logger = logger ?? new NoOpLogAdapter(); mouse.config.mouseSpeed = this.config.mouseSpeed; keyboard.config.autoDelayMs = this.config.keyboardDelay; this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' })); this.templateMatching = new TemplateMatchingService( this.config.templatePath, this.logger.child({ service: 'TemplateMatching' }) ); this.windowFocus = new WindowFocusService( this.config.windowTitle, this.logger.child({ service: 'WindowFocus' }) ); } async connect(): Promise { const startTime = Date.now(); this.logger.info('Initializing nut.js OS-level automation'); try { const width = await screen.width(); const height = await screen.height(); this.connected = true; const durationMs = Date.now() - startTime; this.logger.info('nut.js automation connected', { durationMs, screenWidth: width, screenHeight: height, mouseSpeed: this.config.mouseSpeed, keyboardDelay: this.config.keyboardDelay }); return { success: true }; } catch (error) { const errorMsg = `Screen access failed: ${error}`; this.logger.error('Failed to initialize nut.js', error instanceof Error ? error : new Error(errorMsg)); return { success: false, error: errorMsg }; } } async navigateToPage(url: string): Promise { const startTime = Date.now(); try { const isMac = process.platform === 'darwin'; if (isMac) { await keyboard.pressKey(Key.LeftSuper, Key.L); await keyboard.releaseKey(Key.LeftSuper, Key.L); } else { await keyboard.pressKey(Key.LeftControl, Key.L); await keyboard.releaseKey(Key.LeftControl, Key.L); } await this.delay(100); await keyboard.type(url); await keyboard.pressKey(Key.Enter); await keyboard.releaseKey(Key.Enter); await this.delay(2000); return { success: true, url, loadTime: Date.now() - startTime }; } catch (error) { return { success: false, url, loadTime: 0, error: String(error) }; } } async fillFormField(fieldName: string, value: string): Promise { try { const isMac = process.platform === 'darwin'; if (isMac) { await keyboard.pressKey(Key.LeftSuper, Key.A); await keyboard.releaseKey(Key.LeftSuper, Key.A); } else { await keyboard.pressKey(Key.LeftControl, Key.A); await keyboard.releaseKey(Key.LeftControl, Key.A); } await this.delay(50); await keyboard.type(value); return { success: true, fieldName, valueSet: value }; } catch (error) { return { success: false, fieldName, valueSet: '', error: String(error) }; } } async clickElement(target: string): Promise { try { const point = this.parseTarget(target); await mouse.move([point]); await mouse.leftClick(); return { success: true, target }; } catch (error) { return { success: false, target, error: String(error) }; } } async waitForElement(target: string, maxWaitMs?: number): Promise { const startTime = Date.now(); const timeout = maxWaitMs ?? this.config.defaultTimeout; await this.delay(Math.min(1000, timeout)); return { success: true, target, waitedMs: Date.now() - startTime, found: true, }; } async handleModal(stepId: StepId, action: string): Promise { try { if (action === 'confirm') { await keyboard.pressKey(Key.Enter); await keyboard.releaseKey(Key.Enter); } else if (action === 'cancel') { await keyboard.pressKey(Key.Escape); await keyboard.releaseKey(Key.Escape); } return { success: true, stepId: stepId.value, action }; } catch (error) { return { success: false, stepId: stepId.value, action, error: String(error) }; } } async disconnect(): Promise { this.logger.info('Disconnecting nut.js automation'); this.connected = false; this.logger.debug('nut.js disconnected'); } isConnected(): boolean { return this.connected; } async detectLoginState(): Promise { this.logger.debug('Detecting login state via template matching'); try { const detectedIndicators: string[] = []; let isLoggedIn = false; let highestConfidence = 0; const loginIndicators = getLoginIndicators(); for (const indicator of loginIndicators) { const location = await this.templateMatching.findElement(indicator); if (location) { detectedIndicators.push(indicator.id); isLoggedIn = true; highestConfidence = Math.max(highestConfidence, location.confidence); this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence }); } } if (!isLoggedIn) { const logoutIndicators = getLogoutIndicators(); for (const indicator of logoutIndicators) { const location = await this.templateMatching.findElement(indicator); if (location) { detectedIndicators.push(indicator.id); highestConfidence = Math.max(highestConfidence, location.confidence); this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence }); } } } this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence }); return { isLoggedIn, confidence: highestConfidence, detectedIndicators, }; } catch (error) { const errorMsg = `Login state detection failed: ${error}`; this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg)); return { isLoggedIn: false, confidence: 0, detectedIndicators: [], error: errorMsg, }; } } async findElement(template: ImageTemplate): Promise { return this.templateMatching.findElement(template); } async focusBrowserWindow(titlePattern?: string): Promise { return this.windowFocus.focusBrowserWindow(titlePattern); } async captureScreen(region?: ScreenRegion): Promise { if (region) { return this.screenRecognition.captureRegion(region); } return this.screenRecognition.captureFullScreen(); } async clickAtLocation(location: ElementLocation): Promise { try { const point = new Point(location.center.x, location.center.y); await mouse.move([point]); await mouse.leftClick(); this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y }); return { success: true, target: `${location.center.x},${location.center.y}` }; } catch (error) { return { success: false, target: `${location.center.x},${location.center.y}`, error: String(error) }; } } async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise { const timeout = maxWaitMs ?? this.config.defaultTimeout; const result = await this.templateMatching.waitForTemplate(template, timeout); return { success: result.found, target: template.id, waitedMs: result.searchDurationMs, found: result.found, error: result.error, }; } async executeStep(stepId: StepId, config: Record): Promise { const stepNumber = stepId.value; const startTime = Date.now(); const stepName = getStepName(stepNumber); this.logger.info('Executing step via OS-level automation', { stepId: stepNumber, stepName }); try { // Step 1: LOGIN - Skip (user handles manually) if (stepNumber === 1) { this.logger.debug('Skipping login step - user pre-authenticated', { stepId: stepNumber }); return { success: true, metadata: { skipped: true, reason: 'User pre-authenticated', step: stepName, }, }; } // Step 18: TRACK_CONDITIONS - Safety stop before checkout if (stepNumber === 18) { this.logger.info('Safety stop at final step', { stepId: stepNumber }); return { success: true, metadata: { step: stepName, safetyStop: true, message: 'Automation stopped at final step. User must review configuration and click checkout manually.', }, }; } // Steps 2-17: Real automation // 1. Focus browser window const focusResult = await this.windowFocus.focusBrowserWindow(); if (!focusResult.success) { this.logger.warn('Failed to focus browser window, continuing anyway', { error: focusResult.error }); } // Small delay after focusing await this.delay(200); // 2. Get templates for this step const stepTemplates = getStepTemplates(stepNumber); if (!stepTemplates) { this.logger.warn('No templates defined for step', { stepId: stepNumber, stepName }); return { success: false, error: `No templates defined for step ${stepNumber} (${stepName})`, metadata: { step: stepName }, }; } // 3. Execute step-specific automation const result = await this.executeStepActions(stepNumber, stepName, stepTemplates, config); const durationMs = Date.now() - startTime; this.logger.info('Step execution completed', { stepId: stepNumber, stepName, durationMs, success: result.success }); return { ...result, metadata: { ...result.metadata, step: stepName, durationMs, }, }; } catch (error) { const durationMs = Date.now() - startTime; this.logger.error('Step execution failed', error instanceof Error ? error : new Error(String(error)), { stepId: stepNumber, stepName, durationMs }); return { success: false, error: String(error), metadata: { step: stepName, durationMs }, }; } } /** * Execute step-specific actions based on the step number. */ private async executeStepActions( stepNumber: number, stepName: string, templates: StepTemplates, config: Record ): Promise { switch (stepNumber) { // Step 2: HOSTED_RACING - Click "Create a Race" button case 2: return this.executeClickStep(templates, 'createRace', 'Navigate to hosted racing'); // Step 3: CREATE_RACE - Confirm race creation modal case 3: return this.executeClickStep(templates, 'confirm', 'Confirm race creation'); // Step 4: RACE_INFORMATION - Fill session name and details, then click next case 4: return this.executeFormStep(templates, config, [ { field: 'sessionName', configKey: 'sessionName' }, { field: 'password', configKey: 'sessionPassword' }, { field: 'description', configKey: 'description' }, ], 'next'); // Step 5: SERVER_DETAILS - Configure server settings, then click next case 5: return this.executeFormStep(templates, config, [ { field: 'region', configKey: 'serverRegion' }, ], 'next'); // Step 6: SET_ADMINS - Modal step for adding admins case 6: return this.executeModalStep(templates, config, 'adminName', 'next'); // Step 7: TIME_LIMITS - Fill time fields case 7: return this.executeFormStep(templates, config, [ { field: 'practice', configKey: 'practiceLength' }, { field: 'qualify', configKey: 'qualifyLength' }, { field: 'race', configKey: 'raceLength' }, ], 'next'); // Step 8: SET_CARS - Click add car button case 8: return this.executeClickStep(templates, 'addCar', 'Open car selection'); // Step 9: ADD_CAR - Modal for car selection case 9: return this.executeModalStep(templates, config, 'carName', 'select'); // Step 10: SET_CAR_CLASSES - Configure car classes case 10: return this.executeFormStep(templates, config, [ { field: 'class', configKey: 'carClass' }, ], 'next'); // Step 11: SET_TRACK - Click add track button case 11: return this.executeClickStep(templates, 'addTrack', 'Open track selection'); // Step 12: ADD_TRACK - Modal for track selection case 12: return this.executeModalStep(templates, config, 'trackName', 'select'); // Step 13: TRACK_OPTIONS - Configure track options case 13: return this.executeFormStep(templates, config, [ { field: 'config', configKey: 'trackConfig' }, ], 'next'); // Step 14: TIME_OF_DAY - Configure time settings case 14: return this.executeFormStep(templates, config, [ { field: 'time', configKey: 'timeOfDay' }, { field: 'date', configKey: 'raceDate' }, ], 'next'); // Step 15: WEATHER - Configure weather settings case 15: return this.executeFormStep(templates, config, [ { field: 'weather', configKey: 'weatherType' }, { field: 'temperature', configKey: 'temperature' }, ], 'next'); // Step 16: RACE_OPTIONS - Configure race options case 16: return this.executeFormStep(templates, config, [ { field: 'maxDrivers', configKey: 'maxDrivers' }, { field: 'rollingStart', configKey: 'rollingStart' }, ], 'next'); // Step 17: TEAM_DRIVING - Configure team settings case 17: return this.executeFormStep(templates, config, [ { field: 'teamDriving', configKey: 'teamDriving' }, ], 'next'); default: this.logger.warn('Unhandled step number', { stepNumber, stepName }); return { success: false, error: `No automation handler for step ${stepNumber} (${stepName})`, }; } } /** * Execute a simple click step - find and click a button. */ private async executeClickStep( templates: StepTemplates, buttonKey: string, actionDescription: string ): Promise { const buttonTemplate = templates.buttons[buttonKey]; if (!buttonTemplate) { this.logger.warn('Button template not defined', { buttonKey }); return { success: false, error: `Button template '${buttonKey}' not defined for this step`, }; } // Find the button on screen const location = await this.templateMatching.findElement(buttonTemplate); if (!location) { this.logger.warn('Button not found on screen', { buttonKey, templateId: buttonTemplate.id, description: buttonTemplate.description }); return { success: false, error: `Button '${buttonKey}' not found on screen (template: ${buttonTemplate.id})`, metadata: { templateId: buttonTemplate.id, action: actionDescription }, }; } // Click the button const clickResult = await this.clickAtLocation(location); if (!clickResult.success) { return { success: false, error: `Failed to click button '${buttonKey}': ${clickResult.error}`, metadata: { templateId: buttonTemplate.id, action: actionDescription }, }; } // Small delay after clicking await this.delay(300); return { success: true, metadata: { action: actionDescription, templateId: buttonTemplate.id, clickLocation: location.center, }, }; } /** * Execute a form step - fill fields and click next. */ private async executeFormStep( templates: StepTemplates, config: Record, fieldMappings: Array<{ field: string; configKey: string }>, nextButtonKey: string ): Promise { const filledFields: string[] = []; const skippedFields: string[] = []; // Process each field mapping for (const mapping of fieldMappings) { const fieldTemplate = templates.fields?.[mapping.field]; const configValue = config[mapping.configKey]; // Skip if no value provided in config if (configValue === undefined || configValue === null || configValue === '') { skippedFields.push(mapping.field); continue; } // Skip if no template defined if (!fieldTemplate) { this.logger.debug('Field template not defined, skipping', { field: mapping.field }); skippedFields.push(mapping.field); continue; } // Find the field on screen const location = await this.templateMatching.findElement(fieldTemplate); if (!location) { this.logger.warn('Field not found on screen', { field: mapping.field, templateId: fieldTemplate.id }); skippedFields.push(mapping.field); continue; } // Click the field to focus it await this.clickAtLocation(location); await this.delay(100); // Fill the field const fillResult = await this.fillFormField(mapping.field, String(configValue)); if (fillResult.success) { filledFields.push(mapping.field); } else { this.logger.warn('Failed to fill field', { field: mapping.field, error: fillResult.error }); skippedFields.push(mapping.field); } await this.delay(100); } // Click the next button const nextResult = await this.executeClickStep(templates, nextButtonKey, 'Proceed to next step'); return { success: nextResult.success, error: nextResult.error, metadata: { ...nextResult.metadata, filledFields, skippedFields, }, }; } /** * Execute a modal step - interact with a modal dialog. */ private async executeModalStep( templates: StepTemplates, config: Record, searchConfigKey: string, confirmButtonKey: string ): Promise { // If modal has search input and we have a search value, use it if (templates.modal?.searchInput) { const searchValue = config[searchConfigKey]; if (searchValue && typeof searchValue === 'string') { const searchLocation = await this.templateMatching.findElement(templates.modal.searchInput); if (searchLocation) { await this.clickAtLocation(searchLocation); await this.delay(100); await this.fillFormField('search', searchValue); await this.delay(500); // Wait for search results } } } // Click the confirm/select button return this.executeClickStep(templates, confirmButtonKey, 'Confirm modal selection'); } private parseTarget(target: string): Point { if (target.includes(',')) { const [x, y] = target.split(',').map(Number); return new Point(x, y); } return new Point( Math.floor(this.config.screenResolution.width / 2), Math.floor(this.config.screenResolution.height / 2) ); } private delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } }