655 lines
21 KiB
TypeScript
655 lines
21 KiB
TypeScript
import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js';
|
|
import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation';
|
|
import {
|
|
AutomationResult,
|
|
NavigationResult,
|
|
FormFillResult,
|
|
ClickResult,
|
|
WaitResult,
|
|
ModalResult,
|
|
} from '../../../application/ports/AutomationResults';
|
|
import { StepId } from '../../../domain/value-objects/StepId';
|
|
import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate';
|
|
import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion';
|
|
import type { ILogger } from '../../../application/ports/ILogger';
|
|
import { NoOpLogAdapter } from '../logging/NoOpLogAdapter';
|
|
import { ScreenRecognitionService } from './ScreenRecognitionService';
|
|
import { TemplateMatchingService } from './TemplateMatchingService';
|
|
import { WindowFocusService } from './WindowFocusService';
|
|
import { getLoginIndicators, getLogoutIndicators, getStepTemplates, getStepName, type StepTemplates } from './templates/IRacingTemplateMap';
|
|
|
|
export interface NutJsConfig {
|
|
mouseSpeed?: number;
|
|
keyboardDelay?: number;
|
|
screenResolution?: { width: number; height: number };
|
|
defaultTimeout?: number;
|
|
templatePath?: string;
|
|
windowTitle?: string;
|
|
}
|
|
|
|
export class NutJsAutomationAdapter implements IScreenAutomation {
|
|
private config: Required<NutJsConfig>;
|
|
private connected: boolean = false;
|
|
private logger: ILogger;
|
|
private screenRecognition: ScreenRecognitionService;
|
|
private templateMatching: TemplateMatchingService;
|
|
private windowFocus: WindowFocusService;
|
|
|
|
constructor(config: NutJsConfig = {}, logger?: ILogger) {
|
|
this.config = {
|
|
mouseSpeed: config.mouseSpeed ?? 1000,
|
|
keyboardDelay: config.keyboardDelay ?? 50,
|
|
screenResolution: config.screenResolution ?? { width: 1920, height: 1080 },
|
|
defaultTimeout: config.defaultTimeout ?? 30000,
|
|
templatePath: config.templatePath ?? './resources/templates/iracing',
|
|
windowTitle: config.windowTitle ?? 'iRacing',
|
|
};
|
|
this.logger = logger ?? new NoOpLogAdapter();
|
|
|
|
mouse.config.mouseSpeed = this.config.mouseSpeed;
|
|
keyboard.config.autoDelayMs = this.config.keyboardDelay;
|
|
|
|
this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' }));
|
|
this.templateMatching = new TemplateMatchingService(
|
|
this.config.templatePath,
|
|
this.logger.child({ service: 'TemplateMatching' })
|
|
);
|
|
this.windowFocus = new WindowFocusService(
|
|
this.config.windowTitle,
|
|
this.logger.child({ service: 'WindowFocus' })
|
|
);
|
|
}
|
|
|
|
async connect(): Promise<AutomationResult> {
|
|
const startTime = Date.now();
|
|
this.logger.info('Initializing nut.js OS-level automation');
|
|
|
|
try {
|
|
const width = await screen.width();
|
|
const height = await screen.height();
|
|
this.connected = true;
|
|
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.info('nut.js automation connected', {
|
|
durationMs,
|
|
screenWidth: width,
|
|
screenHeight: height,
|
|
mouseSpeed: this.config.mouseSpeed,
|
|
keyboardDelay: this.config.keyboardDelay
|
|
});
|
|
return { success: true };
|
|
} catch (error) {
|
|
const errorMsg = `Screen access failed: ${error}`;
|
|
this.logger.error('Failed to initialize nut.js', error instanceof Error ? error : new Error(errorMsg));
|
|
return { success: false, error: errorMsg };
|
|
}
|
|
}
|
|
|
|
async navigateToPage(url: string): Promise<NavigationResult> {
|
|
const startTime = Date.now();
|
|
try {
|
|
const isMac = process.platform === 'darwin';
|
|
|
|
if (isMac) {
|
|
await keyboard.pressKey(Key.LeftSuper, Key.L);
|
|
await keyboard.releaseKey(Key.LeftSuper, Key.L);
|
|
} else {
|
|
await keyboard.pressKey(Key.LeftControl, Key.L);
|
|
await keyboard.releaseKey(Key.LeftControl, Key.L);
|
|
}
|
|
|
|
await this.delay(100);
|
|
|
|
await keyboard.type(url);
|
|
|
|
await keyboard.pressKey(Key.Enter);
|
|
await keyboard.releaseKey(Key.Enter);
|
|
|
|
await this.delay(2000);
|
|
|
|
return { success: true, url, loadTime: Date.now() - startTime };
|
|
} catch (error) {
|
|
return { success: false, url, loadTime: 0, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async fillFormField(fieldName: string, value: string): Promise<FormFillResult> {
|
|
try {
|
|
const isMac = process.platform === 'darwin';
|
|
|
|
if (isMac) {
|
|
await keyboard.pressKey(Key.LeftSuper, Key.A);
|
|
await keyboard.releaseKey(Key.LeftSuper, Key.A);
|
|
} else {
|
|
await keyboard.pressKey(Key.LeftControl, Key.A);
|
|
await keyboard.releaseKey(Key.LeftControl, Key.A);
|
|
}
|
|
|
|
await this.delay(50);
|
|
|
|
await keyboard.type(value);
|
|
|
|
return { success: true, fieldName, valueSet: value };
|
|
} catch (error) {
|
|
return { success: false, fieldName, valueSet: '', error: String(error) };
|
|
}
|
|
}
|
|
|
|
async clickElement(target: string): Promise<ClickResult> {
|
|
try {
|
|
const point = this.parseTarget(target);
|
|
await mouse.move([point]);
|
|
await mouse.leftClick();
|
|
return { success: true, target };
|
|
} catch (error) {
|
|
return { success: false, target, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async waitForElement(target: string, maxWaitMs?: number): Promise<WaitResult> {
|
|
const startTime = Date.now();
|
|
const timeout = maxWaitMs ?? this.config.defaultTimeout;
|
|
|
|
await this.delay(Math.min(1000, timeout));
|
|
|
|
return {
|
|
success: true,
|
|
target,
|
|
waitedMs: Date.now() - startTime,
|
|
found: true,
|
|
};
|
|
}
|
|
|
|
async handleModal(stepId: StepId, action: string): Promise<ModalResult> {
|
|
try {
|
|
if (action === 'confirm') {
|
|
await keyboard.pressKey(Key.Enter);
|
|
await keyboard.releaseKey(Key.Enter);
|
|
} else if (action === 'cancel') {
|
|
await keyboard.pressKey(Key.Escape);
|
|
await keyboard.releaseKey(Key.Escape);
|
|
}
|
|
|
|
return { success: true, stepId: stepId.value, action };
|
|
} catch (error) {
|
|
return { success: false, stepId: stepId.value, action, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async disconnect(): Promise<void> {
|
|
this.logger.info('Disconnecting nut.js automation');
|
|
this.connected = false;
|
|
this.logger.debug('nut.js disconnected');
|
|
}
|
|
|
|
isConnected(): boolean {
|
|
return this.connected;
|
|
}
|
|
|
|
async detectLoginState(): Promise<LoginDetectionResult> {
|
|
this.logger.debug('Detecting login state via template matching');
|
|
|
|
try {
|
|
const detectedIndicators: string[] = [];
|
|
let isLoggedIn = false;
|
|
let highestConfidence = 0;
|
|
|
|
const loginIndicators = getLoginIndicators();
|
|
for (const indicator of loginIndicators) {
|
|
const location = await this.templateMatching.findElement(indicator);
|
|
if (location) {
|
|
detectedIndicators.push(indicator.id);
|
|
isLoggedIn = true;
|
|
highestConfidence = Math.max(highestConfidence, location.confidence);
|
|
this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence });
|
|
}
|
|
}
|
|
|
|
if (!isLoggedIn) {
|
|
const logoutIndicators = getLogoutIndicators();
|
|
for (const indicator of logoutIndicators) {
|
|
const location = await this.templateMatching.findElement(indicator);
|
|
if (location) {
|
|
detectedIndicators.push(indicator.id);
|
|
highestConfidence = Math.max(highestConfidence, location.confidence);
|
|
this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence });
|
|
}
|
|
}
|
|
}
|
|
|
|
this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence });
|
|
|
|
return {
|
|
isLoggedIn,
|
|
confidence: highestConfidence,
|
|
detectedIndicators,
|
|
};
|
|
} catch (error) {
|
|
const errorMsg = `Login state detection failed: ${error}`;
|
|
this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg));
|
|
|
|
return {
|
|
isLoggedIn: false,
|
|
confidence: 0,
|
|
detectedIndicators: [],
|
|
error: errorMsg,
|
|
};
|
|
}
|
|
}
|
|
|
|
async findElement(template: ImageTemplate): Promise<ElementLocation | null> {
|
|
return this.templateMatching.findElement(template);
|
|
}
|
|
|
|
async focusBrowserWindow(titlePattern?: string): Promise<WindowFocusResult> {
|
|
return this.windowFocus.focusBrowserWindow(titlePattern);
|
|
}
|
|
|
|
async captureScreen(region?: ScreenRegion): Promise<ScreenCaptureResult> {
|
|
if (region) {
|
|
return this.screenRecognition.captureRegion(region);
|
|
}
|
|
return this.screenRecognition.captureFullScreen();
|
|
}
|
|
|
|
async clickAtLocation(location: ElementLocation): Promise<ClickResult> {
|
|
try {
|
|
const point = new Point(location.center.x, location.center.y);
|
|
await mouse.move([point]);
|
|
await mouse.leftClick();
|
|
|
|
this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y });
|
|
|
|
return {
|
|
success: true,
|
|
target: `${location.center.x},${location.center.y}`
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
target: `${location.center.x},${location.center.y}`,
|
|
error: String(error)
|
|
};
|
|
}
|
|
}
|
|
|
|
async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise<WaitResult> {
|
|
const timeout = maxWaitMs ?? this.config.defaultTimeout;
|
|
const result = await this.templateMatching.waitForTemplate(template, timeout);
|
|
|
|
return {
|
|
success: result.found,
|
|
target: template.id,
|
|
waitedMs: result.searchDurationMs,
|
|
found: result.found,
|
|
error: result.error,
|
|
};
|
|
}
|
|
|
|
async executeStep(stepId: StepId, config: Record<string, unknown>): Promise<AutomationResult> {
|
|
const stepNumber = stepId.value;
|
|
const startTime = Date.now();
|
|
const stepName = getStepName(stepNumber);
|
|
|
|
this.logger.info('Executing step via OS-level automation', { stepId: stepNumber, stepName });
|
|
|
|
try {
|
|
// Step 1: LOGIN - Skip (user handles manually)
|
|
if (stepNumber === 1) {
|
|
this.logger.debug('Skipping login step - user pre-authenticated', { stepId: stepNumber });
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
skipped: true,
|
|
reason: 'User pre-authenticated',
|
|
step: stepName,
|
|
},
|
|
};
|
|
}
|
|
|
|
// Step 18: TRACK_CONDITIONS - Safety stop before checkout
|
|
if (stepNumber === 18) {
|
|
this.logger.info('Safety stop at final step', { stepId: stepNumber });
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
step: stepName,
|
|
safetyStop: true,
|
|
message: 'Automation stopped at final step. User must review configuration and click checkout manually.',
|
|
},
|
|
};
|
|
}
|
|
|
|
// Steps 2-17: Real automation
|
|
// 1. Focus browser window
|
|
const focusResult = await this.windowFocus.focusBrowserWindow();
|
|
if (!focusResult.success) {
|
|
this.logger.warn('Failed to focus browser window, continuing anyway', { error: focusResult.error });
|
|
}
|
|
|
|
// Small delay after focusing
|
|
await this.delay(200);
|
|
|
|
// 2. Get templates for this step
|
|
const stepTemplates = getStepTemplates(stepNumber);
|
|
if (!stepTemplates) {
|
|
this.logger.warn('No templates defined for step', { stepId: stepNumber, stepName });
|
|
return {
|
|
success: false,
|
|
error: `No templates defined for step ${stepNumber} (${stepName})`,
|
|
metadata: { step: stepName },
|
|
};
|
|
}
|
|
|
|
// 3. Execute step-specific automation
|
|
const result = await this.executeStepActions(stepNumber, stepName, stepTemplates, config);
|
|
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.info('Step execution completed', { stepId: stepNumber, stepName, durationMs, success: result.success });
|
|
|
|
return {
|
|
...result,
|
|
metadata: {
|
|
...result.metadata,
|
|
step: stepName,
|
|
durationMs,
|
|
},
|
|
};
|
|
} catch (error) {
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.error('Step execution failed', error instanceof Error ? error : new Error(String(error)), {
|
|
stepId: stepNumber,
|
|
stepName,
|
|
durationMs
|
|
});
|
|
return {
|
|
success: false,
|
|
error: String(error),
|
|
metadata: { step: stepName, durationMs },
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute step-specific actions based on the step number.
|
|
*/
|
|
private async executeStepActions(
|
|
stepNumber: number,
|
|
stepName: string,
|
|
templates: StepTemplates,
|
|
config: Record<string, unknown>
|
|
): Promise<AutomationResult> {
|
|
switch (stepNumber) {
|
|
// Step 2: HOSTED_RACING - Click "Create a Race" button
|
|
case 2:
|
|
return this.executeClickStep(templates, 'createRace', 'Navigate to hosted racing');
|
|
|
|
// Step 3: CREATE_RACE - Confirm race creation modal
|
|
case 3:
|
|
return this.executeClickStep(templates, 'confirm', 'Confirm race creation');
|
|
|
|
// Step 4: RACE_INFORMATION - Fill session name and details, then click next
|
|
case 4:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'sessionName', configKey: 'sessionName' },
|
|
{ field: 'password', configKey: 'sessionPassword' },
|
|
{ field: 'description', configKey: 'description' },
|
|
], 'next');
|
|
|
|
// Step 5: SERVER_DETAILS - Configure server settings, then click next
|
|
case 5:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'region', configKey: 'serverRegion' },
|
|
], 'next');
|
|
|
|
// Step 6: SET_ADMINS - Modal step for adding admins
|
|
case 6:
|
|
return this.executeModalStep(templates, config, 'adminName', 'next');
|
|
|
|
// Step 7: TIME_LIMITS - Fill time fields
|
|
case 7:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'practice', configKey: 'practiceLength' },
|
|
{ field: 'qualify', configKey: 'qualifyLength' },
|
|
{ field: 'race', configKey: 'raceLength' },
|
|
], 'next');
|
|
|
|
// Step 8: SET_CARS - Click add car button
|
|
case 8:
|
|
return this.executeClickStep(templates, 'addCar', 'Open car selection');
|
|
|
|
// Step 9: ADD_CAR - Modal for car selection
|
|
case 9:
|
|
return this.executeModalStep(templates, config, 'carName', 'select');
|
|
|
|
// Step 10: SET_CAR_CLASSES - Configure car classes
|
|
case 10:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'class', configKey: 'carClass' },
|
|
], 'next');
|
|
|
|
// Step 11: SET_TRACK - Click add track button
|
|
case 11:
|
|
return this.executeClickStep(templates, 'addTrack', 'Open track selection');
|
|
|
|
// Step 12: ADD_TRACK - Modal for track selection
|
|
case 12:
|
|
return this.executeModalStep(templates, config, 'trackName', 'select');
|
|
|
|
// Step 13: TRACK_OPTIONS - Configure track options
|
|
case 13:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'config', configKey: 'trackConfig' },
|
|
], 'next');
|
|
|
|
// Step 14: TIME_OF_DAY - Configure time settings
|
|
case 14:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'time', configKey: 'timeOfDay' },
|
|
{ field: 'date', configKey: 'raceDate' },
|
|
], 'next');
|
|
|
|
// Step 15: WEATHER - Configure weather settings
|
|
case 15:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'weather', configKey: 'weatherType' },
|
|
{ field: 'temperature', configKey: 'temperature' },
|
|
], 'next');
|
|
|
|
// Step 16: RACE_OPTIONS - Configure race options
|
|
case 16:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'maxDrivers', configKey: 'maxDrivers' },
|
|
{ field: 'rollingStart', configKey: 'rollingStart' },
|
|
], 'next');
|
|
|
|
// Step 17: TEAM_DRIVING - Configure team settings
|
|
case 17:
|
|
return this.executeFormStep(templates, config, [
|
|
{ field: 'teamDriving', configKey: 'teamDriving' },
|
|
], 'next');
|
|
|
|
default:
|
|
this.logger.warn('Unhandled step number', { stepNumber, stepName });
|
|
return {
|
|
success: false,
|
|
error: `No automation handler for step ${stepNumber} (${stepName})`,
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute a simple click step - find and click a button.
|
|
*/
|
|
private async executeClickStep(
|
|
templates: StepTemplates,
|
|
buttonKey: string,
|
|
actionDescription: string
|
|
): Promise<AutomationResult> {
|
|
const buttonTemplate = templates.buttons[buttonKey];
|
|
|
|
if (!buttonTemplate) {
|
|
this.logger.warn('Button template not defined', { buttonKey });
|
|
return {
|
|
success: false,
|
|
error: `Button template '${buttonKey}' not defined for this step`,
|
|
};
|
|
}
|
|
|
|
// Find the button on screen
|
|
const location = await this.templateMatching.findElement(buttonTemplate);
|
|
|
|
if (!location) {
|
|
this.logger.warn('Button not found on screen', {
|
|
buttonKey,
|
|
templateId: buttonTemplate.id,
|
|
description: buttonTemplate.description
|
|
});
|
|
return {
|
|
success: false,
|
|
error: `Button '${buttonKey}' not found on screen (template: ${buttonTemplate.id})`,
|
|
metadata: { templateId: buttonTemplate.id, action: actionDescription },
|
|
};
|
|
}
|
|
|
|
// Click the button
|
|
const clickResult = await this.clickAtLocation(location);
|
|
|
|
if (!clickResult.success) {
|
|
return {
|
|
success: false,
|
|
error: `Failed to click button '${buttonKey}': ${clickResult.error}`,
|
|
metadata: { templateId: buttonTemplate.id, action: actionDescription },
|
|
};
|
|
}
|
|
|
|
// Small delay after clicking
|
|
await this.delay(300);
|
|
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
action: actionDescription,
|
|
templateId: buttonTemplate.id,
|
|
clickLocation: location.center,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Execute a form step - fill fields and click next.
|
|
*/
|
|
private async executeFormStep(
|
|
templates: StepTemplates,
|
|
config: Record<string, unknown>,
|
|
fieldMappings: Array<{ field: string; configKey: string }>,
|
|
nextButtonKey: string
|
|
): Promise<AutomationResult> {
|
|
const filledFields: string[] = [];
|
|
const skippedFields: string[] = [];
|
|
|
|
// Process each field mapping
|
|
for (const mapping of fieldMappings) {
|
|
const fieldTemplate = templates.fields?.[mapping.field];
|
|
const configValue = config[mapping.configKey];
|
|
|
|
// Skip if no value provided in config
|
|
if (configValue === undefined || configValue === null || configValue === '') {
|
|
skippedFields.push(mapping.field);
|
|
continue;
|
|
}
|
|
|
|
// Skip if no template defined
|
|
if (!fieldTemplate) {
|
|
this.logger.debug('Field template not defined, skipping', { field: mapping.field });
|
|
skippedFields.push(mapping.field);
|
|
continue;
|
|
}
|
|
|
|
// Find the field on screen
|
|
const location = await this.templateMatching.findElement(fieldTemplate);
|
|
|
|
if (!location) {
|
|
this.logger.warn('Field not found on screen', {
|
|
field: mapping.field,
|
|
templateId: fieldTemplate.id
|
|
});
|
|
skippedFields.push(mapping.field);
|
|
continue;
|
|
}
|
|
|
|
// Click the field to focus it
|
|
await this.clickAtLocation(location);
|
|
await this.delay(100);
|
|
|
|
// Fill the field
|
|
const fillResult = await this.fillFormField(mapping.field, String(configValue));
|
|
|
|
if (fillResult.success) {
|
|
filledFields.push(mapping.field);
|
|
} else {
|
|
this.logger.warn('Failed to fill field', { field: mapping.field, error: fillResult.error });
|
|
skippedFields.push(mapping.field);
|
|
}
|
|
|
|
await this.delay(100);
|
|
}
|
|
|
|
// Click the next button
|
|
const nextResult = await this.executeClickStep(templates, nextButtonKey, 'Proceed to next step');
|
|
|
|
return {
|
|
success: nextResult.success,
|
|
error: nextResult.error,
|
|
metadata: {
|
|
...nextResult.metadata,
|
|
filledFields,
|
|
skippedFields,
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Execute a modal step - interact with a modal dialog.
|
|
*/
|
|
private async executeModalStep(
|
|
templates: StepTemplates,
|
|
config: Record<string, unknown>,
|
|
searchConfigKey: string,
|
|
confirmButtonKey: string
|
|
): Promise<AutomationResult> {
|
|
// If modal has search input and we have a search value, use it
|
|
if (templates.modal?.searchInput) {
|
|
const searchValue = config[searchConfigKey];
|
|
|
|
if (searchValue && typeof searchValue === 'string') {
|
|
const searchLocation = await this.templateMatching.findElement(templates.modal.searchInput);
|
|
|
|
if (searchLocation) {
|
|
await this.clickAtLocation(searchLocation);
|
|
await this.delay(100);
|
|
await this.fillFormField('search', searchValue);
|
|
await this.delay(500); // Wait for search results
|
|
}
|
|
}
|
|
}
|
|
|
|
// Click the confirm/select button
|
|
return this.executeClickStep(templates, confirmButtonKey, 'Confirm modal selection');
|
|
}
|
|
|
|
private parseTarget(target: string): Point {
|
|
if (target.includes(',')) {
|
|
const [x, y] = target.split(',').map(Number);
|
|
return new Point(x, y);
|
|
}
|
|
return new Point(
|
|
Math.floor(this.config.screenResolution.width / 2),
|
|
Math.floor(this.config.screenResolution.height / 2)
|
|
);
|
|
}
|
|
|
|
private delay(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
} |