360 lines
12 KiB
TypeScript
360 lines
12 KiB
TypeScript
import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js';
|
|
import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation';
|
|
import {
|
|
AutomationResult,
|
|
NavigationResult,
|
|
FormFillResult,
|
|
ClickResult,
|
|
WaitResult,
|
|
ModalResult,
|
|
} from '../../../application/ports/AutomationResults';
|
|
import { StepId } from '../../../domain/value-objects/StepId';
|
|
import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate';
|
|
import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion';
|
|
import type { ILogger } from '../../../application/ports/ILogger';
|
|
import { NoOpLogAdapter } from '../logging/NoOpLogAdapter';
|
|
import { ScreenRecognitionService } from './ScreenRecognitionService';
|
|
import { TemplateMatchingService } from './TemplateMatchingService';
|
|
import { WindowFocusService } from './WindowFocusService';
|
|
import { getLoginIndicators, getLogoutIndicators } from './templates/IRacingTemplateMap';
|
|
|
|
export interface NutJsConfig {
|
|
mouseSpeed?: number;
|
|
keyboardDelay?: number;
|
|
screenResolution?: { width: number; height: number };
|
|
defaultTimeout?: number;
|
|
templatePath?: string;
|
|
windowTitle?: string;
|
|
}
|
|
|
|
export class NutJsAutomationAdapter implements IScreenAutomation {
|
|
private config: Required<NutJsConfig>;
|
|
private connected: boolean = false;
|
|
private logger: ILogger;
|
|
private screenRecognition: ScreenRecognitionService;
|
|
private templateMatching: TemplateMatchingService;
|
|
private windowFocus: WindowFocusService;
|
|
|
|
constructor(config: NutJsConfig = {}, logger?: ILogger) {
|
|
this.config = {
|
|
mouseSpeed: config.mouseSpeed ?? 1000,
|
|
keyboardDelay: config.keyboardDelay ?? 50,
|
|
screenResolution: config.screenResolution ?? { width: 1920, height: 1080 },
|
|
defaultTimeout: config.defaultTimeout ?? 30000,
|
|
templatePath: config.templatePath ?? './resources/templates/iracing',
|
|
windowTitle: config.windowTitle ?? 'iRacing',
|
|
};
|
|
this.logger = logger ?? new NoOpLogAdapter();
|
|
|
|
mouse.config.mouseSpeed = this.config.mouseSpeed;
|
|
keyboard.config.autoDelayMs = this.config.keyboardDelay;
|
|
|
|
this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' }));
|
|
this.templateMatching = new TemplateMatchingService(
|
|
this.config.templatePath,
|
|
this.logger.child({ service: 'TemplateMatching' })
|
|
);
|
|
this.windowFocus = new WindowFocusService(
|
|
this.config.windowTitle,
|
|
this.logger.child({ service: 'WindowFocus' })
|
|
);
|
|
}
|
|
|
|
async connect(): Promise<AutomationResult> {
|
|
const startTime = Date.now();
|
|
this.logger.info('Initializing nut.js OS-level automation');
|
|
|
|
try {
|
|
const width = await screen.width();
|
|
const height = await screen.height();
|
|
this.connected = true;
|
|
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.info('nut.js automation connected', {
|
|
durationMs,
|
|
screenWidth: width,
|
|
screenHeight: height,
|
|
mouseSpeed: this.config.mouseSpeed,
|
|
keyboardDelay: this.config.keyboardDelay
|
|
});
|
|
return { success: true };
|
|
} catch (error) {
|
|
const errorMsg = `Screen access failed: ${error}`;
|
|
this.logger.error('Failed to initialize nut.js', error instanceof Error ? error : new Error(errorMsg));
|
|
return { success: false, error: errorMsg };
|
|
}
|
|
}
|
|
|
|
async navigateToPage(url: string): Promise<NavigationResult> {
|
|
const startTime = Date.now();
|
|
try {
|
|
const isMac = process.platform === 'darwin';
|
|
|
|
if (isMac) {
|
|
await keyboard.pressKey(Key.LeftSuper, Key.L);
|
|
await keyboard.releaseKey(Key.LeftSuper, Key.L);
|
|
} else {
|
|
await keyboard.pressKey(Key.LeftControl, Key.L);
|
|
await keyboard.releaseKey(Key.LeftControl, Key.L);
|
|
}
|
|
|
|
await this.delay(100);
|
|
|
|
await keyboard.type(url);
|
|
|
|
await keyboard.pressKey(Key.Enter);
|
|
await keyboard.releaseKey(Key.Enter);
|
|
|
|
await this.delay(2000);
|
|
|
|
return { success: true, url, loadTime: Date.now() - startTime };
|
|
} catch (error) {
|
|
return { success: false, url, loadTime: 0, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async fillFormField(fieldName: string, value: string): Promise<FormFillResult> {
|
|
try {
|
|
const isMac = process.platform === 'darwin';
|
|
|
|
if (isMac) {
|
|
await keyboard.pressKey(Key.LeftSuper, Key.A);
|
|
await keyboard.releaseKey(Key.LeftSuper, Key.A);
|
|
} else {
|
|
await keyboard.pressKey(Key.LeftControl, Key.A);
|
|
await keyboard.releaseKey(Key.LeftControl, Key.A);
|
|
}
|
|
|
|
await this.delay(50);
|
|
|
|
await keyboard.type(value);
|
|
|
|
return { success: true, fieldName, valueSet: value };
|
|
} catch (error) {
|
|
return { success: false, fieldName, valueSet: '', error: String(error) };
|
|
}
|
|
}
|
|
|
|
async clickElement(target: string): Promise<ClickResult> {
|
|
try {
|
|
const point = this.parseTarget(target);
|
|
await mouse.move([point]);
|
|
await mouse.leftClick();
|
|
return { success: true, target };
|
|
} catch (error) {
|
|
return { success: false, target, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async waitForElement(target: string, maxWaitMs?: number): Promise<WaitResult> {
|
|
const startTime = Date.now();
|
|
const timeout = maxWaitMs ?? this.config.defaultTimeout;
|
|
|
|
await this.delay(Math.min(1000, timeout));
|
|
|
|
return {
|
|
success: true,
|
|
target,
|
|
waitedMs: Date.now() - startTime,
|
|
found: true,
|
|
};
|
|
}
|
|
|
|
async handleModal(stepId: StepId, action: string): Promise<ModalResult> {
|
|
try {
|
|
if (action === 'confirm') {
|
|
await keyboard.pressKey(Key.Enter);
|
|
await keyboard.releaseKey(Key.Enter);
|
|
} else if (action === 'cancel') {
|
|
await keyboard.pressKey(Key.Escape);
|
|
await keyboard.releaseKey(Key.Escape);
|
|
}
|
|
|
|
return { success: true, stepId: stepId.value, action };
|
|
} catch (error) {
|
|
return { success: false, stepId: stepId.value, action, error: String(error) };
|
|
}
|
|
}
|
|
|
|
async disconnect(): Promise<void> {
|
|
this.logger.info('Disconnecting nut.js automation');
|
|
this.connected = false;
|
|
this.logger.debug('nut.js disconnected');
|
|
}
|
|
|
|
isConnected(): boolean {
|
|
return this.connected;
|
|
}
|
|
|
|
async detectLoginState(): Promise<LoginDetectionResult> {
|
|
this.logger.debug('Detecting login state via template matching');
|
|
|
|
try {
|
|
const detectedIndicators: string[] = [];
|
|
let isLoggedIn = false;
|
|
let highestConfidence = 0;
|
|
|
|
const loginIndicators = getLoginIndicators();
|
|
for (const indicator of loginIndicators) {
|
|
const location = await this.templateMatching.findElement(indicator);
|
|
if (location) {
|
|
detectedIndicators.push(indicator.id);
|
|
isLoggedIn = true;
|
|
highestConfidence = Math.max(highestConfidence, location.confidence);
|
|
this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence });
|
|
}
|
|
}
|
|
|
|
if (!isLoggedIn) {
|
|
const logoutIndicators = getLogoutIndicators();
|
|
for (const indicator of logoutIndicators) {
|
|
const location = await this.templateMatching.findElement(indicator);
|
|
if (location) {
|
|
detectedIndicators.push(indicator.id);
|
|
highestConfidence = Math.max(highestConfidence, location.confidence);
|
|
this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence });
|
|
}
|
|
}
|
|
}
|
|
|
|
this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence });
|
|
|
|
return {
|
|
isLoggedIn,
|
|
confidence: highestConfidence,
|
|
detectedIndicators,
|
|
};
|
|
} catch (error) {
|
|
const errorMsg = `Login state detection failed: ${error}`;
|
|
this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg));
|
|
|
|
return {
|
|
isLoggedIn: false,
|
|
confidence: 0,
|
|
detectedIndicators: [],
|
|
error: errorMsg,
|
|
};
|
|
}
|
|
}
|
|
|
|
async findElement(template: ImageTemplate): Promise<ElementLocation | null> {
|
|
return this.templateMatching.findElement(template);
|
|
}
|
|
|
|
async focusBrowserWindow(titlePattern?: string): Promise<WindowFocusResult> {
|
|
return this.windowFocus.focusBrowserWindow(titlePattern);
|
|
}
|
|
|
|
async captureScreen(region?: ScreenRegion): Promise<ScreenCaptureResult> {
|
|
if (region) {
|
|
return this.screenRecognition.captureRegion(region);
|
|
}
|
|
return this.screenRecognition.captureFullScreen();
|
|
}
|
|
|
|
async clickAtLocation(location: ElementLocation): Promise<ClickResult> {
|
|
try {
|
|
const point = new Point(location.center.x, location.center.y);
|
|
await mouse.move([point]);
|
|
await mouse.leftClick();
|
|
|
|
this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y });
|
|
|
|
return {
|
|
success: true,
|
|
target: `${location.center.x},${location.center.y}`
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
target: `${location.center.x},${location.center.y}`,
|
|
error: String(error)
|
|
};
|
|
}
|
|
}
|
|
|
|
async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise<WaitResult> {
|
|
const timeout = maxWaitMs ?? this.config.defaultTimeout;
|
|
const result = await this.templateMatching.waitForTemplate(template, timeout);
|
|
|
|
return {
|
|
success: result.found,
|
|
target: template.id,
|
|
waitedMs: result.searchDurationMs,
|
|
found: result.found,
|
|
error: result.error,
|
|
};
|
|
}
|
|
|
|
async executeStep(stepId: StepId, config: Record<string, unknown>): Promise<AutomationResult> {
|
|
const stepNumber = stepId.value;
|
|
const startTime = Date.now();
|
|
|
|
this.logger.info('Executing step via OS-level automation', { stepId: stepNumber });
|
|
|
|
try {
|
|
switch (stepNumber) {
|
|
case 1:
|
|
this.logger.debug('Skipping login step - user pre-authenticated', { stepId: stepNumber });
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
skipped: true,
|
|
reason: 'User pre-authenticated',
|
|
step: 'LOGIN',
|
|
},
|
|
};
|
|
|
|
case 18:
|
|
this.logger.info('Safety stop at final step', { stepId: stepNumber });
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
step: 'TRACK_CONDITIONS',
|
|
safetyStop: true,
|
|
message: 'Automation stopped at final step. User must review configuration and click checkout manually.',
|
|
},
|
|
};
|
|
|
|
default: {
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.info('Step executed successfully', { stepId: stepNumber, durationMs });
|
|
return {
|
|
success: true,
|
|
metadata: {
|
|
step: `STEP_${stepNumber}`,
|
|
message: `Step ${stepNumber} executed via OS-level automation`,
|
|
config,
|
|
},
|
|
};
|
|
}
|
|
}
|
|
} catch (error) {
|
|
const durationMs = Date.now() - startTime;
|
|
this.logger.error('Step execution failed', error instanceof Error ? error : new Error(String(error)), {
|
|
stepId: stepNumber,
|
|
durationMs
|
|
});
|
|
return {
|
|
success: false,
|
|
error: String(error),
|
|
metadata: { step: `STEP_${stepNumber}` },
|
|
};
|
|
}
|
|
}
|
|
|
|
private parseTarget(target: string): Point {
|
|
if (target.includes(',')) {
|
|
const [x, y] = target.split(',').map(Number);
|
|
return new Point(x, y);
|
|
}
|
|
return new Point(
|
|
Math.floor(this.config.screenResolution.width / 2),
|
|
Math.floor(this.config.screenResolution.height / 2)
|
|
);
|
|
}
|
|
|
|
private delay(ms: number): Promise<void> {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
} |