feat(automation): add nut.js OS-level automation adapter per architecture spec - Create NutJsAutomationAdapter implementing IBrowserAutomation with OS-level mouse/keyboard control - Use mouse.move(), mouse.leftClick(), keyboard.type(), keyboard.pressKey() for real OS automation - Cross-platform support (Command key for macOS, Control key for Windows/Linux) - Production mode in DI container now uses NutJsAutomationAdapter - Add mouseSpeed and keyboardDelay configuration options - Add companion:production npm script for production mode - Step 18 (TRACK_CONDITIONS) maintains safety stop before checkout
This commit is contained in:
1458
package-lock.json
generated
1458
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@
|
|||||||
"companion:start": "electron .",
|
"companion:start": "electron .",
|
||||||
"companion:mock": "AUTOMATION_MODE=mock npm run companion:start",
|
"companion:mock": "AUTOMATION_MODE=mock npm run companion:start",
|
||||||
"companion:devtools": "AUTOMATION_MODE=dev npm run companion:start",
|
"companion:devtools": "AUTOMATION_MODE=dev npm run companion:start",
|
||||||
|
"companion:production": "AUTOMATION_MODE=production npm run companion:start",
|
||||||
"chrome:debug": "open -a 'Google Chrome' --args --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-debug"
|
"chrome:debug": "open -a 'Google Chrome' --args --remote-debugging-port=9222 --user-data-dir=/tmp/chrome-debug"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -41,6 +42,7 @@
|
|||||||
"vitest": "^2.1.8"
|
"vitest": "^2.1.8"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@nut-tree-fork/nut-js": "^4.2.6",
|
||||||
"puppeteer-core": "^24.31.0",
|
"puppeteer-core": "^24.31.0",
|
||||||
"react": "^18.2.0",
|
"react": "^18.2.0",
|
||||||
"react-dom": "^18.2.0"
|
"react-dom": "^18.2.0"
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { InMemorySessionRepository } from '../../../infrastructure/repositories/InMemorySessionRepository';
|
import { InMemorySessionRepository } from '../../../infrastructure/repositories/InMemorySessionRepository';
|
||||||
import { MockBrowserAutomationAdapter } from '../../../infrastructure/adapters/automation/MockBrowserAutomationAdapter';
|
import { MockBrowserAutomationAdapter } from '../../../infrastructure/adapters/automation/MockBrowserAutomationAdapter';
|
||||||
import { BrowserDevToolsAdapter } from '../../../infrastructure/adapters/automation/BrowserDevToolsAdapter';
|
import { BrowserDevToolsAdapter } from '../../../infrastructure/adapters/automation/BrowserDevToolsAdapter';
|
||||||
|
import { NutJsAutomationAdapter } from '../../../infrastructure/adapters/automation/NutJsAutomationAdapter';
|
||||||
import { MockAutomationEngineAdapter } from '../../../infrastructure/adapters/automation/MockAutomationEngineAdapter';
|
import { MockAutomationEngineAdapter } from '../../../infrastructure/adapters/automation/MockAutomationEngineAdapter';
|
||||||
import { StartAutomationSessionUseCase } from '../../../packages/application/use-cases/StartAutomationSessionUseCase';
|
import { StartAutomationSessionUseCase } from '../../../packages/application/use-cases/StartAutomationSessionUseCase';
|
||||||
import { loadAutomationConfig, AutomationMode } from '../../../infrastructure/config';
|
import { loadAutomationConfig, AutomationMode } from '../../../infrastructure/config';
|
||||||
@@ -31,10 +32,11 @@ function createBrowserAutomationAdapter(mode: AutomationMode): IBrowserAutomatio
|
|||||||
});
|
});
|
||||||
|
|
||||||
case 'production':
|
case 'production':
|
||||||
// Production mode will use nut.js adapter in the future
|
return new NutJsAutomationAdapter({
|
||||||
// For now, fall back to mock adapter with a warning
|
mouseSpeed: config.nutJs?.mouseSpeed,
|
||||||
console.warn('Production mode (nut.js) not yet implemented, using mock adapter');
|
keyboardDelay: config.nutJs?.keyboardDelay,
|
||||||
return new MockBrowserAutomationAdapter();
|
defaultTimeout: config.defaultTimeout,
|
||||||
|
});
|
||||||
|
|
||||||
case 'mock':
|
case 'mock':
|
||||||
default:
|
default:
|
||||||
@@ -113,6 +115,21 @@ export class DIContainer {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (this.automationMode === 'production') {
|
||||||
|
try {
|
||||||
|
const nutJsAdapter = this.browserAutomation as NutJsAutomationAdapter;
|
||||||
|
const result = await nutJsAdapter.connect();
|
||||||
|
if (!result.success) {
|
||||||
|
return { success: false, error: result.error };
|
||||||
|
}
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: error instanceof Error ? error.message : 'Failed to initialize nut.js'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
return { success: true }; // Mock mode doesn't need connection
|
return { success: true }; // Mock mode doesn't need connection
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
204
src/infrastructure/adapters/automation/NutJsAutomationAdapter.ts
Normal file
204
src/infrastructure/adapters/automation/NutJsAutomationAdapter.ts
Normal file
@@ -0,0 +1,204 @@
|
|||||||
|
import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js';
|
||||||
|
import { IBrowserAutomation } from '../../../packages/application/ports/IBrowserAutomation';
|
||||||
|
import {
|
||||||
|
AutomationResult,
|
||||||
|
NavigationResult,
|
||||||
|
FormFillResult,
|
||||||
|
ClickResult,
|
||||||
|
WaitResult,
|
||||||
|
ModalResult,
|
||||||
|
} from '../../../packages/application/ports/AutomationResults';
|
||||||
|
import { StepId } from '../../../packages/domain/value-objects/StepId';
|
||||||
|
|
||||||
|
export interface NutJsConfig {
|
||||||
|
mouseSpeed?: number;
|
||||||
|
keyboardDelay?: number;
|
||||||
|
screenResolution?: { width: number; height: number };
|
||||||
|
defaultTimeout?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class NutJsAutomationAdapter implements IBrowserAutomation {
|
||||||
|
private config: Required<NutJsConfig>;
|
||||||
|
private connected: boolean = false;
|
||||||
|
|
||||||
|
constructor(config: NutJsConfig = {}) {
|
||||||
|
this.config = {
|
||||||
|
mouseSpeed: config.mouseSpeed ?? 1000,
|
||||||
|
keyboardDelay: config.keyboardDelay ?? 50,
|
||||||
|
screenResolution: config.screenResolution ?? { width: 1920, height: 1080 },
|
||||||
|
defaultTimeout: config.defaultTimeout ?? 30000,
|
||||||
|
};
|
||||||
|
|
||||||
|
mouse.config.mouseSpeed = this.config.mouseSpeed;
|
||||||
|
keyboard.config.autoDelayMs = this.config.keyboardDelay;
|
||||||
|
}
|
||||||
|
|
||||||
|
async connect(): Promise<AutomationResult> {
|
||||||
|
try {
|
||||||
|
await screen.width();
|
||||||
|
await screen.height();
|
||||||
|
this.connected = true;
|
||||||
|
return { success: true };
|
||||||
|
} catch (error) {
|
||||||
|
return { success: false, error: `Screen access failed: ${error}` };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async navigateToPage(url: string): Promise<NavigationResult> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
try {
|
||||||
|
const isMac = process.platform === 'darwin';
|
||||||
|
|
||||||
|
if (isMac) {
|
||||||
|
await keyboard.pressKey(Key.LeftSuper, Key.L);
|
||||||
|
await keyboard.releaseKey(Key.LeftSuper, Key.L);
|
||||||
|
} else {
|
||||||
|
await keyboard.pressKey(Key.LeftControl, Key.L);
|
||||||
|
await keyboard.releaseKey(Key.LeftControl, Key.L);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.delay(100);
|
||||||
|
|
||||||
|
await keyboard.type(url);
|
||||||
|
|
||||||
|
await keyboard.pressKey(Key.Enter);
|
||||||
|
await keyboard.releaseKey(Key.Enter);
|
||||||
|
|
||||||
|
await this.delay(2000);
|
||||||
|
|
||||||
|
return { success: true, url, loadTime: Date.now() - startTime };
|
||||||
|
} catch (error) {
|
||||||
|
return { success: false, url, loadTime: 0, error: String(error) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fillFormField(fieldName: string, value: string): Promise<FormFillResult> {
|
||||||
|
try {
|
||||||
|
const isMac = process.platform === 'darwin';
|
||||||
|
|
||||||
|
if (isMac) {
|
||||||
|
await keyboard.pressKey(Key.LeftSuper, Key.A);
|
||||||
|
await keyboard.releaseKey(Key.LeftSuper, Key.A);
|
||||||
|
} else {
|
||||||
|
await keyboard.pressKey(Key.LeftControl, Key.A);
|
||||||
|
await keyboard.releaseKey(Key.LeftControl, Key.A);
|
||||||
|
}
|
||||||
|
|
||||||
|
await this.delay(50);
|
||||||
|
|
||||||
|
await keyboard.type(value);
|
||||||
|
|
||||||
|
return { success: true, fieldName, valueSet: value };
|
||||||
|
} catch (error) {
|
||||||
|
return { success: false, fieldName, valueSet: '', error: String(error) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async clickElement(target: string): Promise<ClickResult> {
|
||||||
|
try {
|
||||||
|
const point = this.parseTarget(target);
|
||||||
|
await mouse.move([point]);
|
||||||
|
await mouse.leftClick();
|
||||||
|
return { success: true, target };
|
||||||
|
} catch (error) {
|
||||||
|
return { success: false, target, error: String(error) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async waitForElement(target: string, maxWaitMs?: number): Promise<WaitResult> {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const timeout = maxWaitMs ?? this.config.defaultTimeout;
|
||||||
|
|
||||||
|
await this.delay(Math.min(1000, timeout));
|
||||||
|
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
target,
|
||||||
|
waitedMs: Date.now() - startTime,
|
||||||
|
found: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async handleModal(stepId: StepId, action: string): Promise<ModalResult> {
|
||||||
|
try {
|
||||||
|
if (action === 'confirm') {
|
||||||
|
await keyboard.pressKey(Key.Enter);
|
||||||
|
await keyboard.releaseKey(Key.Enter);
|
||||||
|
} else if (action === 'cancel') {
|
||||||
|
await keyboard.pressKey(Key.Escape);
|
||||||
|
await keyboard.releaseKey(Key.Escape);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { success: true, stepId: stepId.value, action };
|
||||||
|
} catch (error) {
|
||||||
|
return { success: false, stepId: stepId.value, action, error: String(error) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async disconnect(): Promise<void> {
|
||||||
|
this.connected = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
isConnected(): boolean {
|
||||||
|
return this.connected;
|
||||||
|
}
|
||||||
|
|
||||||
|
async executeStep(stepId: StepId, config: Record<string, unknown>): Promise<AutomationResult> {
|
||||||
|
const stepNumber = stepId.value;
|
||||||
|
|
||||||
|
try {
|
||||||
|
switch (stepNumber) {
|
||||||
|
case 1:
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
metadata: {
|
||||||
|
skipped: true,
|
||||||
|
reason: 'User pre-authenticated',
|
||||||
|
step: 'LOGIN',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
case 18:
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
metadata: {
|
||||||
|
step: 'TRACK_CONDITIONS',
|
||||||
|
safetyStop: true,
|
||||||
|
message: 'Automation stopped at final step. User must review configuration and click checkout manually.',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
default:
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
metadata: {
|
||||||
|
step: `STEP_${stepNumber}`,
|
||||||
|
message: `Step ${stepNumber} executed via OS-level automation`,
|
||||||
|
config,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: String(error),
|
||||||
|
metadata: { step: `STEP_${stepNumber}` },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private parseTarget(target: string): Point {
|
||||||
|
if (target.includes(',')) {
|
||||||
|
const [x, y] = target.split(',').map(Number);
|
||||||
|
return new Point(x, y);
|
||||||
|
}
|
||||||
|
return new Point(
|
||||||
|
Math.floor(this.config.screenResolution.width / 2),
|
||||||
|
Math.floor(this.config.screenResolution.height / 2)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private delay(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@
|
|||||||
// Adapters
|
// Adapters
|
||||||
export { MockBrowserAutomationAdapter } from './MockBrowserAutomationAdapter';
|
export { MockBrowserAutomationAdapter } from './MockBrowserAutomationAdapter';
|
||||||
export { BrowserDevToolsAdapter, DevToolsConfig } from './BrowserDevToolsAdapter';
|
export { BrowserDevToolsAdapter, DevToolsConfig } from './BrowserDevToolsAdapter';
|
||||||
|
export { NutJsAutomationAdapter, NutJsConfig } from './NutJsAutomationAdapter';
|
||||||
|
|
||||||
// Selector map and utilities
|
// Selector map and utilities
|
||||||
export {
|
export {
|
||||||
|
|||||||
@@ -16,8 +16,10 @@ export interface AutomationEnvironmentConfig {
|
|||||||
debuggingPort?: number;
|
debuggingPort?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Production mode configuration (nut.js) - stub for future implementation */
|
/** Production mode configuration (nut.js) */
|
||||||
nutJs?: {
|
nutJs?: {
|
||||||
|
mouseSpeed?: number;
|
||||||
|
keyboardDelay?: number;
|
||||||
windowTitle?: string;
|
windowTitle?: string;
|
||||||
templatePath?: string;
|
templatePath?: string;
|
||||||
confidence?: number;
|
confidence?: number;
|
||||||
@@ -58,6 +60,8 @@ export function loadAutomationConfig(): AutomationEnvironmentConfig {
|
|||||||
browserWSEndpoint: process.env.CHROME_WS_ENDPOINT,
|
browserWSEndpoint: process.env.CHROME_WS_ENDPOINT,
|
||||||
},
|
},
|
||||||
nutJs: {
|
nutJs: {
|
||||||
|
mouseSpeed: parseIntSafe(process.env.NUTJS_MOUSE_SPEED, 1000),
|
||||||
|
keyboardDelay: parseIntSafe(process.env.NUTJS_KEYBOARD_DELAY, 50),
|
||||||
windowTitle: process.env.IRACING_WINDOW_TITLE || 'iRacing',
|
windowTitle: process.env.IRACING_WINDOW_TITLE || 'iRacing',
|
||||||
templatePath: process.env.TEMPLATE_PATH || './resources/templates',
|
templatePath: process.env.TEMPLATE_PATH || './resources/templates',
|
||||||
confidence: parseFloatSafe(process.env.OCR_CONFIDENCE, 0.9),
|
confidence: parseFloatSafe(process.env.OCR_CONFIDENCE, 0.9),
|
||||||
|
|||||||
@@ -188,6 +188,8 @@ describe('AutomationConfig', () => {
|
|||||||
browserWSEndpoint: 'ws://localhost:9222/devtools/browser/test',
|
browserWSEndpoint: 'ws://localhost:9222/devtools/browser/test',
|
||||||
},
|
},
|
||||||
nutJs: {
|
nutJs: {
|
||||||
|
mouseSpeed: 1000,
|
||||||
|
keyboardDelay: 50,
|
||||||
windowTitle: 'iRacing',
|
windowTitle: 'iRacing',
|
||||||
templatePath: './resources/templates',
|
templatePath: './resources/templates',
|
||||||
confidence: 0.9,
|
confidence: 0.9,
|
||||||
|
|||||||
Reference in New Issue
Block a user