diff --git a/packages/application/ports/IScreenAutomation.ts b/packages/application/ports/IScreenAutomation.ts new file mode 100644 index 000000000..c45cdd11d --- /dev/null +++ b/packages/application/ports/IScreenAutomation.ts @@ -0,0 +1,179 @@ +import { StepId } from '../../domain/value-objects/StepId'; +import type { ImageTemplate } from '../../domain/value-objects/ImageTemplate'; +import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../domain/value-objects/ScreenRegion'; +import { + NavigationResult, + FormFillResult, + ClickResult, + WaitResult, + ModalResult, + AutomationResult, +} from './AutomationResults'; + +/** + * Screen capture result containing the captured image data. + */ +export interface ScreenCaptureResult { + success: boolean; + data?: Buffer; + width?: number; + height?: number; + error?: string; +} + +/** + * Window information for browser window management. + */ +export interface WindowInfo { + title: string; + bounds: ScreenRegion; + handle: number; + isActive: boolean; +} + +/** + * Result of window focus operation. + */ +export interface WindowFocusResult { + success: boolean; + window?: WindowInfo; + error?: string; +} + +/** + * Extended browser automation interface for OS-level screen automation. + * + * This interface extends the base IBrowserAutomation with screen recognition + * capabilities for TOS-compliant automation that doesn't use browser DevTools. + * + * Key differences from browser-based automation: + * - Uses image template matching instead of CSS selectors + * - Works with screen coordinates instead of DOM elements + * - Requires explicit window focus management + * - No direct access to page content or JavaScript execution + */ +export interface IScreenAutomation { + // ============================================ + // LEGACY BROWSER AUTOMATION METHODS + // (Maintained for backward compatibility) + // ============================================ + + /** + * Navigate to a URL using keyboard shortcuts (Cmd/Ctrl+L, type URL, Enter). + * Requires browser window to be focused. + */ + navigateToPage(url: string): Promise; + + /** + * Fill a form field by selecting all text and typing new value. + * Requires the field to already be focused. + */ + fillFormField(fieldName: string, value: string): Promise; + + /** + * Click at a screen position (accepts coordinates or template ID). + */ + clickElement(target: string): Promise; + + /** + * Wait for a condition (time-based in screen automation mode). + */ + waitForElement(target: string, maxWaitMs?: number): Promise; + + /** + * Handle modal dialogs using keyboard (Enter/Escape). + */ + handleModal(stepId: StepId, action: string): Promise; + + /** + * Execute a complete workflow step. + */ + executeStep?(stepId: StepId, config: Record): Promise; + + /** + * Initialize the automation connection. + */ + connect?(): Promise; + + /** + * Clean up resources. + */ + disconnect?(): Promise; + + /** + * Check if automation is ready. + */ + isConnected?(): boolean; + + // ============================================ + // SCREEN AUTOMATION METHODS + // (New methods for OS-level automation) + // ============================================ + + /** + * Detect login state by searching for known UI indicators on screen. + * Uses template matching to find login-related elements. + * + * @returns LoginDetectionResult with confidence and detected indicators + */ + detectLoginState?(): Promise; + + /** + * Find a UI element on screen using image template matching. + * + * @param template - The image template to search for + * @returns ElementLocation if found, null if not found + */ + findElement?(template: ImageTemplate): Promise; + + /** + * Bring the browser window to the foreground. + * Searches for windows matching a title pattern (e.g., "iRacing"). + * + * @param titlePattern - Optional window title pattern to match + * @returns WindowFocusResult indicating success/failure + */ + focusBrowserWindow?(titlePattern?: string): Promise; + + /** + * Capture a region of the screen. + * + * @param region - Optional region to capture (full screen if omitted) + * @returns ScreenCaptureResult with image data + */ + captureScreen?(region?: ScreenRegion): Promise; + + /** + * Click on a found element location. + * + * @param location - The element location from findElement + * @returns ClickResult indicating success/failure + */ + clickAtLocation?(location: ElementLocation): Promise; + + /** + * Wait for a template to appear on screen. + * + * @param template - The image template to wait for + * @param maxWaitMs - Maximum time to wait in milliseconds + * @returns WaitResult with timing information + */ + waitForTemplate?(template: ImageTemplate, maxWaitMs?: number): Promise; +} + +/** + * Type alias for backward compatibility. + * IBrowserAutomation is now a subset of IScreenAutomation. + */ +export type IBrowserAutomation = Pick< + IScreenAutomation, + | 'navigateToPage' + | 'fillFormField' + | 'clickElement' + | 'waitForElement' + | 'handleModal' + | 'executeStep' + | 'connect' + | 'disconnect' + | 'isConnected' +>; \ No newline at end of file diff --git a/packages/domain/value-objects/ImageTemplate.ts b/packages/domain/value-objects/ImageTemplate.ts new file mode 100644 index 000000000..117f93ee8 --- /dev/null +++ b/packages/domain/value-objects/ImageTemplate.ts @@ -0,0 +1,96 @@ +import type { ScreenRegion } from './ScreenRegion'; + +/** + * Represents an image template used for visual element detection. + * Templates are reference images that are matched against screen captures + * to locate UI elements without relying on CSS selectors or DOM access. + */ +export interface ImageTemplate { + /** Unique identifier for the template */ + id: string; + /** Path to the template image file (relative to resources directory) */ + imagePath: string; + /** Confidence threshold for matching (0.0-1.0, higher = more strict) */ + confidence: number; + /** Optional region to limit search area for better performance */ + searchRegion?: ScreenRegion; + /** Human-readable description of what this template represents */ + description: string; +} + +/** + * Template categories for organization and filtering. + */ +export type TemplateCategory = + | 'login' + | 'navigation' + | 'wizard' + | 'button' + | 'field' + | 'modal' + | 'indicator'; + +/** + * Extended template with category metadata. + */ +export interface CategorizedTemplate extends ImageTemplate { + category: TemplateCategory; + stepId?: number; +} + +/** + * Create an ImageTemplate with default confidence. + */ +export function createImageTemplate( + id: string, + imagePath: string, + description: string, + options?: { + confidence?: number; + searchRegion?: ScreenRegion; + } +): ImageTemplate { + return { + id, + imagePath, + description, + confidence: options?.confidence ?? 0.9, + searchRegion: options?.searchRegion, + }; +} + +/** + * Validate that an ImageTemplate has all required fields. + */ +export function isValidTemplate(template: unknown): template is ImageTemplate { + if (typeof template !== 'object' || template === null) { + return false; + } + + const t = template as Record; + + return ( + typeof t.id === 'string' && + t.id.length > 0 && + typeof t.imagePath === 'string' && + t.imagePath.length > 0 && + typeof t.confidence === 'number' && + t.confidence >= 0 && + t.confidence <= 1 && + typeof t.description === 'string' + ); +} + +/** + * Default confidence thresholds for different template types. + */ +export const DEFAULT_CONFIDENCE = { + /** High confidence for exact matches (buttons, icons) */ + HIGH: 0.95, + /** Standard confidence for most UI elements */ + STANDARD: 0.9, + /** Lower confidence for variable elements (text fields with content) */ + LOW: 0.8, + /** Minimum acceptable confidence */ + MINIMUM: 0.7, +} as const; \ No newline at end of file diff --git a/packages/domain/value-objects/ScreenRegion.ts b/packages/domain/value-objects/ScreenRegion.ts new file mode 100644 index 000000000..918db3ea3 --- /dev/null +++ b/packages/domain/value-objects/ScreenRegion.ts @@ -0,0 +1,86 @@ +/** + * Represents a rectangular region on the screen. + * Used for targeted screen capture and element location. + */ +export interface ScreenRegion { + x: number; + y: number; + width: number; + height: number; +} + +/** + * Represents a point on the screen with x,y coordinates. + */ +export interface Point { + x: number; + y: number; +} + +/** + * Represents the location of a detected UI element on screen. + * Contains the center point, bounding box, and confidence score. + */ +export interface ElementLocation { + center: Point; + bounds: ScreenRegion; + confidence: number; +} + +/** + * Result of login state detection via screen recognition. + */ +export interface LoginDetectionResult { + isLoggedIn: boolean; + confidence: number; + detectedIndicators: string[]; + error?: string; +} + +/** + * Create a ScreenRegion from coordinates. + */ +export function createScreenRegion(x: number, y: number, width: number, height: number): ScreenRegion { + return { x, y, width, height }; +} + +/** + * Create a Point from coordinates. + */ +export function createPoint(x: number, y: number): Point { + return { x, y }; +} + +/** + * Calculate the center point of a ScreenRegion. + */ +export function getRegionCenter(region: ScreenRegion): Point { + return { + x: region.x + Math.floor(region.width / 2), + y: region.y + Math.floor(region.height / 2), + }; +} + +/** + * Check if a point is within a screen region. + */ +export function isPointInRegion(point: Point, region: ScreenRegion): boolean { + return ( + point.x >= region.x && + point.x <= region.x + region.width && + point.y >= region.y && + point.y <= region.y + region.height + ); +} + +/** + * Check if two screen regions overlap. + */ +export function regionsOverlap(a: ScreenRegion, b: ScreenRegion): boolean { + return !( + a.x + a.width < b.x || + b.x + b.width < a.x || + a.y + a.height < b.y || + b.y + b.height < a.y + ); +} \ No newline at end of file diff --git a/packages/infrastructure/adapters/automation/NutJsAutomationAdapter.ts b/packages/infrastructure/adapters/automation/NutJsAutomationAdapter.ts index 2ed93cbe8..2e508f1b7 100644 --- a/packages/infrastructure/adapters/automation/NutJsAutomationAdapter.ts +++ b/packages/infrastructure/adapters/automation/NutJsAutomationAdapter.ts @@ -1,5 +1,5 @@ import { mouse, keyboard, screen, Point, Key } from '@nut-tree-fork/nut-js'; -import { IBrowserAutomation } from '../../../packages/application/ports/IBrowserAutomation'; +import type { IScreenAutomation, ScreenCaptureResult, WindowFocusResult } from '../../../application/ports/IScreenAutomation'; import { AutomationResult, NavigationResult, @@ -7,22 +7,33 @@ import { ClickResult, WaitResult, ModalResult, -} from '../../../packages/application/ports/AutomationResults'; -import { StepId } from '../../../packages/domain/value-objects/StepId'; +} from '../../../application/ports/AutomationResults'; +import { StepId } from '../../../domain/value-objects/StepId'; +import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate'; +import type { ElementLocation, LoginDetectionResult, ScreenRegion } from '../../../domain/value-objects/ScreenRegion'; import type { ILogger } from '../../../application/ports/ILogger'; import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; +import { ScreenRecognitionService } from './ScreenRecognitionService'; +import { TemplateMatchingService } from './TemplateMatchingService'; +import { WindowFocusService } from './WindowFocusService'; +import { getLoginIndicators, getLogoutIndicators } from './templates/IRacingTemplateMap'; export interface NutJsConfig { mouseSpeed?: number; keyboardDelay?: number; screenResolution?: { width: number; height: number }; defaultTimeout?: number; + templatePath?: string; + windowTitle?: string; } -export class NutJsAutomationAdapter implements IBrowserAutomation { +export class NutJsAutomationAdapter implements IScreenAutomation { private config: Required; private connected: boolean = false; private logger: ILogger; + private screenRecognition: ScreenRecognitionService; + private templateMatching: TemplateMatchingService; + private windowFocus: WindowFocusService; constructor(config: NutJsConfig = {}, logger?: ILogger) { this.config = { @@ -30,11 +41,23 @@ export class NutJsAutomationAdapter implements IBrowserAutomation { keyboardDelay: config.keyboardDelay ?? 50, screenResolution: config.screenResolution ?? { width: 1920, height: 1080 }, defaultTimeout: config.defaultTimeout ?? 30000, + templatePath: config.templatePath ?? './resources/templates/iracing', + windowTitle: config.windowTitle ?? 'iRacing', }; this.logger = logger ?? new NoOpLogAdapter(); mouse.config.mouseSpeed = this.config.mouseSpeed; keyboard.config.autoDelayMs = this.config.keyboardDelay; + + this.screenRecognition = new ScreenRecognitionService(this.logger.child({ service: 'ScreenRecognition' })); + this.templateMatching = new TemplateMatchingService( + this.config.templatePath, + this.logger.child({ service: 'TemplateMatching' }) + ); + this.windowFocus = new WindowFocusService( + this.config.windowTitle, + this.logger.child({ service: 'WindowFocus' }) + ); } async connect(): Promise { @@ -163,6 +186,106 @@ export class NutJsAutomationAdapter implements IBrowserAutomation { return this.connected; } + async detectLoginState(): Promise { + this.logger.debug('Detecting login state via template matching'); + + try { + const detectedIndicators: string[] = []; + let isLoggedIn = false; + let highestConfidence = 0; + + const loginIndicators = getLoginIndicators(); + for (const indicator of loginIndicators) { + const location = await this.templateMatching.findElement(indicator); + if (location) { + detectedIndicators.push(indicator.id); + isLoggedIn = true; + highestConfidence = Math.max(highestConfidence, location.confidence); + this.logger.debug('Login indicator found', { indicatorId: indicator.id, confidence: location.confidence }); + } + } + + if (!isLoggedIn) { + const logoutIndicators = getLogoutIndicators(); + for (const indicator of logoutIndicators) { + const location = await this.templateMatching.findElement(indicator); + if (location) { + detectedIndicators.push(indicator.id); + highestConfidence = Math.max(highestConfidence, location.confidence); + this.logger.debug('Logout indicator found', { indicatorId: indicator.id, confidence: location.confidence }); + } + } + } + + this.logger.info('Login state detection complete', { isLoggedIn, detectedIndicators, confidence: highestConfidence }); + + return { + isLoggedIn, + confidence: highestConfidence, + detectedIndicators, + }; + } catch (error) { + const errorMsg = `Login state detection failed: ${error}`; + this.logger.error('Login state detection failed', error instanceof Error ? error : new Error(errorMsg)); + + return { + isLoggedIn: false, + confidence: 0, + detectedIndicators: [], + error: errorMsg, + }; + } + } + + async findElement(template: ImageTemplate): Promise { + return this.templateMatching.findElement(template); + } + + async focusBrowserWindow(titlePattern?: string): Promise { + return this.windowFocus.focusBrowserWindow(titlePattern); + } + + async captureScreen(region?: ScreenRegion): Promise { + if (region) { + return this.screenRecognition.captureRegion(region); + } + return this.screenRecognition.captureFullScreen(); + } + + async clickAtLocation(location: ElementLocation): Promise { + try { + const point = new Point(location.center.x, location.center.y); + await mouse.move([point]); + await mouse.leftClick(); + + this.logger.debug('Clicked at location', { x: location.center.x, y: location.center.y }); + + return { + success: true, + target: `${location.center.x},${location.center.y}` + }; + } catch (error) { + return { + success: false, + target: `${location.center.x},${location.center.y}`, + error: String(error) + }; + } + } + + async waitForTemplate(template: ImageTemplate, maxWaitMs?: number): Promise { + const timeout = maxWaitMs ?? this.config.defaultTimeout; + const result = await this.templateMatching.waitForTemplate(template, timeout); + + return { + success: result.found, + target: template.id, + waitedMs: result.searchDurationMs, + found: result.found, + error: result.error, + }; + } + async executeStep(stepId: StepId, config: Record): Promise { const stepNumber = stepId.value; const startTime = Date.now(); diff --git a/packages/infrastructure/adapters/automation/ScreenRecognitionService.ts b/packages/infrastructure/adapters/automation/ScreenRecognitionService.ts new file mode 100644 index 000000000..c153cee5a --- /dev/null +++ b/packages/infrastructure/adapters/automation/ScreenRecognitionService.ts @@ -0,0 +1,124 @@ +import { screen, Region } from '@nut-tree-fork/nut-js'; +import type { ScreenRegion } from '../../../domain/value-objects/ScreenRegion'; +import type { ScreenCaptureResult } from '../../../application/ports/IScreenAutomation'; +import type { ILogger } from '../../../application/ports/ILogger'; +import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; + +/** + * Service for capturing screen regions using nut.js. + * Provides screen capture functionality for template matching and visual automation. + */ +export class ScreenRecognitionService { + private logger: ILogger; + + constructor(logger?: ILogger) { + this.logger = logger ?? new NoOpLogAdapter(); + } + + /** + * Capture the entire screen. + * @returns ScreenCaptureResult with image buffer data + */ + async captureFullScreen(): Promise { + const startTime = Date.now(); + + try { + const width = await screen.width(); + const height = await screen.height(); + + this.logger.debug('Capturing full screen', { width, height }); + + const image = await screen.grab(); + const data = await image.toRGB(); + + const durationMs = Date.now() - startTime; + this.logger.debug('Screen capture completed', { durationMs, width, height }); + + return { + success: true, + data: Buffer.from(data.data), + width: data.width, + height: data.height, + }; + } catch (error) { + const errorMsg = `Screen capture failed: ${error}`; + this.logger.error('Screen capture failed', error instanceof Error ? error : new Error(errorMsg)); + + return { + success: false, + error: errorMsg, + }; + } + } + + /** + * Capture a specific region of the screen. + * @param region - The rectangular region to capture + * @returns ScreenCaptureResult with image buffer data + */ + async captureRegion(region: ScreenRegion): Promise { + const startTime = Date.now(); + + try { + this.logger.debug('Capturing screen region', { region }); + + const nutRegion = new Region(region.x, region.y, region.width, region.height); + const image = await screen.grabRegion(nutRegion); + const data = await image.toRGB(); + + const durationMs = Date.now() - startTime; + this.logger.debug('Region capture completed', { durationMs, region }); + + return { + success: true, + data: Buffer.from(data.data), + width: data.width, + height: data.height, + }; + } catch (error) { + const errorMsg = `Region capture failed: ${error}`; + this.logger.error('Region capture failed', error instanceof Error ? error : new Error(errorMsg), { region }); + + return { + success: false, + error: errorMsg, + }; + } + } + + /** + * Get the current screen dimensions. + * @returns Object with width and height, or null on error + */ + async getScreenDimensions(): Promise<{ width: number; height: number } | null> { + try { + const width = await screen.width(); + const height = await screen.height(); + return { width, height }; + } catch (error) { + this.logger.error('Failed to get screen dimensions', error instanceof Error ? error : new Error(String(error))); + return null; + } + } + + /** + * Convert ScreenRegion to nut.js Region. + * Utility method for internal use. + */ + toNutRegion(region: ScreenRegion): Region { + return new Region(region.x, region.y, region.width, region.height); + } + + /** + * Convert nut.js Region to ScreenRegion. + * Utility method for internal use. + */ + fromNutRegion(nutRegion: Region): ScreenRegion { + return { + x: nutRegion.left, + y: nutRegion.top, + width: nutRegion.width, + height: nutRegion.height, + }; + } +} \ No newline at end of file diff --git a/packages/infrastructure/adapters/automation/TemplateMatchingService.ts b/packages/infrastructure/adapters/automation/TemplateMatchingService.ts new file mode 100644 index 000000000..0da095864 --- /dev/null +++ b/packages/infrastructure/adapters/automation/TemplateMatchingService.ts @@ -0,0 +1,283 @@ +import { screen, imageResource, Region } from '@nut-tree-fork/nut-js'; +import type { ImageTemplate } from '../../../domain/value-objects/ImageTemplate'; +import type { ElementLocation, ScreenRegion, Point } from '../../../domain/value-objects/ScreenRegion'; +import type { ILogger } from '../../../application/ports/ILogger'; +import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Result of a template matching operation. + */ +export interface TemplateMatchResult { + found: boolean; + location?: ElementLocation; + searchDurationMs: number; + error?: string; +} + +/** + * Service for finding UI elements on screen using image template matching. + * Uses nut.js screen.find() with @nut-tree-fork/template-matcher for image comparison. + */ +export class TemplateMatchingService { + private logger: ILogger; + private templateBasePath: string; + private loadedTemplates: Map = new Map(); + + constructor(templateBasePath?: string, logger?: ILogger) { + this.logger = logger ?? new NoOpLogAdapter(); + this.templateBasePath = templateBasePath ?? './resources/templates'; + } + + /** + * Find an element on screen using image template matching. + * @param template - The image template to search for + * @returns ElementLocation if found, null if not found + */ + async findElement(template: ImageTemplate): Promise { + const startTime = Date.now(); + + try { + const fullPath = this.resolveTemplatePath(template.imagePath); + + if (!this.templateExists(fullPath)) { + this.logger.warn('Template image not found, skipping search', { + templateId: template.id, + imagePath: fullPath, + description: template.description + }); + return null; + } + + this.logger.debug('Searching for template', { + templateId: template.id, + confidence: template.confidence, + searchRegion: template.searchRegion + }); + + const templateImage = await imageResource(fullPath); + + // Set confidence threshold for this search + screen.config.confidence = template.confidence; + + let foundRegion: Region; + + if (template.searchRegion) { + // Search within a specific region for better performance + const searchArea = new Region( + template.searchRegion.x, + template.searchRegion.y, + template.searchRegion.width, + template.searchRegion.height + ); + foundRegion = await screen.findRegion(templateImage, searchArea); + } else { + // Search entire screen + foundRegion = await screen.find(templateImage); + } + + const location = this.regionToElementLocation(foundRegion, template.confidence); + + const durationMs = Date.now() - startTime; + this.logger.debug('Template found', { + templateId: template.id, + location: location.center, + confidence: location.confidence, + durationMs + }); + + return location; + } catch (error) { + const durationMs = Date.now() - startTime; + + // nut.js throws when template is not found - this is expected behavior + if (String(error).includes('No match')) { + this.logger.debug('Template not found on screen', { + templateId: template.id, + durationMs + }); + return null; + } + + this.logger.error('Template matching error', error instanceof Error ? error : new Error(String(error)), { + templateId: template.id, + durationMs + }); + return null; + } + } + + /** + * Find multiple occurrences of a template on screen. + * @param template - The image template to search for + * @returns Array of ElementLocation for all matches + */ + async findAllElements(template: ImageTemplate): Promise { + const startTime = Date.now(); + + try { + const fullPath = this.resolveTemplatePath(template.imagePath); + + if (!this.templateExists(fullPath)) { + this.logger.warn('Template image not found', { templateId: template.id, imagePath: fullPath }); + return []; + } + + const templateImage = await imageResource(fullPath); + screen.config.confidence = template.confidence; + + const foundRegions = await screen.findAll(templateImage); + + const locations = foundRegions.map(region => this.regionToElementLocation(region, template.confidence)); + + const durationMs = Date.now() - startTime; + this.logger.debug('Found multiple templates', { + templateId: template.id, + count: locations.length, + durationMs + }); + + return locations; + } catch (error) { + const durationMs = Date.now() - startTime; + + if (String(error).includes('No match')) { + this.logger.debug('No templates found on screen', { templateId: template.id, durationMs }); + return []; + } + + this.logger.error('Template matching error', error instanceof Error ? error : new Error(String(error)), { + templateId: template.id, + durationMs + }); + return []; + } + } + + /** + * Wait for a template to appear on screen. + * @param template - The image template to wait for + * @param maxWaitMs - Maximum time to wait in milliseconds + * @param pollIntervalMs - How often to check for the template + * @returns TemplateMatchResult with timing information + */ + async waitForTemplate( + template: ImageTemplate, + maxWaitMs: number = 10000, + pollIntervalMs: number = 500 + ): Promise { + const startTime = Date.now(); + + this.logger.debug('Waiting for template', { + templateId: template.id, + maxWaitMs, + pollIntervalMs + }); + + while (Date.now() - startTime < maxWaitMs) { + const location = await this.findElement(template); + + if (location) { + const durationMs = Date.now() - startTime; + this.logger.debug('Template appeared', { templateId: template.id, durationMs }); + + return { + found: true, + location, + searchDurationMs: durationMs, + }; + } + + await this.delay(pollIntervalMs); + } + + const durationMs = Date.now() - startTime; + this.logger.debug('Template did not appear within timeout', { + templateId: template.id, + durationMs, + maxWaitMs + }); + + return { + found: false, + searchDurationMs: durationMs, + error: `Template '${template.id}' not found within ${maxWaitMs}ms`, + }; + } + + /** + * Check if a template exists on screen without returning location. + * Useful for boolean state checks. + */ + async isTemplateVisible(template: ImageTemplate): Promise { + const location = await this.findElement(template); + return location !== null; + } + + /** + * Resolve a template path relative to the base template directory. + */ + private resolveTemplatePath(imagePath: string): string { + if (path.isAbsolute(imagePath)) { + return imagePath; + } + return path.join(this.templateBasePath, imagePath); + } + + /** + * Check if a template file exists. + */ + private templateExists(fullPath: string): boolean { + try { + return fs.existsSync(fullPath); + } catch { + return false; + } + } + + /** + * Convert nut.js Region to ElementLocation. + */ + private regionToElementLocation(region: Region, confidence: number): ElementLocation { + const bounds: ScreenRegion = { + x: region.left, + y: region.top, + width: region.width, + height: region.height, + }; + + const center: Point = { + x: region.left + Math.floor(region.width / 2), + y: region.top + Math.floor(region.height / 2), + }; + + return { + center, + bounds, + confidence, + }; + } + + /** + * Utility delay function. + */ + private delay(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } + + /** + * Get the configured template base path. + */ + getTemplateBasePath(): string { + return this.templateBasePath; + } + + /** + * Set the template base path. + */ + setTemplateBasePath(basePath: string): void { + this.templateBasePath = basePath; + this.loadedTemplates.clear(); + } +} \ No newline at end of file diff --git a/packages/infrastructure/adapters/automation/WindowFocusService.ts b/packages/infrastructure/adapters/automation/WindowFocusService.ts new file mode 100644 index 000000000..a54a786b6 --- /dev/null +++ b/packages/infrastructure/adapters/automation/WindowFocusService.ts @@ -0,0 +1,244 @@ +import { getWindows, getActiveWindow, focusWindow, Window } from '@nut-tree-fork/nut-js'; +import type { ScreenRegion } from '../../../domain/value-objects/ScreenRegion'; +import type { WindowInfo, WindowFocusResult } from '../../../application/ports/IScreenAutomation'; +import type { ILogger } from '../../../application/ports/ILogger'; +import { NoOpLogAdapter } from '../logging/NoOpLogAdapter'; + +/** + * Service for managing browser window focus and enumeration. + * Uses nut.js window management APIs to find and focus browser windows. + */ +export class WindowFocusService { + private logger: ILogger; + private defaultTitlePattern: string; + + constructor(defaultTitlePattern?: string, logger?: ILogger) { + this.logger = logger ?? new NoOpLogAdapter(); + this.defaultTitlePattern = defaultTitlePattern ?? 'iRacing'; + } + + /** + * Find and focus a browser window matching the title pattern. + * @param titlePattern - Pattern to match in window title (default: 'iRacing') + * @returns WindowFocusResult with window info if successful + */ + async focusBrowserWindow(titlePattern?: string): Promise { + const pattern = titlePattern ?? this.defaultTitlePattern; + + try { + this.logger.debug('Searching for browser window', { titlePattern: pattern }); + + const windows = await getWindows(); + + for (const windowHandle of windows) { + try { + const windowRef = new Window(windowHandle); + const title = await windowRef.getTitle(); + + if (title.toLowerCase().includes(pattern.toLowerCase())) { + this.logger.debug('Found matching window', { title, handle: windowHandle }); + + await focusWindow(windowRef); + + // Get window bounds after focusing + const region = await windowRef.getRegion(); + const bounds: ScreenRegion = { + x: region.left, + y: region.top, + width: region.width, + height: region.height, + }; + + const windowInfo: WindowInfo = { + title, + bounds, + handle: windowHandle, + isActive: true, + }; + + this.logger.info('Browser window focused', { title, bounds }); + + return { + success: true, + window: windowInfo, + }; + } + } catch (windowError) { + // Skip windows that can't be accessed + this.logger.debug('Could not access window', { handle: windowHandle, error: String(windowError) }); + continue; + } + } + + this.logger.warn('No matching browser window found', { titlePattern: pattern }); + + return { + success: false, + error: `No window found matching pattern: ${pattern}`, + }; + } catch (error) { + const errorMsg = `Window focus failed: ${error}`; + this.logger.error('Window focus failed', error instanceof Error ? error : new Error(errorMsg)); + + return { + success: false, + error: errorMsg, + }; + } + } + + /** + * Get the currently active window. + * @returns WindowInfo for the active window, or null if unable to determine + */ + async getActiveWindow(): Promise { + try { + const activeWindow = await getActiveWindow(); + const title = await activeWindow.getTitle(); + const region = await activeWindow.getRegion(); + + const bounds: ScreenRegion = { + x: region.left, + y: region.top, + width: region.width, + height: region.height, + }; + + return { + title, + bounds, + handle: 0, // Active window doesn't expose handle directly + isActive: true, + }; + } catch (error) { + this.logger.error('Failed to get active window', error instanceof Error ? error : new Error(String(error))); + return null; + } + } + + /** + * List all visible windows. + * @returns Array of WindowInfo for all accessible windows + */ + async listWindows(): Promise { + const result: WindowInfo[] = []; + + try { + const windows = await getWindows(); + const activeWindow = await getActiveWindow(); + const activeTitle = await activeWindow.getTitle(); + + for (const windowHandle of windows) { + try { + const windowRef = new Window(windowHandle); + const title = await windowRef.getTitle(); + const region = await windowRef.getRegion(); + + const bounds: ScreenRegion = { + x: region.left, + y: region.top, + width: region.width, + height: region.height, + }; + + result.push({ + title, + bounds, + handle: windowHandle, + isActive: title === activeTitle, + }); + } catch { + // Skip inaccessible windows + continue; + } + } + } catch (error) { + this.logger.error('Failed to list windows', error instanceof Error ? error : new Error(String(error))); + } + + return result; + } + + /** + * Check if a window matching the pattern is currently visible. + * @param titlePattern - Pattern to match in window title + * @returns true if a matching window exists + */ + async isWindowVisible(titlePattern?: string): Promise { + const pattern = titlePattern ?? this.defaultTitlePattern; + + try { + const windows = await getWindows(); + + for (const windowHandle of windows) { + try { + const windowRef = new Window(windowHandle); + const title = await windowRef.getTitle(); + + if (title.toLowerCase().includes(pattern.toLowerCase())) { + return true; + } + } catch { + continue; + } + } + + return false; + } catch { + return false; + } + } + + /** + * Get the bounds of a window matching the pattern. + * Useful for targeted screen capture. + * @param titlePattern - Pattern to match in window title + * @returns ScreenRegion of the window, or null if not found + */ + async getWindowBounds(titlePattern?: string): Promise { + const pattern = titlePattern ?? this.defaultTitlePattern; + + try { + const windows = await getWindows(); + + for (const windowHandle of windows) { + try { + const windowRef = new Window(windowHandle); + const title = await windowRef.getTitle(); + + if (title.toLowerCase().includes(pattern.toLowerCase())) { + const region = await windowRef.getRegion(); + + return { + x: region.left, + y: region.top, + width: region.width, + height: region.height, + }; + } + } catch { + continue; + } + } + + return null; + } catch (error) { + this.logger.error('Failed to get window bounds', error instanceof Error ? error : new Error(String(error))); + return null; + } + } + + /** + * Set the default title pattern for window searches. + */ + setDefaultTitlePattern(pattern: string): void { + this.defaultTitlePattern = pattern; + } + + /** + * Get the current default title pattern. + */ + getDefaultTitlePattern(): string { + return this.defaultTitlePattern; + } +} \ No newline at end of file diff --git a/packages/infrastructure/adapters/automation/templates/IRacingTemplateMap.ts b/packages/infrastructure/adapters/automation/templates/IRacingTemplateMap.ts new file mode 100644 index 000000000..f94fef5da --- /dev/null +++ b/packages/infrastructure/adapters/automation/templates/IRacingTemplateMap.ts @@ -0,0 +1,887 @@ +import { createImageTemplate, DEFAULT_CONFIDENCE, type CategorizedTemplate } from '../../../../domain/value-objects/ImageTemplate'; +import type { ImageTemplate } from '../../../../domain/value-objects/ImageTemplate'; + +/** + * Template definitions for iRacing UI elements. + * + * These templates replace CSS selectors with image-based matching for TOS-compliant + * OS-level automation. Templates reference images in resources/templates/iracing/ + * + * Template images should be captured from the actual iRacing UI at standard resolution. + * Recommended: 1920x1080 or 2560x1440 with PNG format for lossless quality. + */ + +/** + * Step template configuration containing all templates needed for a workflow step. + */ +export interface StepTemplates { + /** Templates to detect if we're on this step */ + indicators: ImageTemplate[]; + /** Button templates for navigation and actions */ + buttons: Record; + /** Field templates for form inputs */ + fields?: Record; + /** Modal-related templates if applicable */ + modal?: { + indicator: ImageTemplate; + closeButton: ImageTemplate; + confirmButton?: ImageTemplate; + searchInput?: ImageTemplate; + }; +} + +/** + * Complete template map type for iRacing automation. + */ +export interface IRacingTemplateMapType { + /** Common templates used across multiple steps */ + common: { + /** Logged-in state indicators */ + loginIndicators: ImageTemplate[]; + /** Logged-out state indicators */ + logoutIndicators: ImageTemplate[]; + /** Generic navigation buttons */ + navigation: Record; + /** Loading indicators */ + loading: ImageTemplate[]; + }; + /** Step-specific templates */ + steps: Record; + /** Base path for template images */ + templateBasePath: string; +} + +/** + * Template paths for iRacing UI elements. + * All paths are relative to resources/templates/iracing/ + */ +const TEMPLATE_PATHS = { + common: { + login: 'common/login-indicator.png', + logout: 'common/logout-indicator.png', + userAvatar: 'common/user-avatar.png', + memberBadge: 'common/member-badge.png', + loginButton: 'common/login-button.png', + loadingSpinner: 'common/loading-spinner.png', + nextButton: 'common/next-button.png', + backButton: 'common/back-button.png', + checkoutButton: 'common/checkout-button.png', + closeModal: 'common/close-modal-button.png', + }, + steps: { + 1: { + loginForm: 'step01-login/login-form.png', + emailField: 'step01-login/email-field.png', + passwordField: 'step01-login/password-field.png', + submitButton: 'step01-login/submit-button.png', + }, + 2: { + hostedRacingTab: 'step02-hosted/hosted-racing-tab.png', + createRaceButton: 'step02-hosted/create-race-button.png', + sessionList: 'step02-hosted/session-list.png', + }, + 3: { + createRaceModal: 'step03-create/create-race-modal.png', + confirmButton: 'step03-create/confirm-button.png', + }, + 4: { + stepIndicator: 'step04-info/race-info-indicator.png', + sessionNameField: 'step04-info/session-name-field.png', + passwordField: 'step04-info/password-field.png', + descriptionField: 'step04-info/description-field.png', + nextButton: 'step04-info/next-button.png', + }, + 5: { + stepIndicator: 'step05-server/server-details-indicator.png', + regionDropdown: 'step05-server/region-dropdown.png', + startNowToggle: 'step05-server/start-now-toggle.png', + nextButton: 'step05-server/next-button.png', + }, + 6: { + stepIndicator: 'step06-admins/admins-indicator.png', + addAdminButton: 'step06-admins/add-admin-button.png', + adminModal: 'step06-admins/admin-modal.png', + searchField: 'step06-admins/search-field.png', + nextButton: 'step06-admins/next-button.png', + }, + 7: { + stepIndicator: 'step07-time/time-limits-indicator.png', + practiceField: 'step07-time/practice-field.png', + qualifyField: 'step07-time/qualify-field.png', + raceField: 'step07-time/race-field.png', + nextButton: 'step07-time/next-button.png', + }, + 8: { + stepIndicator: 'step08-cars/cars-indicator.png', + addCarButton: 'step08-cars/add-car-button.png', + carList: 'step08-cars/car-list.png', + nextButton: 'step08-cars/next-button.png', + }, + 9: { + carModal: 'step09-addcar/car-modal.png', + searchField: 'step09-addcar/search-field.png', + carGrid: 'step09-addcar/car-grid.png', + selectButton: 'step09-addcar/select-button.png', + closeButton: 'step09-addcar/close-button.png', + }, + 10: { + stepIndicator: 'step10-classes/car-classes-indicator.png', + classDropdown: 'step10-classes/class-dropdown.png', + nextButton: 'step10-classes/next-button.png', + }, + 11: { + stepIndicator: 'step11-track/track-indicator.png', + addTrackButton: 'step11-track/add-track-button.png', + trackList: 'step11-track/track-list.png', + nextButton: 'step11-track/next-button.png', + }, + 12: { + trackModal: 'step12-addtrack/track-modal.png', + searchField: 'step12-addtrack/search-field.png', + trackGrid: 'step12-addtrack/track-grid.png', + selectButton: 'step12-addtrack/select-button.png', + closeButton: 'step12-addtrack/close-button.png', + }, + 13: { + stepIndicator: 'step13-trackopts/track-options-indicator.png', + configDropdown: 'step13-trackopts/config-dropdown.png', + nextButton: 'step13-trackopts/next-button.png', + }, + 14: { + stepIndicator: 'step14-tod/time-of-day-indicator.png', + timeSlider: 'step14-tod/time-slider.png', + datePicker: 'step14-tod/date-picker.png', + nextButton: 'step14-tod/next-button.png', + }, + 15: { + stepIndicator: 'step15-weather/weather-indicator.png', + weatherDropdown: 'step15-weather/weather-dropdown.png', + temperatureField: 'step15-weather/temperature-field.png', + nextButton: 'step15-weather/next-button.png', + }, + 16: { + stepIndicator: 'step16-race/race-options-indicator.png', + maxDriversField: 'step16-race/max-drivers-field.png', + rollingStartToggle: 'step16-race/rolling-start-toggle.png', + nextButton: 'step16-race/next-button.png', + }, + 17: { + stepIndicator: 'step17-team/team-driving-indicator.png', + teamDrivingToggle: 'step17-team/team-driving-toggle.png', + nextButton: 'step17-team/next-button.png', + }, + 18: { + stepIndicator: 'step18-conditions/track-conditions-indicator.png', + trackStateDropdown: 'step18-conditions/track-state-dropdown.png', + marblesToggle: 'step18-conditions/marbles-toggle.png', + // NOTE: No checkout button template - automation stops here for safety + }, + }, +} as const; + +/** + * Complete template map for iRacing hosted session automation. + * Templates are organized by common elements and workflow steps. + */ +export const IRacingTemplateMap: IRacingTemplateMapType = { + templateBasePath: 'resources/templates/iracing', + + common: { + loginIndicators: [ + createImageTemplate( + 'login-user-avatar', + TEMPLATE_PATHS.common.userAvatar, + 'User avatar indicating logged-in state', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + createImageTemplate( + 'login-member-badge', + TEMPLATE_PATHS.common.memberBadge, + 'Member badge indicating logged-in state', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + logoutIndicators: [ + createImageTemplate( + 'logout-login-button', + TEMPLATE_PATHS.common.loginButton, + 'Login button indicating logged-out state', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + ], + navigation: { + next: createImageTemplate( + 'nav-next', + TEMPLATE_PATHS.common.nextButton, + 'Next button for wizard navigation', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + back: createImageTemplate( + 'nav-back', + TEMPLATE_PATHS.common.backButton, + 'Back button for wizard navigation', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + checkout: createImageTemplate( + 'nav-checkout', + TEMPLATE_PATHS.common.checkoutButton, + 'Checkout/submit button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + closeModal: createImageTemplate( + 'nav-close-modal', + TEMPLATE_PATHS.common.closeModal, + 'Close modal button', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + loading: [ + createImageTemplate( + 'loading-spinner', + TEMPLATE_PATHS.common.loadingSpinner, + 'Loading spinner indicator', + { confidence: DEFAULT_CONFIDENCE.LOW } + ), + ], + }, + + steps: { + // Step 1: LOGIN (handled externally, templates for detection only) + 1: { + indicators: [ + createImageTemplate( + 'step1-login-form', + TEMPLATE_PATHS.steps[1].loginForm, + 'Login form indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + submit: createImageTemplate( + 'step1-submit', + TEMPLATE_PATHS.steps[1].submitButton, + 'Login submit button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + email: createImageTemplate( + 'step1-email', + TEMPLATE_PATHS.steps[1].emailField, + 'Email input field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + password: createImageTemplate( + 'step1-password', + TEMPLATE_PATHS.steps[1].passwordField, + 'Password input field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 2: HOSTED_RACING + 2: { + indicators: [ + createImageTemplate( + 'step2-hosted-tab', + TEMPLATE_PATHS.steps[2].hostedRacingTab, + 'Hosted racing tab indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + createRace: createImageTemplate( + 'step2-create-race', + TEMPLATE_PATHS.steps[2].createRaceButton, + 'Create a Race button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + }, + + // Step 3: CREATE_RACE + 3: { + indicators: [ + createImageTemplate( + 'step3-modal', + TEMPLATE_PATHS.steps[3].createRaceModal, + 'Create race modal indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + confirm: createImageTemplate( + 'step3-confirm', + TEMPLATE_PATHS.steps[3].confirmButton, + 'Confirm create race button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + }, + + // Step 4: RACE_INFORMATION + 4: { + indicators: [ + createImageTemplate( + 'step4-indicator', + TEMPLATE_PATHS.steps[4].stepIndicator, + 'Race information step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step4-next', + TEMPLATE_PATHS.steps[4].nextButton, + 'Next to Server Details button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + sessionName: createImageTemplate( + 'step4-session-name', + TEMPLATE_PATHS.steps[4].sessionNameField, + 'Session name input field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + password: createImageTemplate( + 'step4-password', + TEMPLATE_PATHS.steps[4].passwordField, + 'Session password input field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + description: createImageTemplate( + 'step4-description', + TEMPLATE_PATHS.steps[4].descriptionField, + 'Session description textarea', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 5: SERVER_DETAILS + 5: { + indicators: [ + createImageTemplate( + 'step5-indicator', + TEMPLATE_PATHS.steps[5].stepIndicator, + 'Server details step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step5-next', + TEMPLATE_PATHS.steps[5].nextButton, + 'Next to Admins button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + region: createImageTemplate( + 'step5-region', + TEMPLATE_PATHS.steps[5].regionDropdown, + 'Server region dropdown', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + startNow: createImageTemplate( + 'step5-start-now', + TEMPLATE_PATHS.steps[5].startNowToggle, + 'Start now toggle', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 6: SET_ADMINS (modal step) + 6: { + indicators: [ + createImageTemplate( + 'step6-indicator', + TEMPLATE_PATHS.steps[6].stepIndicator, + 'Admins step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + addAdmin: createImageTemplate( + 'step6-add-admin', + TEMPLATE_PATHS.steps[6].addAdminButton, + 'Add admin button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + next: createImageTemplate( + 'step6-next', + TEMPLATE_PATHS.steps[6].nextButton, + 'Next to Time Limits button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + modal: { + indicator: createImageTemplate( + 'step6-modal', + TEMPLATE_PATHS.steps[6].adminModal, + 'Add admin modal indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + closeButton: createImageTemplate( + 'step6-modal-close', + TEMPLATE_PATHS.common.closeModal, + 'Close admin modal button', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + searchInput: createImageTemplate( + 'step6-search', + TEMPLATE_PATHS.steps[6].searchField, + 'Admin search field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 7: TIME_LIMITS + 7: { + indicators: [ + createImageTemplate( + 'step7-indicator', + TEMPLATE_PATHS.steps[7].stepIndicator, + 'Time limits step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step7-next', + TEMPLATE_PATHS.steps[7].nextButton, + 'Next to Cars button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + practice: createImageTemplate( + 'step7-practice', + TEMPLATE_PATHS.steps[7].practiceField, + 'Practice length field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + qualify: createImageTemplate( + 'step7-qualify', + TEMPLATE_PATHS.steps[7].qualifyField, + 'Qualify length field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + race: createImageTemplate( + 'step7-race', + TEMPLATE_PATHS.steps[7].raceField, + 'Race length field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 8: SET_CARS + 8: { + indicators: [ + createImageTemplate( + 'step8-indicator', + TEMPLATE_PATHS.steps[8].stepIndicator, + 'Cars step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + addCar: createImageTemplate( + 'step8-add-car', + TEMPLATE_PATHS.steps[8].addCarButton, + 'Add car button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + next: createImageTemplate( + 'step8-next', + TEMPLATE_PATHS.steps[8].nextButton, + 'Next to Track button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + }, + + // Step 9: ADD_CAR (modal step) + 9: { + indicators: [ + createImageTemplate( + 'step9-modal', + TEMPLATE_PATHS.steps[9].carModal, + 'Add car modal indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + select: createImageTemplate( + 'step9-select', + TEMPLATE_PATHS.steps[9].selectButton, + 'Select car button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + modal: { + indicator: createImageTemplate( + 'step9-modal-indicator', + TEMPLATE_PATHS.steps[9].carModal, + 'Car selection modal', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + closeButton: createImageTemplate( + 'step9-close', + TEMPLATE_PATHS.steps[9].closeButton, + 'Close car modal button', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + searchInput: createImageTemplate( + 'step9-search', + TEMPLATE_PATHS.steps[9].searchField, + 'Car search field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 10: SET_CAR_CLASSES + 10: { + indicators: [ + createImageTemplate( + 'step10-indicator', + TEMPLATE_PATHS.steps[10].stepIndicator, + 'Car classes step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step10-next', + TEMPLATE_PATHS.steps[10].nextButton, + 'Next to Track button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + class: createImageTemplate( + 'step10-class', + TEMPLATE_PATHS.steps[10].classDropdown, + 'Car class dropdown', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 11: SET_TRACK + 11: { + indicators: [ + createImageTemplate( + 'step11-indicator', + TEMPLATE_PATHS.steps[11].stepIndicator, + 'Track step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + addTrack: createImageTemplate( + 'step11-add-track', + TEMPLATE_PATHS.steps[11].addTrackButton, + 'Add track button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + next: createImageTemplate( + 'step11-next', + TEMPLATE_PATHS.steps[11].nextButton, + 'Next to Track Options button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + }, + + // Step 12: ADD_TRACK (modal step) + 12: { + indicators: [ + createImageTemplate( + 'step12-modal', + TEMPLATE_PATHS.steps[12].trackModal, + 'Add track modal indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + select: createImageTemplate( + 'step12-select', + TEMPLATE_PATHS.steps[12].selectButton, + 'Select track button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + modal: { + indicator: createImageTemplate( + 'step12-modal-indicator', + TEMPLATE_PATHS.steps[12].trackModal, + 'Track selection modal', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + closeButton: createImageTemplate( + 'step12-close', + TEMPLATE_PATHS.steps[12].closeButton, + 'Close track modal button', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + searchInput: createImageTemplate( + 'step12-search', + TEMPLATE_PATHS.steps[12].searchField, + 'Track search field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 13: TRACK_OPTIONS + 13: { + indicators: [ + createImageTemplate( + 'step13-indicator', + TEMPLATE_PATHS.steps[13].stepIndicator, + 'Track options step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step13-next', + TEMPLATE_PATHS.steps[13].nextButton, + 'Next to Time of Day button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + config: createImageTemplate( + 'step13-config', + TEMPLATE_PATHS.steps[13].configDropdown, + 'Track configuration dropdown', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 14: TIME_OF_DAY + 14: { + indicators: [ + createImageTemplate( + 'step14-indicator', + TEMPLATE_PATHS.steps[14].stepIndicator, + 'Time of day step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step14-next', + TEMPLATE_PATHS.steps[14].nextButton, + 'Next to Weather button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + time: createImageTemplate( + 'step14-time', + TEMPLATE_PATHS.steps[14].timeSlider, + 'Time of day slider', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + date: createImageTemplate( + 'step14-date', + TEMPLATE_PATHS.steps[14].datePicker, + 'Date picker', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 15: WEATHER + 15: { + indicators: [ + createImageTemplate( + 'step15-indicator', + TEMPLATE_PATHS.steps[15].stepIndicator, + 'Weather step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step15-next', + TEMPLATE_PATHS.steps[15].nextButton, + 'Next to Race Options button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + weather: createImageTemplate( + 'step15-weather', + TEMPLATE_PATHS.steps[15].weatherDropdown, + 'Weather type dropdown', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + temperature: createImageTemplate( + 'step15-temperature', + TEMPLATE_PATHS.steps[15].temperatureField, + 'Temperature field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 16: RACE_OPTIONS + 16: { + indicators: [ + createImageTemplate( + 'step16-indicator', + TEMPLATE_PATHS.steps[16].stepIndicator, + 'Race options step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step16-next', + TEMPLATE_PATHS.steps[16].nextButton, + 'Next to Track Conditions button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + maxDrivers: createImageTemplate( + 'step16-max-drivers', + TEMPLATE_PATHS.steps[16].maxDriversField, + 'Maximum drivers field', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + rollingStart: createImageTemplate( + 'step16-rolling-start', + TEMPLATE_PATHS.steps[16].rollingStartToggle, + 'Rolling start toggle', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 17: TEAM_DRIVING + 17: { + indicators: [ + createImageTemplate( + 'step17-indicator', + TEMPLATE_PATHS.steps[17].stepIndicator, + 'Team driving step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + next: createImageTemplate( + 'step17-next', + TEMPLATE_PATHS.steps[17].nextButton, + 'Next to Track Conditions button', + { confidence: DEFAULT_CONFIDENCE.HIGH } + ), + }, + fields: { + teamDriving: createImageTemplate( + 'step17-team-driving', + TEMPLATE_PATHS.steps[17].teamDrivingToggle, + 'Team driving toggle', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + + // Step 18: TRACK_CONDITIONS (final step - no checkout for safety) + 18: { + indicators: [ + createImageTemplate( + 'step18-indicator', + TEMPLATE_PATHS.steps[18].stepIndicator, + 'Track conditions step indicator', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + ], + buttons: { + // NOTE: No checkout button - automation intentionally stops here + // User must manually review and submit + }, + fields: { + trackState: createImageTemplate( + 'step18-track-state', + TEMPLATE_PATHS.steps[18].trackStateDropdown, + 'Track state dropdown', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + marbles: createImageTemplate( + 'step18-marbles', + TEMPLATE_PATHS.steps[18].marblesToggle, + 'Marbles toggle', + { confidence: DEFAULT_CONFIDENCE.STANDARD } + ), + }, + }, + }, +}; + +/** + * Get templates for a specific step. + */ +export function getStepTemplates(stepId: number): StepTemplates | undefined { + return IRacingTemplateMap.steps[stepId]; +} + +/** + * Check if a step is a modal step (requires opening a secondary dialog). + */ +export function isModalStep(stepId: number): boolean { + const templates = IRacingTemplateMap.steps[stepId]; + return templates?.modal !== undefined; +} + +/** + * Get the step name for logging/debugging. + */ +export function getStepName(stepId: number): string { + const stepNames: Record = { + 1: 'LOGIN', + 2: 'HOSTED_RACING', + 3: 'CREATE_RACE', + 4: 'RACE_INFORMATION', + 5: 'SERVER_DETAILS', + 6: 'SET_ADMINS', + 7: 'TIME_LIMITS', + 8: 'SET_CARS', + 9: 'ADD_CAR', + 10: 'SET_CAR_CLASSES', + 11: 'SET_TRACK', + 12: 'ADD_TRACK', + 13: 'TRACK_OPTIONS', + 14: 'TIME_OF_DAY', + 15: 'WEATHER', + 16: 'RACE_OPTIONS', + 17: 'TEAM_DRIVING', + 18: 'TRACK_CONDITIONS', + }; + return stepNames[stepId] || `UNKNOWN_STEP_${stepId}`; +} + +/** + * Get all login indicator templates. + */ +export function getLoginIndicators(): ImageTemplate[] { + return IRacingTemplateMap.common.loginIndicators; +} + +/** + * Get all logout indicator templates. + */ +export function getLogoutIndicators(): ImageTemplate[] { + return IRacingTemplateMap.common.logoutIndicators; +} \ No newline at end of file diff --git a/packages/infrastructure/config/AutomationConfig.ts b/packages/infrastructure/config/AutomationConfig.ts index d554243fc..0200921d0 100644 --- a/packages/infrastructure/config/AutomationConfig.ts +++ b/packages/infrastructure/config/AutomationConfig.ts @@ -51,7 +51,7 @@ export interface AutomationEnvironmentConfig { */ export function loadAutomationConfig(): AutomationEnvironmentConfig { const modeEnv = process.env.AUTOMATION_MODE; - const mode: AutomationMode = isValidAutomationMode(modeEnv) ? modeEnv : 'dev'; + const mode: AutomationMode = isValidAutomationMode(modeEnv) ? modeEnv : 'production'; return { mode, diff --git a/tests/unit/infrastructure/AutomationConfig.test.ts b/tests/unit/infrastructure/AutomationConfig.test.ts index b909b732f..5b8f6403b 100644 --- a/tests/unit/infrastructure/AutomationConfig.test.ts +++ b/tests/unit/infrastructure/AutomationConfig.test.ts @@ -16,12 +16,12 @@ describe('AutomationConfig', () => { describe('loadAutomationConfig', () => { describe('default configuration', () => { - it('should return dev mode when AUTOMATION_MODE is not set', () => { + it('should return production mode when AUTOMATION_MODE is not set', () => { delete process.env.AUTOMATION_MODE; const config = loadAutomationConfig(); - expect(config.mode).toBe('dev'); + expect(config.mode).toBe('production'); }); it('should return default devTools configuration', () => { @@ -161,12 +161,12 @@ describe('AutomationConfig', () => { expect(config.nutJs?.confidence).toBe(0.9); }); - it('should fallback to dev mode for invalid AUTOMATION_MODE', () => { + it('should fallback to production mode for invalid AUTOMATION_MODE', () => { process.env.AUTOMATION_MODE = 'invalid-mode'; const config = loadAutomationConfig(); - expect(config.mode).toBe('dev'); + expect(config.mode).toBe('production'); }); });