working companion prototype

This commit is contained in:
2025-11-24 23:32:36 +01:00
parent e7978024d7
commit e2bea9a126
175 changed files with 23227 additions and 3519 deletions

View File

@@ -0,0 +1,436 @@
/**
* Selector configuration for template generation.
* Maps HTML fixture files to CSS selectors and output PNG paths.
*
* Since the iRacing UI uses Chakra UI with hashed CSS classes,
* we rely on text content, aria-labels, and semantic selectors.
*/
export interface ElementCapture {
selector: string;
outputPath: string;
description: string;
waitFor?: string;
}
export interface FixtureConfig {
htmlFile: string;
captures: ElementCapture[];
}
export const TEMPLATE_BASE_PATH = 'resources/templates/iracing';
export const FIXTURES_BASE_PATH = 'resources/iracing-hosted-sessions';
export const SELECTOR_CONFIG: FixtureConfig[] = [
{
htmlFile: '01-hosted-racing.html',
captures: [
{
selector: 'text="Hosted Racing"',
outputPath: 'step02-hosted/hosted-racing-tab.png',
description: 'Hosted Racing tab indicator',
},
{
selector: 'text="Create a Race"',
outputPath: 'step02-hosted/create-race-button.png',
description: 'Create a Race button',
},
],
},
{
htmlFile: '02-create-a-race.html',
captures: [
{
selector: '[role="dialog"]',
outputPath: 'step03-create/create-race-modal.png',
description: 'Create race modal',
},
{
selector: 'button:has-text("Create")',
outputPath: 'step03-create/confirm-button.png',
description: 'Confirm create race button',
},
],
},
{
htmlFile: '03-race-information.html',
captures: [
{
selector: 'text="Race Information"',
outputPath: 'step04-info/race-info-indicator.png',
description: 'Race information step indicator',
},
{
selector: 'input[placeholder*="Session" i], input[name*="session" i], label:has-text("Session Name") + input',
outputPath: 'step04-info/session-name-field.png',
description: 'Session name input field',
},
{
selector: 'input[type="password"], label:has-text("Password") + input',
outputPath: 'step04-info/password-field.png',
description: 'Session password field',
},
{
selector: 'textarea, label:has-text("Description") + textarea',
outputPath: 'step04-info/description-field.png',
description: 'Session description textarea',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step04-info/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '04-server-details.html',
captures: [
{
selector: 'text="Server Details"',
outputPath: 'step05-server/server-details-indicator.png',
description: 'Server details step indicator',
},
{
selector: 'select, [role="listbox"], label:has-text("Region") ~ select',
outputPath: 'step05-server/region-dropdown.png',
description: 'Server region dropdown',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step05-server/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '05-set-admins.html',
captures: [
{
selector: 'text="Admins"',
outputPath: 'step06-admins/admins-indicator.png',
description: 'Admins step indicator',
},
{
selector: 'button:has-text("Add Admin")',
outputPath: 'step06-admins/add-admin-button.png',
description: 'Add admin button',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step06-admins/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '06-add-an-admin.html',
captures: [
{
selector: '[role="dialog"]',
outputPath: 'step06-admins/admin-modal.png',
description: 'Add admin modal',
},
{
selector: 'input[type="search"], input[placeholder*="search" i]',
outputPath: 'step06-admins/search-field.png',
description: 'Admin search field',
},
],
},
{
htmlFile: '07-time-limits.html',
captures: [
{
selector: 'text="Time Limits"',
outputPath: 'step07-time/time-limits-indicator.png',
description: 'Time limits step indicator',
},
{
selector: 'label:has-text("Practice") ~ input, input[name*="practice" i]',
outputPath: 'step07-time/practice-field.png',
description: 'Practice length field',
},
{
selector: 'label:has-text("Qualify") ~ input, input[name*="qualify" i]',
outputPath: 'step07-time/qualify-field.png',
description: 'Qualify length field',
},
{
selector: 'label:has-text("Race") ~ input, input[name*="race" i]',
outputPath: 'step07-time/race-field.png',
description: 'Race length field',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step07-time/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '08-set-cars.html',
captures: [
{
selector: 'text="Cars"',
outputPath: 'step08-cars/cars-indicator.png',
description: 'Cars step indicator',
},
{
selector: 'button:has-text("Add Car"), button:has-text("Add a Car")',
outputPath: 'step08-cars/add-car-button.png',
description: 'Add car button',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step08-cars/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '09-add-a-car.html',
captures: [
{
selector: '[role="dialog"]',
outputPath: 'step09-addcar/car-modal.png',
description: 'Add car modal',
},
{
selector: 'input[type="search"], input[placeholder*="search" i]',
outputPath: 'step09-addcar/search-field.png',
description: 'Car search field',
},
{
selector: 'button:has-text("Select"), button:has-text("Add")',
outputPath: 'step09-addcar/select-button.png',
description: 'Select car button',
},
{
selector: 'button[aria-label="Close"], button:has-text("Close")',
outputPath: 'step09-addcar/close-button.png',
description: 'Close modal button',
},
],
},
{
htmlFile: '10-set-car-classes.html',
captures: [
{
selector: 'text="Car Classes"',
outputPath: 'step10-classes/car-classes-indicator.png',
description: 'Car classes step indicator',
},
{
selector: 'select, [role="listbox"]',
outputPath: 'step10-classes/class-dropdown.png',
description: 'Car class dropdown',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step10-classes/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '11-set-track.html',
captures: [
{
selector: 'text="Track"',
outputPath: 'step11-track/track-indicator.png',
description: 'Track step indicator',
},
{
selector: 'button:has-text("Add Track"), button:has-text("Add a Track")',
outputPath: 'step11-track/add-track-button.png',
description: 'Add track button',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step11-track/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '12-add-a-track.html',
captures: [
{
selector: '[role="dialog"]',
outputPath: 'step12-addtrack/track-modal.png',
description: 'Add track modal',
},
{
selector: 'input[type="search"], input[placeholder*="search" i]',
outputPath: 'step12-addtrack/search-field.png',
description: 'Track search field',
},
{
selector: 'button:has-text("Select"), button:has-text("Add")',
outputPath: 'step12-addtrack/select-button.png',
description: 'Select track button',
},
{
selector: 'button[aria-label="Close"], button:has-text("Close")',
outputPath: 'step12-addtrack/close-button.png',
description: 'Close modal button',
},
],
},
{
htmlFile: '13-track-options.html',
captures: [
{
selector: 'text="Track Options"',
outputPath: 'step13-trackopts/track-options-indicator.png',
description: 'Track options step indicator',
},
{
selector: 'select, [role="listbox"]',
outputPath: 'step13-trackopts/config-dropdown.png',
description: 'Track configuration dropdown',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step13-trackopts/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '14-time-of-day.html',
captures: [
{
selector: 'text="Time of Day"',
outputPath: 'step14-tod/time-of-day-indicator.png',
description: 'Time of day step indicator',
},
{
selector: 'input[type="range"], [role="slider"]',
outputPath: 'step14-tod/time-slider.png',
description: 'Time of day slider',
},
{
selector: 'input[type="date"], [data-testid*="date"]',
outputPath: 'step14-tod/date-picker.png',
description: 'Date picker',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step14-tod/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '15-weather.html',
captures: [
{
selector: 'text="Weather"',
outputPath: 'step15-weather/weather-indicator.png',
description: 'Weather step indicator',
},
{
selector: 'select, [role="listbox"]',
outputPath: 'step15-weather/weather-dropdown.png',
description: 'Weather type dropdown',
},
{
selector: 'input[type="number"], label:has-text("Temperature") ~ input',
outputPath: 'step15-weather/temperature-field.png',
description: 'Temperature field',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step15-weather/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '16-race-options.html',
captures: [
{
selector: 'text="Race Options"',
outputPath: 'step16-race/race-options-indicator.png',
description: 'Race options step indicator',
},
{
selector: 'input[type="number"], label:has-text("Max") ~ input',
outputPath: 'step16-race/max-drivers-field.png',
description: 'Maximum drivers field',
},
{
selector: '[role="switch"], input[type="checkbox"]',
outputPath: 'step16-race/rolling-start-toggle.png',
description: 'Rolling start toggle',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step16-race/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '17-team-driving.html',
captures: [
{
selector: 'text="Team Driving"',
outputPath: 'step17-team/team-driving-indicator.png',
description: 'Team driving step indicator',
},
{
selector: '[role="switch"], input[type="checkbox"]',
outputPath: 'step17-team/team-driving-toggle.png',
description: 'Team driving toggle',
},
{
selector: 'button:has-text("Next")',
outputPath: 'step17-team/next-button.png',
description: 'Next button',
},
],
},
{
htmlFile: '18-track-conditions.html',
captures: [
{
selector: 'text="Track Conditions"',
outputPath: 'step18-conditions/track-conditions-indicator.png',
description: 'Track conditions step indicator',
},
{
selector: 'select, [role="listbox"]',
outputPath: 'step18-conditions/track-state-dropdown.png',
description: 'Track state dropdown',
},
{
selector: '[role="switch"], input[type="checkbox"]',
outputPath: 'step18-conditions/marbles-toggle.png',
description: 'Marbles toggle',
},
],
},
];
/**
* Common templates that appear across multiple steps
*/
export const COMMON_CAPTURES: ElementCapture[] = [
{
selector: 'button:has-text("Next")',
outputPath: 'common/next-button.png',
description: 'Generic next button for wizard navigation',
},
{
selector: 'button:has-text("Back")',
outputPath: 'common/back-button.png',
description: 'Generic back button for wizard navigation',
},
{
selector: 'button[aria-label="Close"], [aria-label="close"]',
outputPath: 'common/close-modal-button.png',
description: 'Close modal button',
},
];

View File

@@ -0,0 +1,254 @@
#!/usr/bin/env npx tsx
/**
* Template Generation Script
*
* Generates PNG templates from HTML fixtures using Playwright.
* These templates are used for image-based UI matching in OS-level automation.
*
* Usage: npx tsx scripts/generate-templates/index.ts
*/
import { chromium, type Browser, type Page } from 'playwright';
import * as fs from 'fs';
import * as path from 'path';
import {
SELECTOR_CONFIG,
COMMON_CAPTURES,
TEMPLATE_BASE_PATH,
FIXTURES_BASE_PATH,
type ElementCapture,
type FixtureConfig,
} from './SelectorConfig';
const PROJECT_ROOT = process.cwd();
interface CaptureResult {
outputPath: string;
success: boolean;
error?: string;
}
interface FixtureResult {
htmlFile: string;
captures: CaptureResult[];
}
async function ensureDirectoryExists(filePath: string): Promise<void> {
const dir = path.dirname(filePath);
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true });
console.log(` Created directory: ${dir}`);
}
}
async function captureElement(
page: Page,
capture: ElementCapture,
outputBasePath: string
): Promise<CaptureResult> {
const fullOutputPath = path.join(outputBasePath, capture.outputPath);
try {
await ensureDirectoryExists(fullOutputPath);
const element = await page.locator(capture.selector).first();
const isVisible = await element.isVisible().catch(() => false);
if (!isVisible) {
console.log(` ⚠ Element not visible: ${capture.description}`);
return {
outputPath: capture.outputPath,
success: false,
error: 'Element not visible',
};
}
await element.screenshot({ path: fullOutputPath });
console.log(` ✓ Captured: ${capture.description}${capture.outputPath}`);
return {
outputPath: capture.outputPath,
success: true,
};
} catch (error) {
const errorMessage = error instanceof Error ? error.message : String(error);
console.log(` ✗ Failed: ${capture.description} - ${errorMessage}`);
return {
outputPath: capture.outputPath,
success: false,
error: errorMessage,
};
}
}
async function processFixture(
browser: Browser,
config: FixtureConfig,
fixturesBasePath: string,
outputBasePath: string
): Promise<FixtureResult> {
const htmlPath = path.join(fixturesBasePath, config.htmlFile);
const fileUrl = `file://${htmlPath}`;
console.log(`\n📄 Processing: ${config.htmlFile}`);
if (!fs.existsSync(htmlPath)) {
console.log(` ✗ File not found: ${htmlPath}`);
return {
htmlFile: config.htmlFile,
captures: config.captures.map((c) => ({
outputPath: c.outputPath,
success: false,
error: 'HTML file not found',
})),
};
}
const context = await browser.newContext({
viewport: { width: 1920, height: 1080 },
});
const page = await context.newPage();
try {
await page.goto(fileUrl, { waitUntil: 'networkidle' });
await page.waitForTimeout(1000);
const captures: CaptureResult[] = [];
for (const capture of config.captures) {
const result = await captureElement(page, capture, outputBasePath);
captures.push(result);
}
return {
htmlFile: config.htmlFile,
captures,
};
} finally {
await context.close();
}
}
async function captureCommonElements(
browser: Browser,
fixturesBasePath: string,
outputBasePath: string
): Promise<CaptureResult[]> {
console.log('\n📦 Capturing common elements...');
const sampleFixture = SELECTOR_CONFIG.find((c) =>
fs.existsSync(path.join(fixturesBasePath, c.htmlFile))
);
if (!sampleFixture) {
console.log(' ✗ No fixture files found for common element capture');
return COMMON_CAPTURES.map((c) => ({
outputPath: c.outputPath,
success: false,
error: 'No fixture files available',
}));
}
const htmlPath = path.join(fixturesBasePath, sampleFixture.htmlFile);
const fileUrl = `file://${htmlPath}`;
const context = await browser.newContext({
viewport: { width: 1920, height: 1080 },
});
const page = await context.newPage();
try {
await page.goto(fileUrl, { waitUntil: 'networkidle' });
await page.waitForTimeout(1000);
const captures: CaptureResult[] = [];
for (const capture of COMMON_CAPTURES) {
const result = await captureElement(page, capture, outputBasePath);
captures.push(result);
}
return captures;
} finally {
await context.close();
}
}
async function main(): Promise<void> {
console.log('🚀 Starting template generation...\n');
const fixturesBasePath = path.join(PROJECT_ROOT, FIXTURES_BASE_PATH);
const outputBasePath = path.join(PROJECT_ROOT, TEMPLATE_BASE_PATH);
console.log(`📁 Fixtures path: ${fixturesBasePath}`);
console.log(`📁 Output path: ${outputBasePath}`);
if (!fs.existsSync(fixturesBasePath)) {
console.error(`\n❌ Fixtures directory not found: ${fixturesBasePath}`);
process.exit(1);
}
await ensureDirectoryExists(path.join(outputBasePath, '.gitkeep'));
console.log('\n🌐 Launching browser...');
const browser = await chromium.launch({
headless: true,
});
try {
const results: FixtureResult[] = [];
for (const config of SELECTOR_CONFIG) {
const result = await processFixture(
browser,
config,
fixturesBasePath,
outputBasePath
);
results.push(result);
}
const commonResults = await captureCommonElements(
browser,
fixturesBasePath,
outputBasePath
);
console.log('\n📊 Summary:');
console.log('─'.repeat(50));
let totalCaptures = 0;
let successfulCaptures = 0;
for (const result of results) {
const successful = result.captures.filter((c) => c.success).length;
const total = result.captures.length;
totalCaptures += total;
successfulCaptures += successful;
console.log(` ${result.htmlFile}: ${successful}/${total} captures`);
}
const commonSuccessful = commonResults.filter((c) => c.success).length;
totalCaptures += commonResults.length;
successfulCaptures += commonSuccessful;
console.log(` common elements: ${commonSuccessful}/${commonResults.length} captures`);
console.log('─'.repeat(50));
console.log(` Total: ${successfulCaptures}/${totalCaptures} captures successful`);
if (successfulCaptures < totalCaptures) {
console.log('\n⚠ Some captures failed. This may be due to:');
console.log(' - Elements not present in the HTML fixtures');
console.log(' - CSS selectors needing adjustment');
console.log(' - Dynamic content not rendering in static HTML');
}
console.log('\n✅ Template generation complete!');
console.log(` Templates saved to: ${outputBasePath}`);
} finally {
await browser.close();
}
}
main().catch((error) => {
console.error('\n❌ Fatal error:', error);
process.exit(1);
});

View File

@@ -0,0 +1,226 @@
/**
* Generate test fixtures by taking screenshots of static HTML fixture pages.
* This creates controlled test images for template matching verification.
*/
import puppeteer from 'puppeteer';
import * as path from 'path';
import * as fs from 'fs';
const FIXTURE_HTML_DIR = path.join(__dirname, '../resources/iracing-hosted-sessions');
const OUTPUT_DIR = path.join(__dirname, '../resources/test-fixtures');
async function generateFixtures(): Promise<void> {
console.log('🚀 Starting fixture generation...');
// Ensure output directory exists
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
console.log(`📁 Created output directory: ${OUTPUT_DIR}`);
}
const browser = await puppeteer.launch({
headless: true,
});
try {
const page = await browser.newPage();
// Set viewport to match typical screen size (Retina 2x)
await page.setViewport({
width: 1920,
height: 1080,
deviceScaleFactor: 2, // Retina display
});
// List of HTML fixtures to screenshot
const fixtures = [
{ file: '01-hosted-racing.html', name: 'hosted-racing' },
{ file: '02-create-a-race.html', name: 'create-race' },
{ file: '03-race-information.html', name: 'race-information' },
];
for (const fixture of fixtures) {
const htmlPath = path.join(FIXTURE_HTML_DIR, fixture.file);
if (!fs.existsSync(htmlPath)) {
console.log(`⚠️ Skipping ${fixture.file} - file not found`);
continue;
}
console.log(`📸 Processing ${fixture.file}...`);
// Load the HTML file
await page.goto(`file://${htmlPath}`, {
waitUntil: 'networkidle0',
timeout: 30000,
});
// Take screenshot
const outputPath = path.join(OUTPUT_DIR, `${fixture.name}-screenshot.png`);
await page.screenshot({
path: outputPath,
fullPage: false, // Just the viewport
});
console.log(`✅ Saved: ${outputPath}`);
}
console.log('\n🎉 Fixture generation complete!');
console.log(`📁 Screenshots saved to: ${OUTPUT_DIR}`);
} finally {
await browser.close();
}
}
// Also create a simple synthetic test pattern for algorithm verification
async function createSyntheticTestPattern(): Promise<void> {
const sharp = (await import('sharp')).default;
console.log('\n🔧 Creating synthetic test patterns...');
// Ensure output directory exists
if (!fs.existsSync(OUTPUT_DIR)) {
fs.mkdirSync(OUTPUT_DIR, { recursive: true });
console.log(`📁 Created output directory: ${OUTPUT_DIR}`);
}
// Create a simple test image (red square on white background)
const width = 200;
const height = 200;
const channels = 4;
// White background with a distinct blue rectangle in the center
const imageData = Buffer.alloc(width * height * channels);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const idx = (y * width + x) * channels;
// Create a blue rectangle from (50,50) to (150,150)
if (x >= 50 && x < 150 && y >= 50 && y < 150) {
imageData[idx] = 0; // R
imageData[idx + 1] = 0; // G
imageData[idx + 2] = 255; // B
imageData[idx + 3] = 255; // A
} else {
// White background
imageData[idx] = 255; // R
imageData[idx + 1] = 255; // G
imageData[idx + 2] = 255; // B
imageData[idx + 3] = 255; // A
}
}
}
const testImagePath = path.join(OUTPUT_DIR, 'synthetic-test-image.png');
await sharp(imageData, {
raw: { width, height, channels },
})
.png()
.toFile(testImagePath);
console.log(`✅ Saved synthetic test image: ${testImagePath}`);
// Create a template (the blue rectangle portion)
const templateWidth = 100;
const templateHeight = 100;
const templateData = Buffer.alloc(templateWidth * templateHeight * channels);
for (let y = 0; y < templateHeight; y++) {
for (let x = 0; x < templateWidth; x++) {
const idx = (y * templateWidth + x) * channels;
// Blue fill
templateData[idx] = 0; // R
templateData[idx + 1] = 0; // G
templateData[idx + 2] = 255; // B
templateData[idx + 3] = 255; // A
}
}
const templatePath = path.join(OUTPUT_DIR, 'synthetic-template.png');
await sharp(templateData, {
raw: { width: templateWidth, height: templateHeight, channels },
})
.png()
.toFile(templatePath);
console.log(`✅ Saved synthetic template: ${templatePath}`);
// Create a more realistic pattern with gradients (better for NCC)
const gradientWidth = 400;
const gradientHeight = 300;
const gradientData = Buffer.alloc(gradientWidth * gradientHeight * channels);
for (let y = 0; y < gradientHeight; y++) {
for (let x = 0; x < gradientWidth; x++) {
const idx = (y * gradientWidth + x) * channels;
// Create gradient background
const bgGray = Math.floor((x / gradientWidth) * 128 + 64);
// Add a distinct pattern in the center (button-like)
if (x >= 150 && x < 250 && y >= 100 && y < 150) {
// Darker rectangle with slight gradient
const buttonGray = 50 + Math.floor((x - 150) / 100 * 30);
gradientData[idx] = buttonGray;
gradientData[idx + 1] = buttonGray;
gradientData[idx + 2] = buttonGray + 20; // Slight blue tint
gradientData[idx + 3] = 255;
} else {
gradientData[idx] = bgGray;
gradientData[idx + 1] = bgGray;
gradientData[idx + 2] = bgGray;
gradientData[idx + 3] = 255;
}
}
}
const gradientImagePath = path.join(OUTPUT_DIR, 'gradient-test-image.png');
await sharp(gradientData, {
raw: { width: gradientWidth, height: gradientHeight, channels },
})
.png()
.toFile(gradientImagePath);
console.log(`✅ Saved gradient test image: ${gradientImagePath}`);
// Extract the button region as a template
const buttonTemplateWidth = 100;
const buttonTemplateHeight = 50;
const buttonTemplateData = Buffer.alloc(buttonTemplateWidth * buttonTemplateHeight * channels);
for (let y = 0; y < buttonTemplateHeight; y++) {
for (let x = 0; x < buttonTemplateWidth; x++) {
const idx = (y * buttonTemplateWidth + x) * channels;
const buttonGray = 50 + Math.floor(x / 100 * 30);
buttonTemplateData[idx] = buttonGray;
buttonTemplateData[idx + 1] = buttonGray;
buttonTemplateData[idx + 2] = buttonGray + 20;
buttonTemplateData[idx + 3] = 255;
}
}
const buttonTemplatePath = path.join(OUTPUT_DIR, 'gradient-button-template.png');
await sharp(buttonTemplateData, {
raw: { width: buttonTemplateWidth, height: buttonTemplateHeight, channels },
})
.png()
.toFile(buttonTemplatePath);
console.log(`✅ Saved gradient button template: ${buttonTemplatePath}`);
}
// Run both
async function main(): Promise<void> {
try {
await createSyntheticTestPattern();
await generateFixtures();
} catch (error) {
console.error('❌ Error generating fixtures:', error);
process.exit(1);
}
}
main();