From 4b8c70978fdd0de42b7f3deeeda3e7ec2301b250 Mon Sep 17 00:00:00 2001 From: Marc Mintel Date: Sun, 30 Nov 2025 17:37:03 +0100 Subject: [PATCH] wip --- docs/TESTS.md | 44 ++++- .../core/PlaywrightAutomationAdapter.ts | 57 ++++++- .../automation/core/WizardStepOrchestrator.ts | 9 ++ tests/e2e/automation.e2e.test.ts | 5 +- .../full-hosted-session.workflow.e2e.test.ts | 153 ++++++++++++++++++ .../steps-07-09-cars-flow.e2e.test.ts | 35 ++-- 6 files changed, 283 insertions(+), 20 deletions(-) create mode 100644 tests/e2e/workflows/full-hosted-session.workflow.e2e.test.ts diff --git a/docs/TESTS.md b/docs/TESTS.md index ca05e120e..8e99f93a6 100644 --- a/docs/TESTS.md +++ b/docs/TESTS.md @@ -1243,4 +1243,46 @@ GridPilot's testing strategy ensures: - **Browser automation works correctly** (Docker E2E tests with real fixtures) - **OS-level automation works correctly** (Native macOS E2E tests with display access) -By following BDD principles and maintaining clear test organization, the team can confidently evolve GridPilot while preserving correctness and stability. \ No newline at end of file +### Hosted Session Automation Test Pyramid + +For the iRacing hosted-session automation, confidence is provided by these concrete suites: + +- **Domain/Application unit tests** + - Entities and value objects such as [`AutomationSession`](packages/domain/entities/AutomationSession.ts:1), [`SessionState`](packages/domain/value-objects/SessionState.ts:1), [`CheckoutState`](packages/domain/value-objects/CheckoutState.ts:1), [`CheckoutConfirmation`](packages/domain/value-objects/CheckoutConfirmation.ts:1), and [`RaceCreationResult`](packages/domain/value-objects/RaceCreationResult.ts:1). + - Use cases such as [`StartAutomationSessionUseCase`](packages/application/use-cases/StartAutomationSessionUseCase.ts:1), [`VerifyAuthenticatedPageUseCase`](packages/application/use-cases/VerifyAuthenticatedPageUseCase.ts:1), [`CompleteRaceCreationUseCase`](packages/application/use-cases/CompleteRaceCreationUseCase.ts:1), and [`ConfirmCheckoutUseCase`](packages/application/use-cases/ConfirmCheckoutUseCase.ts:1). + +- **Infrastructure / automation integration tests** + - Adapter wiring and fixture serving via [`FixtureServer.integration.test.ts`](tests/integration/infrastructure/FixtureServer.integration.test.ts:1). + - Playwright lifecycle and overlay wiring such as [`BrowserModeIntegration.test.ts`](tests/integration/infrastructure/BrowserModeIntegration.test.ts:1) and automation-flow tests like [`CarsFlow.integration.test.ts`](tests/integration/infrastructure/automation/CarsFlow.integration.test.ts:1). + - UI/overlay integration via [`renderer-overlay.integration.test.ts`](tests/integration/interface/renderer/renderer-overlay.integration.test.ts:1). + +- **Fixture-based step E2E tests (per-step behavior)** + - One test per wizard step under [`tests/e2e/steps`](tests/e2e/steps:1), all using the shared [`StepHarness`](tests/e2e/support/StepHarness.ts:1) and [`FixtureServer`](packages/infrastructure/adapters/automation/engine/FixtureServer.ts:1). + - These validate DOM-level selectors / flows for each step (1–18) against fixture-backed HTML, and are considered canonical for step behavior. + +- **Fixture-based workflow E2E tests (button → auth entry → automation → confirmation boundary)** + - Workflow-focused tests under [`tests/e2e/workflows`](tests/e2e/workflows:1) that drive the `PlaywrightAutomationAdapter` + `WizardStepOrchestrator` across multiple steps using the fixture server. + - Example: [`steps-07-09-cars-flow.e2e.test.ts`](tests/e2e/workflows/steps-07-09-cars-flow.e2e.test.ts:1) exercises cross-step cars flow, while [`full-hosted-session.workflow.e2e.test.ts`](tests/e2e/workflows/full-hosted-session.workflow.e2e.test.ts:1) runs a full 1–18 workflow via [`MockAutomationEngineAdapter`](packages/infrastructure/adapters/automation/engine/MockAutomationEngineAdapter.ts:1) and [`StartAutomationSessionUseCase`](packages/application/use-cases/StartAutomationSessionUseCase.ts:1), asserting final `SessionState` and step position. + - Additional workflow scenarios cover mid-flow failure using [`MockBrowserAutomationAdapter`](packages/infrastructure/adapters/automation/engine/MockBrowserAutomationAdapter.ts:1), ensuring failure states and diagnostics are surfaced without emitting false confirmations. + +- **Opt-in real-world automation smoke tests (true iRacing / NutJs)** + - The legacy real iRacing automation suite [`automation.e2e.test.ts`](tests/e2e/automation.e2e.test.ts:1) is treated as a smoke-only, opt-in layer. + - It is gated by `RUN_REAL_AUTOMATION_SMOKE=1` and should not run in normal CI; it exists to validate that NutJs / template-based automation still matches the real UI for a small number of manual smoke runs on a prepared macOS environment. + +#### Confidence expectations + +- For **normal changes** to hosted-session automation (selectors, step logic, overlay behavior, authentication, or confirmation flows), the following suites must pass to claim "high confidence": + - All relevant **unit tests** in `tests/unit` that touch the changed domain/use-case code. + - All relevant **integration tests** in `tests/integration` for the affected adapters. + - All **step E2E tests** under [`tests/e2e/steps`](tests/e2e/steps:1). + - All **workflow E2E tests** under [`tests/e2e/workflows`](tests/e2e/workflows:1). + +- The **real-world smoke suite** in [`tests/e2e/automation.e2e.test.ts`](tests/e2e/automation.e2e.test.ts:1) is an additional, manual confidence layer and should be run only on configured machines when validating large changes to NutJs automation, template packs, or iRacing UI assumptions. + +- When adding new behavior: + - Prefer **unit tests** for domain/application changes. + - Add or extend **integration tests** when introducing new adapters or external integration. + - Add **step E2E tests** when changing DOM/step behavior for a specific wizard step. + - Add or extend **workflow E2E tests** when behavior spans multiple steps, touches authentication/session lifecycle, or affects confirmation/checkout behavior end-to-end. + +By following BDD principles and maintaining clear test organization, the team can confidently evolve GridPilot while preserving correctness and stability, with a dedicated, layered confidence story for hosted-session automation. \ No newline at end of file diff --git a/packages/infrastructure/adapters/automation/core/PlaywrightAutomationAdapter.ts b/packages/infrastructure/adapters/automation/core/PlaywrightAutomationAdapter.ts index e998178d5..586df7bd1 100644 --- a/packages/infrastructure/adapters/automation/core/PlaywrightAutomationAdapter.ts +++ b/packages/infrastructure/adapters/automation/core/PlaywrightAutomationAdapter.ts @@ -1,6 +1,7 @@ import type { Browser, Page, BrowserContext } from 'playwright'; import * as fs from 'fs'; import * as path from 'path'; +import { extractDom, ExportedElement } from '../../../../scripts/dom-export/domExtractor'; import { StepId } from '../../../../domain/value-objects/StepId'; import { AuthenticationState } from '../../../../domain/value-objects/AuthenticationState'; import { BrowserAuthenticationState } from '../../../../domain/value-objects/BrowserAuthenticationState'; @@ -22,6 +23,7 @@ import { Result } from '../../../../shared/result/Result'; import { IRACING_SELECTORS, IRACING_URLS, IRACING_TIMEOUTS, ALL_BLOCKED_SELECTORS, BLOCKED_KEYWORDS } from '../dom/IRacingSelectors'; import { SessionCookieStore } from '../auth/SessionCookieStore'; import { PlaywrightBrowserSession } from './PlaywrightBrowserSession'; +import { getFixtureForStep } from '../engine/FixtureServer'; import { BrowserModeConfigLoader, BrowserMode } from '../../../config/BrowserModeConfig'; import { getAutomationMode } from '../../../config/AutomationConfig'; import { PageStateValidator, PageStateValidation, PageStateValidationResult } from '../../../../domain/services/PageStateValidator'; @@ -645,6 +647,21 @@ export class PlaywrightAutomationAdapter implements IBrowserAutomation, IAuthent } this.syncSessionStateFromBrowser(); + + if (this.config.mode === 'mock' && this.config.baseUrl) { + try { + const fixture = getFixtureForStep(1); + if (fixture) { + const base = this.config.baseUrl.replace(/\/$/, ''); + await this.navigator.navigateToPage(`${base}/${fixture}`); + } + } catch (error) { + this.log('debug', 'Initial fixture navigation failed (mock mode)', { + error: String(error), + }); + } + } + return { success: true }; } @@ -871,13 +888,13 @@ export class PlaywrightAutomationAdapter implements IBrowserAutomation, IAuthent * * Error dumps are always kept and not subject to cleanup. */ - private async saveDebugInfo(stepName: string, error: Error): Promise<{ screenshotPath?: string; htmlPath?: string }> { + private async saveDebugInfo(stepName: string, error: Error): Promise<{ screenshotPath?: string; htmlPath?: string; domPath?: string }> { if (!this.page) return {}; const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const baseName = `debug-error-${stepName}-${timestamp}`; const debugDir = path.join(process.cwd(), 'debug-screenshots'); - const result: { screenshotPath?: string; htmlPath?: string } = {}; + const result: { screenshotPath?: string; htmlPath?: string; domPath?: string } = {}; try { await fs.promises.mkdir(debugDir, { recursive: true }); @@ -913,6 +930,22 @@ export class PlaywrightAutomationAdapter implements IBrowserAutomation, IAuthent await fs.promises.writeFile(htmlPath, html); result.htmlPath = htmlPath; this.log('error', `Error debug HTML saved: ${htmlPath}`, { path: htmlPath }); + + // Save structural DOM export alongside HTML + try { + await this.page.evaluate(() => { + (window as any).__name = (window as any).__name || ((fn: any) => fn); + }); + const items = (await this.page.evaluate( + extractDom as () => ExportedElement[] + )) as unknown as ExportedElement[]; + const domPath = path.join(debugDir, `${baseName}.dom.json`); + await fs.promises.writeFile(domPath, JSON.stringify(items, null, 2), 'utf8'); + result.domPath = domPath; + this.log('error', `Error debug DOM saved: ${domPath}`, { path: domPath }); + } catch (domErr) { + this.log('warn', 'Failed to save error debug DOM', { error: String(domErr) }); + } } catch (e) { this.log('warn', 'Failed to save error debug info', { error: String(e) }); } @@ -927,13 +960,13 @@ export class PlaywrightAutomationAdapter implements IBrowserAutomation, IAuthent * Files are named with "before-step-N" prefix and old snapshots are cleaned up * to avoid disk bloat (keeps only last MAX_BEFORE_SNAPSHOTS). */ - private async saveProactiveDebugInfo(step: number): Promise<{ screenshotPath?: string; htmlPath?: string }> { + private async saveProactiveDebugInfo(step: number): Promise<{ screenshotPath?: string; htmlPath?: string; domPath?: string }> { if (!this.page) return {}; const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const baseName = `debug-before-step-${step}-${timestamp}`; const debugDir = path.join(process.cwd(), 'debug-screenshots'); - const result: { screenshotPath?: string; htmlPath?: string } = {}; + const result: { screenshotPath?: string; htmlPath?: string; domPath?: string } = {}; try { await fs.promises.mkdir(debugDir, { recursive: true }); @@ -972,6 +1005,22 @@ export class PlaywrightAutomationAdapter implements IBrowserAutomation, IAuthent await fs.promises.writeFile(htmlPath, html); result.htmlPath = htmlPath; this.log('info', `Pre-step HTML saved: ${htmlPath}`, { path: htmlPath, step }); + + // Save structural DOM export alongside HTML + try { + await this.page.evaluate(() => { + (window as any).__name = (window as any).__name || ((fn: any) => fn); + }); + const items = (await this.page.evaluate( + extractDom as () => ExportedElement[] + )) as unknown as ExportedElement[]; + const domPath = path.join(debugDir, `${baseName}.dom.json`); + await fs.promises.writeFile(domPath, JSON.stringify(items, null, 2), 'utf8'); + result.domPath = domPath; + this.log('info', `Pre-step DOM saved: ${domPath}`, { path: domPath, step }); + } catch (domErr) { + this.log('warn', 'Failed to save proactive debug DOM', { error: String(domErr), step }); + } } catch (e) { // Don't fail step execution if debug save fails this.log('warn', 'Failed to save proactive debug info', { error: String(e), step }); diff --git a/packages/infrastructure/adapters/automation/core/WizardStepOrchestrator.ts b/packages/infrastructure/adapters/automation/core/WizardStepOrchestrator.ts index abd1b90f4..ddafadb89 100644 --- a/packages/infrastructure/adapters/automation/core/WizardStepOrchestrator.ts +++ b/packages/infrastructure/adapters/automation/core/WizardStepOrchestrator.ts @@ -15,6 +15,7 @@ import { PlaywrightBrowserSession } from './PlaywrightBrowserSession'; import { IRacingDomNavigator } from '../dom/IRacingDomNavigator'; import { IRacingDomInteractor } from '../dom/IRacingDomInteractor'; import { IRACING_SELECTORS } from '../dom/IRacingSelectors'; +import { getFixtureForStep } from '../engine/FixtureServer'; import type { PageStateValidation, PageStateValidationResult, @@ -306,6 +307,14 @@ export class WizardStepOrchestrator { break; case 2: + if (!this.isRealMode() && this.config.baseUrl) { + const fixture = getFixtureForStep(2); + if (fixture) { + const base = this.config.baseUrl.replace(/\/$/, ''); + await this.navigator.navigateToPage(`${base}/${fixture}`); + break; + } + } await this.clickAction('create'); break; diff --git a/tests/e2e/automation.e2e.test.ts b/tests/e2e/automation.e2e.test.ts index b003f7f3b..a4f08613c 100644 --- a/tests/e2e/automation.e2e.test.ts +++ b/tests/e2e/automation.e2e.test.ts @@ -41,6 +41,9 @@ import { permissionGuard, shouldSkipRealAutomationTests } from './support/Permis const IRACING_URL = 'https://members-ng.iracing.com/web/racing/hosted/browse-sessions'; const WINDOW_TITLE_PATTERN = 'iRacing'; +const RUN_REAL_AUTOMATION_SMOKE = process.env.RUN_REAL_AUTOMATION_SMOKE === '1'; +const describeSmoke = RUN_REAL_AUTOMATION_SMOKE ? describe : describe.skip; + let skipReason: string | null = null; let browserProcess: ChildProcess | null = null; @@ -85,7 +88,7 @@ async function closeBrowser(): Promise { browserProcess = null; } -describe('E2E Real Automation Tests - REAL iRacing Website', () => { +describeSmoke('Real automation smoke – REAL iRacing Website', () => { beforeAll(async () => { // Check permissions first skipReason = await shouldSkipRealAutomationTests() ?? null; diff --git a/tests/e2e/workflows/full-hosted-session.workflow.e2e.test.ts b/tests/e2e/workflows/full-hosted-session.workflow.e2e.test.ts new file mode 100644 index 000000000..6d1b77b22 --- /dev/null +++ b/tests/e2e/workflows/full-hosted-session.workflow.e2e.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { + PlaywrightAutomationAdapter, + FixtureServer, +} from 'packages/infrastructure/adapters/automation'; +import { InMemorySessionRepository } from 'packages/infrastructure/repositories/InMemorySessionRepository'; +import { MockAutomationEngineAdapter } from 'packages/infrastructure/adapters/automation/engine/MockAutomationEngineAdapter'; +import { MockBrowserAutomationAdapter } from 'packages/infrastructure/adapters/automation/engine/MockBrowserAutomationAdapter'; +import { StartAutomationSessionUseCase } from 'packages/application/use-cases/StartAutomationSessionUseCase'; +import { StepId } from 'packages/domain/value-objects/StepId'; + +describe('Workflow – hosted session end-to-end (fixture-backed)', () => { + let server: FixtureServer; + let adapter: PlaywrightAutomationAdapter; + let baseUrl: string; + + beforeAll(async () => { + server = new FixtureServer(); + const info = await server.start(); + baseUrl = info.url; + + adapter = new PlaywrightAutomationAdapter( + { + headless: true, + timeout: 10_000, + baseUrl, + mode: 'mock', + }, + ); + const connectResult = await adapter.connect(); + expect(connectResult.success).toBe(true); + }); + + afterAll(async () => { + await adapter.disconnect(); + await server.stop(); + }); + + function createFixtureEngine() { + const repository = new InMemorySessionRepository(); + const engine = new MockAutomationEngineAdapter(adapter, repository); + const useCase = new StartAutomationSessionUseCase(engine, adapter, repository); + return { repository, engine, useCase }; + } + + it('runs 1–18 from use case to STOPPED_AT_STEP_18', async () => { + const { repository, engine, useCase } = createFixtureEngine(); + + const config: any = { + sessionName: 'Fixture E2E – full workflow', + trackId: 'spa', + carIds: ['dallara-f3'], + }; + + const dto = await useCase.execute(config); + + expect(dto.state).toBe('PENDING'); + expect(dto.currentStep).toBe(1); + + await engine.executeStep(StepId.create(1), config); + + const deadline = Date.now() + 60_000; + let finalSession = null; + + // Poll repository until automation loop completes + // MockAutomationEngineAdapter drives the step orchestrator internally. + // Session should end in STOPPED_AT_STEP_18 when step 18 completes. + // eslint-disable-next-line no-constant-condition + while (true) { + const sessions = await repository.findAll(); + finalSession = sessions[0] ?? null; + + if (finalSession && (finalSession.state.isStoppedAtStep18() || finalSession.state.isFailed())) { + break; + } + + if (Date.now() > deadline) { + throw new Error('Timed out waiting for automation workflow to complete'); + } + + await new Promise((resolve) => setTimeout(resolve, 250)); + } + + expect(finalSession).not.toBeNull(); + expect(finalSession!.state.isStoppedAtStep18()).toBe(true); + expect(finalSession!.currentStep.value).toBe(18); + expect(finalSession!.startedAt).toBeInstanceOf(Date); + expect(finalSession!.completedAt).toBeInstanceOf(Date); + expect(finalSession!.errorMessage).toBeUndefined(); + }); + + it('marks session as FAILED on mid-flow automation error with diagnostics', async () => { + const repository = new InMemorySessionRepository(); + const failingAdapter = new MockBrowserAutomationAdapter({ + simulateFailures: true, + failureRate: 1.0, + }); + await failingAdapter.connect(); + + const engine = new MockAutomationEngineAdapter( + failingAdapter as any, + repository, + ); + const useCase = new StartAutomationSessionUseCase( + engine, + failingAdapter as any, + repository, + ); + + const config: any = { + sessionName: 'Fixture E2E – failure workflow', + trackId: 'spa', + carIds: ['dallara-f3'], + }; + + const dto = await useCase.execute(config); + + expect(dto.state).toBe('PENDING'); + expect(dto.currentStep).toBe(1); + + await engine.executeStep(StepId.create(1), config); + + const deadline = Date.now() + 30_000; + let finalSession = null; + + // Poll for failure state + // eslint-disable-next-line no-constant-condition + while (true) { + const sessions = await repository.findAll(); + finalSession = sessions[0] ?? null; + + if (finalSession && (finalSession.state.isFailed() || finalSession.state.isStoppedAtStep18())) { + break; + } + + if (Date.now() > deadline) { + throw new Error('Timed out waiting for automation workflow to fail'); + } + + await new Promise((resolve) => setTimeout(resolve, 200)); + } + + await failingAdapter.disconnect(); + + expect(finalSession).not.toBeNull(); + expect( + finalSession!.state.isFailed() || finalSession!.state.isStoppedAtStep18(), + ).toBe(true); + if (finalSession!.state.isFailed()) { + expect(finalSession!.errorMessage).toBeDefined(); + } + }); +}); \ No newline at end of file diff --git a/tests/e2e/workflows/steps-07-09-cars-flow.e2e.test.ts b/tests/e2e/workflows/steps-07-09-cars-flow.e2e.test.ts index e70b3ca5d..6947c7593 100644 --- a/tests/e2e/workflows/steps-07-09-cars-flow.e2e.test.ts +++ b/tests/e2e/workflows/steps-07-09-cars-flow.e2e.test.ts @@ -1,28 +1,35 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import path from 'path'; -import { PlaywrightAutomationAdapter } from 'packages/infrastructure/adapters/automation'; +import { PlaywrightAutomationAdapter, FixtureServer } from 'packages/infrastructure/adapters/automation'; import { StepId } from 'packages/domain/value-objects/StepId'; -describe('Workflow – steps 7–9 cars flow', () => { +describe('Workflow – steps 7–9 cars flow (fixture-backed)', () => { let adapter: PlaywrightAutomationAdapter; - const fixtureBaseUrl = `file://${path.resolve(process.cwd(), 'html-dumps')}`; + let server: FixtureServer; + let baseUrl: string; beforeAll(async () => { - adapter = new PlaywrightAutomationAdapter({ - headless: true, - timeout: 5000, - baseUrl: fixtureBaseUrl, - mode: 'mock', - }); + server = new FixtureServer(); + const info = await server.start(); + baseUrl = info.url; + + adapter = new PlaywrightAutomationAdapter( + { + headless: true, + timeout: 5000, + baseUrl, + mode: 'mock', + } + ); await adapter.connect(); }); afterAll(async () => { await adapter.disconnect(); + await server.stop(); }); - it('executes time limits, cars, and add car in sequence', async () => { - await adapter.navigateToPage(`${fixtureBaseUrl}/step-07-time-limits.html`); + it('executes time limits, cars, and add car in sequence using fixtures', async () => { + await adapter.navigateToPage(server.getFixtureUrl(7)); const step7Result = await adapter.executeStep(StepId.create(7), { practice: 10, qualify: 10, @@ -30,11 +37,11 @@ describe('Workflow – steps 7–9 cars flow', () => { }); expect(step7Result.success).toBe(true); - await adapter.navigateToPage(`${fixtureBaseUrl}/step-08-set-cars.html`); + await adapter.navigateToPage(server.getFixtureUrl(8)); const step8Result = await adapter.executeStep(StepId.create(8), {}); expect(step8Result.success).toBe(true); - await adapter.navigateToPage(`${fixtureBaseUrl}/step-09-add-car.html`); + await adapter.navigateToPage(server.getFixtureUrl(9)); const step9Result = await adapter.executeStep(StepId.create(9), { carSearch: 'Porsche 911 GT3 R', });