diff --git a/.changeset/calm-tests-settle.md b/.changeset/calm-tests-settle.md new file mode 100644 index 0000000000..5683922aab --- /dev/null +++ b/.changeset/calm-tests-settle.md @@ -0,0 +1,8 @@ +--- +'@fluentui-react-native/components': patch +'@fluentui-react-native/desktop-driver': patch +'@fluentui-react-native/storybook-desktop': patch +'@fluentui-react-native/storybook-desktop-runtime': patch +--- + +Make JavaScript test fixtures more reliable under CI load and document deterministic test authoring. No runtime API or behavior changes. diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index dc92de9ae6..25947dd871 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -36,7 +36,20 @@ jobs: run: yarn test:agent-permissions - name: Build CI - run: yarn lage buildci + env: + FURN_JEST_MAX_WORKERS: '1' + FURN_TEST_TIMEOUT_SCALE: '2' + FURN_JEST_TIMINGS_DIR: ${{ runner.temp }}/js-pr-jest + run: yarn lage buildci --concurrency 2 + + - name: Upload JS test timings + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: JS_PR_Jest_${{ github.run_id }}_${{ github.run_attempt }} + path: ${{ runner.temp }}/js-pr-jest + if-no-files-found: ignore + retention-days: 14 bundle-size: name: Bundle Size PR diff --git a/packages/agentic/components/jest.config.cjs b/packages/agentic/components/jest.config.cjs index dcc1e5039b..4de89059e0 100644 --- a/packages/agentic/components/jest.config.cjs +++ b/packages/agentic/components/jest.config.cjs @@ -1,6 +1,7 @@ const config = require('@fluentui-react-native/scripts/jest-config'); +const { testBudget } = require('@fluentui-react-native/scripts/test-budget'); module.exports = { ...config, - testTimeout: 10000, + testTimeout: testBudget(10_000), }; diff --git a/packages/agentic/components/references/tests-and-stories.md b/packages/agentic/components/references/tests-and-stories.md index 5691b7398f..4095879f08 100644 --- a/packages/agentic/components/references/tests-and-stories.md +++ b/packages/agentic/components/references/tests-and-stories.md @@ -50,6 +50,14 @@ Button coverage demonstrates: Use table-driven tests for complete finite axes. When fixing one state regression, add a focused assertion even if a snapshot also changes. +Await `render`, `rerender`, and `fireEvent` and then assert the observable +state. Drive native adapter readiness through the explicit fixture event, +not a sleep. Keep representative themed render tests for accessibility and +interaction wiring; test pure selection rules separately when it avoids +repeating a costly popup render without reducing contract coverage. +For slow repository integration fixtures, use the shared `testBudget(ms)` helper +rather than increasing individual wall-clock limits ad hoc. + ## Type tests and primitives Add committed `*.types.test.ts` coverage for compile-time slot acceptance, mutually exclusive props, and public type diff --git a/packages/agentic/components/src/components/dropdown/dropdown.test.tsx b/packages/agentic/components/src/components/dropdown/dropdown.test.tsx index bb4aeb777e..5b8cd4dec9 100644 --- a/packages/agentic/components/src/components/dropdown/dropdown.test.tsx +++ b/packages/agentic/components/src/components/dropdown/dropdown.test.tsx @@ -3,6 +3,7 @@ import * as React from 'react'; import { StyleSheet } from 'react-native'; import type { View } from 'react-native'; import { act, fireEvent, screen } from '@testing-library/react-native'; +import { testBudget } from '@fluentui-react-native/scripts/test-budget'; import { render } from '../../common/renderWithTheme'; import { ListboxItem } from '../listbox-item/listbox-item'; @@ -164,7 +165,7 @@ describe('Dropdown', () => { expect(onAccessibilityAction).toHaveBeenCalledTimes(1); expect(onOpenChange).not.toHaveBeenCalled(); }, - 30_000, + testBudget(30_000), ); it('renders an inaccessible root around a collapsed combobox trigger showing the placeholder', async () => { diff --git a/packages/agentic/desktop-driver/references/contributing.md b/packages/agentic/desktop-driver/references/contributing.md index 8fe7d4cbd0..db02a4c27a 100644 --- a/packages/agentic/desktop-driver/references/contributing.md +++ b/packages/agentic/desktop-driver/references/contributing.md @@ -149,6 +149,13 @@ instructions. ## Validation order +For fake-host recovery or session races, gate the actual dispatch, storage write, +or settlement with deferred promises. Release the gate in `finally` and drain +the operation before closing the harness. Test deadline math with fake timers +as in `src/protocol/timeouts.test.ts`, and scope intentional short real deadlines +so release and application cleanup keep their normal budgets. Include nested +cleanup causes in failures. + During iteration, run the smallest affected test. Before completing a package change: diff --git a/packages/agentic/desktop-driver/src/server/recovery.test.ts b/packages/agentic/desktop-driver/src/server/recovery.test.ts index 8ed4fb1e2a..e2cd106067 100644 --- a/packages/agentic/desktop-driver/src/server/recovery.test.ts +++ b/packages/agentic/desktop-driver/src/server/recovery.test.ts @@ -21,8 +21,23 @@ class MemoryStorage implements PrivateStorage { /** Loses every record right after it is published, as an operator cleanup would. */ dropAfterPublish = false; failRead = false; - /** Holds every storage write until it resolves. */ - gate: Promise | undefined; + private writeGate?: () => Promise; + + blockWrites() { + const entered = Promise.withResolvers(); + const released = Promise.withResolvers(); + this.writeGate = () => { + entered.resolve(); + return released.promise; + }; + return { + entered: entered.promise, + release: () => { + this.writeGate = undefined; + released.resolve(); + }, + }; + } async read(name: string): Promise { if (this.failRead) { @@ -32,7 +47,7 @@ class MemoryStorage implements PrivateStorage { } async publish(name: string, contents: string): Promise { - await this.gate; + await this.writeGate?.(); if (this.failPublish) { throw new Error('Storage is unavailable.'); } @@ -49,7 +64,7 @@ class MemoryStorage implements PrivateStorage { } async remove(name: string, expectedContents: string): Promise { - await this.gate; + await this.writeGate?.(); if (this.failRemove) { throw new Error('Storage is unavailable.'); } @@ -86,6 +101,14 @@ async function startService( const harness = await createDesktopDriverTestHarness(hostOptions, undefined, { recovery: { storage, ...(probe ? { probeApplication: probe } : {}) }, }); + const close = harness.close; + harness.close = async () => { + try { + await close(); + } catch (error) { + throw new Error(`Settlement fixture cleanup failed:\n${cleanupDetails(error)}`, { cause: error }); + } + }; services.push(harness); const client = createDesktopDriverClient({ url: harness.server.url }); return { @@ -105,16 +128,36 @@ async function startService( const services: DesktopDriverTestHarness[] = []; afterEach(async () => { - await Promise.all(services.splice(0).map((harness) => harness.close().catch(() => undefined))); + const results = await Promise.allSettled(services.splice(0).map((harness) => harness.close())); + const failed = results.flatMap((result) => (result.status === 'rejected' ? [cleanupDetails(result.reason)] : [])); + if (failed.length) throw new Error(failed.join('\n')); }); +function cleanupDetails(error: unknown): string { + if (error instanceof AggregateError) return [error.message, ...error.errors.map(cleanupDetails)].join('\n'); + if (error instanceof Error) return error.message; + return String(error); +} + +async function withShortCommandDeadline(service: Service, sessionId: string, command: () => Promise): Promise { + const timeouts = service.harness.server.sessions.get(sessionId).desktopTimeouts; + const normalBudget = timeouts.nativeCommand; + timeouts.nativeCommand = 10; + try { + return await command(); + } finally { + timeouts.nativeCommand = normalBudget; + } +} + /** Leaves the service's target quarantined by a click that applied but never answered. */ async function quarantineByLostClick(service: Service): Promise { const session = await service.open(); const element = await session.findElement('accessibility id', 'button-primary'); service.harness.host.injectFault('click', { mode: 'lost-response' }); - service.harness.server.sessions.get(session.id).desktopTimeouts.nativeCommand = 10; - await expect(element.click()).rejects.toMatchObject({ data: { mutation: { quarantined: true } } }); + await expect(withShortCommandDeadline(service, session.id, () => element.click())).rejects.toMatchObject({ + data: { mutation: { quarantined: true } }, + }); } const quarantinedOnOpen = { code: 'session not created', data: { reason: 'target-quarantined' } }; @@ -152,13 +195,29 @@ describe('Desktop Driver settlement journal', () => { const service = await startService(storage); const session = await service.open(); const element = await session.findElement('accessibility id', 'button-primary'); - service.harness.host.injectFault('click', { mode: 'hang' }); - const clicking = element.click().catch(() => undefined); - await waitUntil(() => service.harness.host.actions.some(({ type }) => type === 'fault')); - // What storage holds at this moment is what a service that stopped now would leave behind. - const leftBehind = storage.copy(); - service.harness.server.sessions.get(session.id).commandAbortController.abort(); - await clicking; + const { host, server } = service.harness; + const dispatch = host.click.bind(host); + const started = Promise.withResolvers(); + host.injectFault('click', { mode: 'hang' }); + jest.spyOn(host, 'click').mockImplementationOnce((...args) => { + started.resolve(); + return dispatch(...args); + }); + const clicking = expect(element.click()).rejects.toMatchObject({ data: { mutation: { quarantined: true } } }); + let leftBehind: MemoryStorage; + try { + await Promise.race([ + started.promise, + clicking.then(() => { + throw new Error('Click settled before dispatch.'); + }), + ]); + // Copy the records at dispatch, before the simulated service stop can settle the mutation. + leftBehind = storage.copy(); + } finally { + server.sessions.get(session.id).commandAbortController.abort(); + await clicking; + } const restarted = await startService(leftBehind); await expect(restarted.open()).rejects.toMatchObject({ @@ -277,19 +336,34 @@ describe('Desktop Driver settlement journal', () => { const record = service.harness.server.sessions.getQuarantine(service.harness.target.id)!; const sessions = service.harness.server.sessions; const release = new AbortController(); - service.harness.host.injectFault('releaseActions', { mode: 'hang' }); + const { host } = service.harness; + const started = Promise.withResolvers(); + host.injectFault('releaseActions', { mode: 'hang' }); const releasing = sessions - .runMutation(sessions.get(record.sessionId!), 'Releasing input', () => service.harness.host.releaseActions(release.signal), { - recovery: true, - }) + .runMutation( + sessions.get(record.sessionId!), + 'Releasing input', + () => { + started.resolve(); + return host.releaseActions(release.signal); + }, + { recovery: true }, + ) .catch(() => undefined); - await waitUntil(() => service.harness.host.actions.filter(({ type }) => type === 'fault').length === 2); - - await expect( - service.harness.server.recoverTarget(service.harness.target.id, { evidence: 'operator', quarantineId: record.id }), - ).rejects.toThrow(/mutation in flight/); - release.abort(); - await releasing; + try { + await Promise.race([ + started.promise, + releasing.then(() => { + throw new Error('Release settled before dispatch.'); + }), + ]); + await expect( + service.harness.server.recoverTarget(service.harness.target.id, { evidence: 'operator', quarantineId: record.id }), + ).rejects.toThrow(/mutation in flight/); + } finally { + release.abort(); + await releasing; + } await expect( service.harness.server.recoverTarget(service.harness.target.id, { evidence: 'operator', quarantineId: record.id }), ).resolves.toBeDefined(); @@ -304,26 +378,39 @@ describe('Desktop Driver settlement journal', () => { const record = sessions.getQuarantine(targetId)!; const session = sessions.get(record.sessionId!); const release = () => sessions.runMutation(session, 'Releasing input', async () => undefined, { recovery: true }); - let open!: () => void; - - storage.gate = new Promise((resolve) => (open = resolve)); + const recordingGate = storage.blockWrites(); const recording = release(); - await expect(service.harness.server.recoverTarget(targetId, { evidence: 'operator', quarantineId: record.id })).rejects.toThrow( - /mutation in flight/, - ); - open(); - await recording; + try { + await Promise.race([ + recordingGate.entered, + recording.then(() => { + throw new Error('Mutation settled before recording.'); + }), + ]); + await expect(service.harness.server.recoverTarget(targetId, { evidence: 'operator', quarantineId: record.id })).rejects.toThrow( + /mutation in flight/, + ); + } finally { + recordingGate.release(); + await recording; + } - storage.gate = new Promise((resolve) => (open = resolve)); + const recoveryGate = storage.blockWrites(); const recovering = service.harness.server.recoverTarget(targetId, { evidence: 'operator', quarantineId: record.id }); - // Let recovery reach its storage write. - await new Promise((resolve) => setTimeout(resolve, 0)); - await expect(release()).rejects.toMatchObject({ data: { reason: 'target-quarantined' } }); - open(); - await expect(recovering).resolves.toBeDefined(); + try { + await Promise.race([ + recoveryGate.entered, + recovering.then(() => { + throw new Error('Recovery settled before writing.'); + }), + ]); + await expect(release()).rejects.toMatchObject({ data: { reason: 'target-quarantined' } }); + } finally { + recoveryGate.release(); + await expect(recovering).resolves.toBeDefined(); + } expect(storage.records.size).toBe(0); - storage.gate = undefined; await expect(release()).resolves.toBeUndefined(); expect(storage.records.size).toBe(0); }); @@ -418,9 +505,7 @@ describe('Desktop Driver settlement journal', () => { const session = await service.open(); storage.failPublish = true; service.harness.host.injectFault('releaseActions', { mode: 'lost-response' }); - service.harness.server.sessions.get(session.id).desktopTimeouts.nativeCommand = 10; - - await expect(session.delete()).rejects.toMatchObject({ code: 'timeout' }); + await expect(withShortCommandDeadline(service, session.id, () => session.delete())).rejects.toMatchObject({ code: 'timeout' }); expect(service.harness.server.sessions.getQuarantine(service.harness.target.id)).toMatchObject({ effect: 'unknown', @@ -436,22 +521,26 @@ describe('Desktop Driver settlement journal', () => { const element = await session.findElement('accessibility id', 'button-primary'); service.harness.host.injectFault('click', { mode: 'lost-response' }); const { sessions } = service.harness.server; - sessions.get(session.id).desktopTimeouts.nativeCommand = 10; - await expect(element.click()).rejects.toMatchObject({ data: { mutation: { quarantined: true } } }); + await expect(withShortCommandDeadline(service, session.id, () => element.click())).rejects.toMatchObject({ + data: { mutation: { quarantined: true } }, + }); const targetId = service.harness.target.id; const record = sessions.getQuarantine(targetId)!; - let open!: () => void; - storage.gate = new Promise((resolve) => (open = resolve)); + const gate = storage.blockWrites(); const recovering = service.harness.server.recoverTarget(targetId, { evidence: 'operator', quarantineId: record.id }); - // Let recovery reach its storage write. - await new Promise((resolve) => setTimeout(resolve, 0)); - - await expect(session.delete()).rejects.toMatchObject({ data: { reason: 'target-quarantined' } }); - - expect(service.harness.host.actions.some(({ type }) => type === 'close-application')).toBe(false); - open(); - await recovering; - storage.gate = undefined; + try { + await Promise.race([ + gate.entered, + recovering.then(() => { + throw new Error('Recovery settled before writing.'); + }), + ]); + await expect(session.delete()).rejects.toMatchObject({ data: { reason: 'target-quarantined' } }); + expect(service.harness.host.actions.some(({ type }) => type === 'close-application')).toBe(false); + } finally { + gate.release(); + await recovering; + } await sessions.deleteAll(); expect(service.harness.host.actions).toContainEqual(expect.objectContaining({ type: 'close-application' })); }); @@ -491,8 +580,18 @@ describe('Desktop Driver settlement journal', () => { const journal = new SettlementJournal(storage, { storageDeadlineMs: 20 }); const pending = { command: 'Clicking', sessionId: 'session', targetId: 'target' }; - await expect(journal.load('target')).rejects.toMatchObject({ code: 'timeout' }); - await expect(journal.beginMutation(pending)).rejects.toMatchObject({ code: 'timeout' }); + jest.useFakeTimers(); + try { + const loading = expect(journal.load('target')).rejects.toMatchObject({ code: 'timeout' }); + await jest.advanceTimersByTimeAsync(20); + await loading; + const beginning = expect(journal.beginMutation(pending)).rejects.toMatchObject({ code: 'timeout' }); + // Publication times out, then read-back has its own deadline. + await jest.runAllTimersAsync(); + await beginning; + } finally { + jest.useRealTimers(); + } expect(journal.hasMutation('target')).toBe(false); }); @@ -556,13 +655,3 @@ describe('Desktop Driver settlement journal', () => { expect(new SessionManager().describeQuarantine(record)).toEqual(record); }); }); - -async function waitUntil(predicate: () => boolean): Promise { - for (let attempt = 0; attempt < 100; attempt += 1) { - if (predicate()) { - return; - } - await new Promise((resolve) => setTimeout(resolve, 5)); - } - throw new Error('Timed out waiting for the test condition.'); -} diff --git a/packages/agentic/storybook-desktop-runtime/AGENTS.md b/packages/agentic/storybook-desktop-runtime/AGENTS.md index 0fd91b22f6..9467b67d8d 100644 --- a/packages/agentic/storybook-desktop-runtime/AGENTS.md +++ b/packages/agentic/storybook-desktop-runtime/AGENTS.md @@ -35,5 +35,8 @@ authenticated bridge, Metro, theme, and platform chrome. Run `yarn format`, `yarn lint`, `yarn build`, and `yarn test`. Changes to runtime exports, platform selection, or Metro integration require validation through the consuming Storybook app and its affected native platform. +For preview tests, await the render or input event that drives the generation +change. Use the shared repository `testBudget(ms)` helper for cold-render +budgets rather than adding sleeps. Add a patch changeset for this published package when its shipped guidance or behavior changes. diff --git a/packages/agentic/storybook-desktop-runtime/src/StorybookUI.test.tsx b/packages/agentic/storybook-desktop-runtime/src/StorybookUI.test.tsx index 006ca465d7..df3ad33d63 100644 --- a/packages/agentic/storybook-desktop-runtime/src/StorybookUI.test.tsx +++ b/packages/agentic/storybook-desktop-runtime/src/StorybookUI.test.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { Pressable, ScrollView, StyleSheet, Text, View } from 'react-native'; import { fireEvent, render } from '@testing-library/react-native'; import { theme } from '@storybook/react-native-theming'; +import { testBudget } from '@fluentui-react-native/scripts/test-budget'; import { DesktopStorybookConfigProvider, useDesktopStorybookConfig } from './DesktopStorybookConfig'; import { createDesktopStorybookPreview as createMacOSDesktopStorybookPreview } from './createDesktopStorybookPreview.macos'; @@ -79,19 +80,23 @@ function scene(requestId: string, selectionSequence: number) { } // Cold or contended CI workers can be slow to initialize the real ScrollView. -test('resets the whole Win32 preview for a new test generation without resetting chrome preferences', async () => { - const screen = await render(scene('first', 1)); - await fireEvent.press(screen.getByTestId('scroll-canvas')); - expect(screen.getByTestId('canvas-offset')).toHaveTextContent('99'); - await fireEvent.press(screen.getByTestId('test-win32-hide-sidebar')); - expect(screen.getByTestId('test-win32-show-sidebar')).toBeOnTheScreen(); - expect(screen.getByTestId('canvas-offset')).toHaveTextContent('99'); +test( + 'resets the whole Win32 preview for a new test generation without resetting chrome preferences', + async () => { + const screen = await render(scene('first', 1)); + await fireEvent.press(screen.getByTestId('scroll-canvas')); + expect(screen.getByTestId('canvas-offset')).toHaveTextContent('99'); + await fireEvent.press(screen.getByTestId('test-win32-hide-sidebar')); + expect(screen.getByTestId('test-win32-show-sidebar')).toBeOnTheScreen(); + expect(screen.getByTestId('canvas-offset')).toHaveTextContent('99'); - await screen.rerender(scene('second', 2)); + await screen.rerender(scene('second', 2)); - expect(screen.getByTestId('canvas-offset')).toHaveTextContent('0'); - expect(screen.getByTestId('test-win32-show-sidebar')).toBeOnTheScreen(); -}, 30_000); + expect(screen.getByTestId('canvas-offset')).toHaveTextContent('0'); + expect(screen.getByTestId('test-win32-show-sidebar')).toBeOnTheScreen(); + }, + testBudget(30_000), +); test.each<[string, React.ComponentType<{ children: React.ReactNode }>]>([ ['Win32', Win32StoryViewport], diff --git a/packages/agentic/storybook-desktop/references/contributing.md b/packages/agentic/storybook-desktop/references/contributing.md index 3548b31ef8..b01a620731 100644 --- a/packages/agentic/storybook-desktop/references/contributing.md +++ b/packages/agentic/storybook-desktop/references/contributing.md @@ -151,3 +151,11 @@ upstream reconnection. Run `yarn format`, `yarn lint`, `yarn build`, and `yarn test` from this package. Validate static manifests and the affected app's bundle/native smoke when changing runtime behavior. + +Keep inline WDIO contract modes focused on distinct behaviors (named cases, +settlement, smoke, protection, and input lanes). Retain real workers for +process-boundary assertions. Give contract children and their workers an +owned cancellation/drain path; leave time for cleanup before Jest's outer +deadline. Scale fixture-only startup and watchdog budgets with +the shared repository `testBudget(ms)` helper, +not the intentional worker timeout used to prove cancellation. diff --git a/packages/agentic/storybook-desktop/src/config/sourceConfig.test.ts b/packages/agentic/storybook-desktop/src/config/sourceConfig.test.ts index 7080583fb3..2cc8ca4fe0 100644 --- a/packages/agentic/storybook-desktop/src/config/sourceConfig.test.ts +++ b/packages/agentic/storybook-desktop/src/config/sourceConfig.test.ts @@ -1,5 +1,8 @@ import { spawnSync } from 'node:child_process'; import path from 'node:path'; +import { testBudget } from '@fluentui-react-native/scripts/test-budget'; + +jest.setTimeout(testBudget(40_000)); test('loads the source config through the CommonJS prebuild fallback without emitted JavaScript', () => { const projectRoot = path.resolve(__dirname, '../../../../../apps/storybook'); @@ -31,7 +34,7 @@ test('loads the source config through the CommonJS prebuild fallback without emi `, projectRoot, ], - { cwd: projectRoot, encoding: 'utf8', timeout: 30_000 }, + { cwd: projectRoot, encoding: 'utf8', timeout: testBudget(30_000) }, ); if (result.status !== 0) { @@ -68,7 +71,7 @@ test('the canonical Storybook config resolves target commands and identity witho })); `, ], - { cwd: projectRoot, encoding: 'utf8', timeout: 30_000 }, + { cwd: projectRoot, encoding: 'utf8', timeout: testBudget(30_000) }, ); if (result.status !== 0) throw new Error(result.stderr || result.error?.message || `Canonical config exited with ${result.status}.`); const value = JSON.parse(result.stdout); diff --git a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts index 1818941884..94214805e9 100644 --- a/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts +++ b/packages/agentic/storybook-desktop/src/driver/serverIntegration.test.ts @@ -10,10 +10,11 @@ import { createDesktopDriverClient } from '@fluentui-react-native/desktop-driver import { NodeDesktopCommandRunner } from '@fluentui-react-native/desktop-test/commands'; import { deadline } from '@fluentui-react-native/desktop-test/commands'; import { DesktopRunReporter, startReportTransport } from '@fluentui-react-native/desktop-test/reporting'; +import { testBudget } from '@fluentui-react-native/scripts/test-budget'; import type { StoredDriverManifest } from './generatedManifest.js'; -jest.setTimeout(30_000); +jest.setTimeout(testBudget(30_000)); describe('desktop Storybook server integration', () => { test.each(['resume', 'stop'] as const)( @@ -53,7 +54,7 @@ describe('desktop Storybook server integration', () => { }); try { // Allow for cold imports in the child process on a busy runner. - await waitForMarker(marker, 'disposal registration', 60_000); + await waitForMarker(marker, 'disposal registration', testBudget(60_000)); await expect(child.waitForReadiness!({ tier: 'service-ready', service: 'storybook-channel-listener' }, 1000)).rejects.toThrow( 'correlated readiness', ); @@ -66,8 +67,11 @@ describe('desktop Storybook server integration', () => { await expect(fetch(loopbackUrl(storybookPort, '/index.json'))).rejects.toThrow(); } else { fs.writeFileSync(release, 'continue'); - const listener = await child.waitForReadiness!({ tier: 'service-ready', service: 'storybook-channel-listener' }, 20_000); - const channel = await child.waitForReadiness!({ tier: 'service-ready', service: 'storybook-channel' }, 20_000); + const listener = await child.waitForReadiness!( + { tier: 'service-ready', service: 'storybook-channel-listener' }, + testBudget(20_000), + ); + const channel = await child.waitForReadiness!({ tier: 'service-ready', service: 'storybook-channel' }, testBudget(20_000)); expect(channel.processId).toBe(listener.processId); } } finally { @@ -82,7 +86,7 @@ describe('desktop Storybook server integration', () => { } } }, - 120_000, + testBudget(120_000), ); test.each(['helper-path-and-hash', 'unknown-field', 'helper-bytes'])( diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs index e50bdb5dbf..074c6891ce 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.contract.cjs @@ -1,5 +1,5 @@ const assert = require('node:assert/strict'); -const { spawn } = require('node:child_process'); +const { randomUUID } = require('node:crypto'); const fs = require('node:fs'); const http = require('node:http'); const os = require('node:os'); @@ -7,10 +7,27 @@ const path = require('node:path'); const { pathToFileURL } = require('node:url'); const { inspect } = require('node:util'); const { createFixtureStoredManifest } = require('../driver/fixtures/storedManifest.cjs'); +const { testBudget } = require('@fluentui-react-native/scripts/test-budget'); + +const controller = new AbortController(); +process.once('disconnect', () => controller.abort(new Error('WDIO contract owner disconnected.'))); async function main() { - const mode = process.argv[2] ?? 'all'; - assert.ok(['all', 'protected', 'legacy', 'lanes'].includes(mode), 'Unknown inline WDIO contract mode.'); + const mode = process.argv[2]; + assert.ok(['protected', 'named', 'settlement', 'smoke', 'lanes'].includes(mode), 'Unknown inline WDIO contract mode.'); + const timingDirectory = process.env.FURN_JEST_TIMINGS_DIR; + const timingFile = timingDirectory && path.join(timingDirectory, `wdio-${randomUUID()}.ndjson`); + if (timingDirectory) fs.mkdirSync(timingDirectory, { recursive: true }); + const recordPhases = (cases) => { + if (!timingFile) return; + for (const { status, diagnostics } of cases) { + const { navigationMs, workerStartupMs, workerBodyMs, workerMs, cleanupMs } = diagnostics?.phases ?? {}; + fs.appendFileSync( + timingFile, + `${JSON.stringify({ kind: 'wdio-case', mode, status, navigationMs, workerStartupMs, workerBodyMs, workerMs, cleanupMs })}\n`, + ); + } + }; const packageRoot = path.resolve(__dirname, '..', '..'); const projectRoot = path.resolve(packageRoot, '..', '..', '..', 'apps', 'storybook'); const root = fs.mkdtempSync(path.join(os.tmpdir(), 'furn-inline-wdio-')); @@ -37,6 +54,27 @@ async function main() { import('@fluentui-react-native/desktop-driver/server'), import('@fluentui-react-native/desktop-test/commands'), ]); + const runStories = async (options, runner) => { + let cases = []; + try { + cases = await runWdioStoryTests( + { + timeoutMs: testBudget(30_000), + ...options, + signal: options.signal ? AbortSignal.any([options.signal, controller.signal]) : controller.signal, + }, + runner, + ); + return cases; + } catch (error) { + if (error instanceof Error && error instanceof WdioStoryTestRunError && 'results' in error && Array.isArray(error.results)) { + cases = error.results; + } + throw error; + } finally { + recordPhases(cases); + } + }; fs.writeFileSync( path.join(root, 'fixture.stories.ts'), ` @@ -171,7 +209,7 @@ async function main() { resolvePackage: (name) => ({ root: name === 'inline-fixture' ? root : packageRoot }), }; const run = (story, overrides = {}) => - runWdioStoryTests({ + runStories({ config, manifest, platform: 'windows', @@ -183,7 +221,7 @@ async function main() { ...overrides, }); try { - if (mode === 'all' || mode === 'protected') { + if (mode === 'protected') { const authority = new DesktopAuthority(); const protectedHost = new FakeDesktopHost({ storyRootTestId: 'story-root' }); /** @type {import('@fluentui-react-native/desktop-driver').DesktopTarget} */ @@ -238,7 +276,7 @@ async function main() { }, }; try { - const protectedResults = await runWdioStoryTests( + const protectedResults = await runStories( { config, manifest, @@ -285,12 +323,12 @@ async function main() { runner: { connection: protectedConnection, addSecret: (secret) => secrets.push(secret) }, }); try { - await assert.rejects(runWdioStoryTests(protectedOptions('fixture-button--protected-failure'), leakRunner), (error) => { + await assert.rejects(runStories(protectedOptions('fixture-button--protected-failure'), leakRunner), (error) => { for (const secret of secrets) assert.ok(!inspect(error, { depth: null }).includes(secret)); return error instanceof WdioStoryTestRunError; }); assertNoSecrets(leakReport.root, secrets); - const skipped = await runWdioStoryTests({ ...protectedOptions('fixture-button--protected-skip'), optional: true }, leakRunner); + const skipped = await runStories({ ...protectedOptions('fixture-button--protected-skip'), optional: true }, leakRunner); assert.equal(skipped[0].skipReason, '[REDACTED]'); await leakCommands.close(); await leakReport.finish(); @@ -314,7 +352,7 @@ async function main() { }, }; try { - const running = runWdioStoryTests( + const running = runStories( { config, manifest, @@ -328,7 +366,12 @@ async function main() { }, cancelRunner, ); - await started; + await Promise.race([ + started, + running.then(() => { + throw new Error('Contract completed without starting the worker.'); + }), + ]); controller.abort(cancellation); await assert.rejects(running, (error) => error === cancellation); await cancelCommands.close(); @@ -357,7 +400,7 @@ async function main() { }; try { await assert.rejects( - runWdioStoryTests( + runStories( { config, manifest, @@ -398,23 +441,24 @@ async function main() { await fencedCommands.close(); } } finally { - await protectedServer.close(); + try { + await commandRunner.close(); + } finally { + await protectedServer.close(); + } } - } - - if (mode === 'protected') { process.stdout.write('\nWDIO_PROTECTED_OK\n'); return; } - let legacyResult; - if (mode !== 'lanes') { + let result; + if (mode === 'named') { const platforms = {}; for (const platform of ['macos', 'windows', 'win32']) { const platformManifest = await createDesktopStoryManifest({ projectRoot, getStoryPackages: () => [storyPackage] }, platform); const platformHarness = await createDesktopDriverStoryHarness(platformManifest); try { - const results = await runWdioStoryTests({ + const results = await runStories({ config, manifest: platformManifest, platform, @@ -474,6 +518,13 @@ async function main() { /Executable test source changed/, ); assert.equal(selections.length, selectionsBeforeMismatch); + result = { + platforms, + namedFailureResults: namedFailureResults.map(({ testName, status }) => ({ testName, status })), + }; + } + + if (mode === 'settlement') { let namedTimeout; try { await run('named-timeout'); @@ -553,8 +604,21 @@ async function main() { if (story === 'fail' && !report.tests[0].error.includes('intentional inline assertion')) { throw new Error('Compact reporter lost the worker assertion diagnostic.'); } + if (story === 'timeout' || story === 'spin') { + assert.match(report.tests[0].error, /timeout|timed out|process deadline/i); + } } const repeated = await run('pass'); + result = { + passed: passed.map(({ storyId }) => storyId), + skipped: skipped.filter(({ status }) => status === 'skipped').map(({ storyId }) => storyId), + repeated: repeated.map(({ storyId }) => storyId), + namedTimeout, + failures, + }; + } + + if (mode === 'smoke') { await selectStory({ requestId: 'wrong-page', runId: 'wrong-page', storyId: 'fixture-button--fail' }); selections.length = 0; const grouped = await run('*', { tag: 'grouped' }); @@ -583,7 +647,7 @@ async function main() { ); await selectStory({ requestId: 'smoke-wrong-page', runId: 'smoke-wrong-page', storyId: 'fixture-button--fail' }); selections.length = 0; - const control = await runSmokeControl(packageRoot, root, smokeManifestPath); + const control = await runSmokeControl(NodeDesktopCommandRunner, packageRoot, root, smokeManifestPath); if (control.code !== 0) throw new Error(control.stderr || control.stdout || `Smoke control exited with ${control.code}.`); const reportRoot = path.join(root, 'artifacts', 'desktop'); const runDirectory = fs.readdirSync(reportRoot)[0]; @@ -597,30 +661,23 @@ async function main() { const failureManifest = JSON.parse(fs.readFileSync(smokeManifestPath, 'utf8')); failureManifest.wdio = { story: 'fixture-button--fail', reporter: 'dot' }; fs.writeFileSync(smokeManifestPath, JSON.stringify(failureManifest)); - const failedControl = await runSmokeControl(packageRoot, root, smokeManifestPath); + const failedControl = await runSmokeControl(NodeDesktopCommandRunner, packageRoot, root, smokeManifestPath); const smokeFailureLogged = failedControl.code === 1 && failedControl.stderr.includes('[storybook] FAIL smoke') && failedControl.stderr.includes('fixture-button--fail') && failedControl.stderr.includes('intentional inline assertion'); if (!smokeFailureLogged) throw new Error(`Smoke control lost the callback failure: ${failedControl.stderr}`); - legacyResult = { - passed: passed.map(({ storyId }) => storyId), - skipped: skipped.filter(({ status }) => status === 'skipped').map(({ storyId }) => storyId), - repeated: repeated.map(({ storyId }) => storyId), + result = { grouped: grouped.map(({ storyId }) => storyId), groupedSelections, smoke, smokeFailureLogged, - platforms, - namedFailureResults: namedFailureResults.map(({ testName, status }) => ({ testName, status })), - namedTimeout, - failures, }; } let laneResult; - if (mode !== 'legacy') { + if (mode === 'lanes') { const buttonConfig = makeDesktopStorybookConfig({ projectRoot, storyPackages: ['@fluentui-react-native/components'] }); const buttonManifest = await createDesktopStoryManifest(buttonConfig, 'windows'); const buttonWindows = createFakeStoryWindows(buttonManifest); @@ -643,7 +700,7 @@ async function main() { const buttonHarness = await createDesktopDriverStoryHarness(buttonManifest, { windows: buttonWindows }); let button; try { - button = await runWdioStoryTests({ + button = await runStories({ config: { projectRoot: root, resolvePackage: (name) => buttonConfig.resolvePackage(name) }, manifest: buttonManifest, platform: 'windows', @@ -660,7 +717,7 @@ async function main() { }); let restrictedButton; try { - restrictedButton = await runWdioStoryTests({ + restrictedButton = await runStories({ config: { projectRoot: root, resolvePackage: (name) => buttonConfig.resolvePackage(name) }, manifest: buttonManifest, platform: 'windows', @@ -786,7 +843,7 @@ async function main() { process.stdout.write( '\nWDIO_RESULT ' + JSON.stringify({ - ...legacyResult, + ...result, ...laneResult, attachedOnly: harness.host.actions .filter(({ type }) => type === 'close-application') @@ -866,26 +923,38 @@ async function createUnconfirmingProxy(url) { }; } -/** @returns {Promise<{code: number | null, stdout: string, stderr: string}>} */ -function runSmokeControl(packageRoot, cwd, manifestPath) { - return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [path.join(packageRoot, 'config/storybook-control.cjs'), '--phase', 'tests'], { - cwd, - env: { ...process.env, STORYBOOK_DRIVER_MANIFEST: manifestPath, STORYBOOK_SMOKE_MODE: 'stories-and-tests' }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - const stdout = []; - const stderr = []; - child.stdout.on('data', (chunk) => stdout.push(chunk)); - child.stderr.on('data', (chunk) => stderr.push(chunk)); - child.once('error', reject); - child.once('close', (code) => - resolve({ code, stdout: Buffer.concat(stdout).toString('utf8'), stderr: Buffer.concat(stderr).toString('utf8') }), - ); +async function runSmokeControl(NodeDesktopCommandRunner, packageRoot, cwd, manifestPath) { + const stdout = []; + const stderr = []; + const runner = new NodeDesktopCommandRunner({ + logDirectory: path.join(cwd, 'control-commands'), + output: { write: (text) => stdout.push(text) }, + errorOutput: { write: (text) => stderr.push(text) }, + verbose: true, + shutdownTimeoutMs: testBudget(5_000), }); + try { + const child = runner.start( + { + command: process.execPath, + args: [path.join(packageRoot, 'config/storybook-control.cjs'), '--phase', 'tests'], + cwd, + env: { STORYBOOK_DRIVER_MANIFEST: manifestPath, STORYBOOK_SMOKE_MODE: 'stories-and-tests' }, + }, + { signal: controller.signal }, + ); + const code = await child.completed; + return { code, stdout: stdout.join(''), stderr: stderr.join('') }; + } finally { + await runner.close(); + } } -main().catch((error) => { - process.stderr.write(error.stack ?? error.message); - process.exitCode = 1; -}); +main() + .catch((error) => { + process.stderr.write(inspect(error, { depth: 6 }) + '\n'); + process.exitCode = 1; + }) + .finally(() => { + if (process.connected) process.disconnect(); + }); diff --git a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts index 19596387db..19d4c21e87 100644 --- a/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts +++ b/packages/agentic/storybook-desktop/src/testing/wdioRunner.test.ts @@ -1,167 +1,232 @@ -import { spawn } from 'node:child_process'; +import { fork } from 'node:child_process'; import path from 'node:path'; +import { testBudget } from '@fluentui-react-native/scripts/test-budget'; -function runContract(mode: 'protected' | 'legacy' | 'lanes'): Promise<{ output: string; diagnostics: string }> { +const contractDeadlineMs = testBudget(75_000); +// The watchdog cancels first, leaving time for the fixture to drain owned workers before escalation. +const drainDeadlineMs = testBudget(25_000); +const jestDeadlineMs = contractDeadlineMs + drainDeadlineMs + testBudget(5_000); +type ContractMode = 'protected' | 'named' | 'settlement' | 'smoke' | 'lanes'; + +function runContract(mode: ContractMode): Promise<{ output: string; diagnostics: string }> { return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [path.join(__dirname, 'wdioRunner.contract.cjs'), mode], { - stdio: ['ignore', 'pipe', 'pipe'], + const child = fork(path.join(__dirname, 'wdioRunner.contract.cjs'), [mode], { + execArgv: [], + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], }); const stdout: Buffer[] = []; const stderr: Buffer[] = []; - child.stdout.on('data', (chunk: Buffer) => stdout.push(chunk)); - child.stderr.on('data', (chunk: Buffer) => stderr.push(chunk)); - child.once('error', reject); - child.once('close', (code) => { - if (code !== 0) { - reject(new Error(Buffer.concat(stderr).toString('utf8') || `Inline test contract exited with ${code}.`)); + child.stdout!.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr!.on('data', (chunk: Buffer) => stderr.push(chunk)); + let expired = false; + let escalated = false; + let error: Error | undefined; + let escalation: ReturnType | undefined; + const watchdog = setTimeout(() => { + expired = true; + if (child.connected) child.disconnect(); + escalation = setTimeout(() => { + escalated = child.kill('SIGKILL'); + }, drainDeadlineMs); + }, contractDeadlineMs); + child.once('error', (reason: Error) => { + error = reason; + }); + child.once('close', (code, signal) => { + clearTimeout(watchdog); + clearTimeout(escalation); + const output = Buffer.concat(stdout).toString('utf8'); + const diagnostics = Buffer.concat(stderr).toString('utf8'); + if (expired || error || code !== 0) { + const detail = expired + ? `exceeded ${contractDeadlineMs} ms${escalated ? ' and required forced termination; worker cleanup is unconfirmed' : ''}` + : `exited with ${signal ?? code}`; + reject(new Error(`WDIO ${mode} contract ${detail}.\n${diagnostics.slice(-16_384)}`, { cause: error })); } else { - resolve({ output: Buffer.concat(stdout).toString('utf8'), diagnostics: Buffer.concat(stderr).toString('utf8') }); + resolve({ output, diagnostics }); } }); }); } -test('isolates protected worker grants, cancellation, and cleanup fencing', async () => { - const { output } = await runContract('protected'); - expect(output).toContain('\nWDIO_PROTECTED_OK\n'); -}, 120_000); +test( + 'isolates protected worker grants, cancellation, and cleanup fencing', + async () => { + const { output } = await runContract('protected'); + expect(output).toContain('\nWDIO_PROTECTED_OK\n'); + }, + jestDeadlineMs, +); -// This contract starts dozens of Node workers, including intentional timeouts. -test('executes isolated inline code and releases sessions after assertion failures, timeouts, and crashes', async () => { - const { output, diagnostics } = await runContract('legacy'); - if ( - !diagnostics.includes('intentional inline assertion') || - !diagnostics.includes('wdio "fixture-button--fail" during execution') || - !diagnostics.includes('wdio "fixture-button--navigation-failure" during navigation') || - !diagnostics.includes('Storybook context did not settle.') || - diagnostics.includes('Navigation refused before dispatch') || - output.includes('[storybook] OK wdio "fixture-button--exit-zero"') - ) { - throw new Error('Assertion diagnostics were lost.'); - } - const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); - expect(result).toMatchObject({ - passed: ['fixture-button--pass'], - skipped: ['fixture-button--skip'], - failures: ['fail', 'timeout', 'spin', 'crash', 'exit-zero', 'stale', 'navigation-failure'], - repeated: ['fixture-button--pass'], - grouped: ['fixture-button--pass', 'fixture-button--second'], - groupedSelections: ['fixture-button--pass', 'fixture-button--second'], - smoke: { - plans: ['fixture-button--pass', 'fixture-button--second'], - callbacks: ['fixture-button--pass', 'fixture-button--second'], - // The tests phase first waits for runtime readiness by selecting the manifest's first story. - selections: [ - 'fixture-button--crash', - 'fixture-button--pass', - 'fixture-button--pass', - 'fixture-button--second', - 'fixture-button--second', +test( + 'isolates named cases across endpoints and preserves selection, extraction and failure evidence', + async () => { + const { output } = await runContract('named'); + const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); + expect(result).toMatchObject({ + platforms: Object.fromEntries( + ['macos', 'windows', 'win32'].map((platform) => [ + platform, + [ + { testName: 'mutates state', status: 'passed' }, + { testName: 'starts fresh', status: 'passed' }, + ], + ]), + ), + namedFailureResults: [ + { testName: 'a b', status: 'failed' }, + { testName: 'a-b', status: 'failed' }, + { testName: 'recovers', status: 'passed' }, ], - status: 'passed', - }, - smokeFailureLogged: true, - platforms: Object.fromEntries( - ['macos', 'windows', 'win32'].map((platform) => [ - platform, - [ - { testName: 'mutates state', status: 'passed' }, - { testName: 'starts fresh', status: 'passed' }, - ], - ]), - ), - namedFailureResults: [ - { testName: 'a b', status: 'failed' }, - { testName: 'a-b', status: 'failed' }, - { testName: 'recovers', status: 'passed' }, - ], - namedTimeout: [ - { testName: 'times out', status: 'failed' }, - { testName: 'runs afterward', status: 'passed' }, - ], - generatedFiles: [], - attachedOnly: true, - }); -}, 120_000); + generatedFiles: [], + attachedOnly: true, + }); + }, + jestDeadlineMs, +); -test('enforces input lanes and records delivered input in inline workers', async () => { - const { output, diagnostics } = await runContract('lanes'); - expect(diagnostics).toContain('wdio "fixture-button--lane / accessible only" during input policy verification'); - const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); - expect(result).toMatchObject({ - button: [ - { testName: 'exposes enabled button semantics', status: 'passed' }, - { testName: 'supports native pointer activation', status: 'passed' }, - ], - // The migrated case declares physical-click, so a host without it reports the case unsupported before execution. - restrictedButton: [ - { testName: 'exposes enabled button semantics', status: 'passed' }, - { testName: 'supports native pointer activation', status: 'unsupported' }, - ], - lane: { - accessible: [ - { testName: 'accessible only', status: 'passed' }, - { testName: 'either lane', status: 'passed' }, - ], - physical: [ - { testName: 'either lane', status: 'passed' }, - { testName: 'physical only', status: 'passed' }, - ], - // No lane, and a host that masks physical click: the physical case is unsupported, not a failed session. - compatibility: [ - { testName: 'accessible only', status: 'passed' }, - { testName: 'either lane', status: 'passed' }, - { testName: 'physical only', status: 'unsupported' }, - ], - // The driver holds the policy, so an accessible lane runs on a driver with physical input enabled. - unmasked: [ - { testName: 'accessible only', status: 'passed' }, - { testName: 'either lane', status: 'passed' }, +test( + 'releases sessions after assertion failures, timeouts, and crashes', + async () => { + const { output, diagnostics } = await runContract('settlement'); + if ( + !diagnostics.includes('intentional inline assertion') || + !diagnostics.includes('wdio "fixture-button--fail" during execution') || + !diagnostics.includes('wdio "fixture-button--navigation-failure" during navigation') || + !diagnostics.includes('Storybook context did not settle.') || + diagnostics.includes('Navigation refused before dispatch') || + output.includes('[storybook] OK wdio "fixture-button--exit-zero"') + ) { + throw new Error('Assertion diagnostics were lost.'); + } + const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); + expect(result).toMatchObject({ + passed: ['fixture-button--pass'], + skipped: ['fixture-button--skip'], + failures: ['fail', 'timeout', 'spin', 'crash', 'exit-zero', 'stale', 'navigation-failure'], + repeated: ['fixture-button--pass'], + namedTimeout: [ + { testName: 'times out', status: 'failed' }, + { testName: 'runs afterward', status: 'passed' }, ], - // A driver that does not confirm the policy stops the lane at its first case, reported once. - unconfirmed: { - results: [{ testName: 'accessible only', status: 'failed' }], - status: 'failed', - failed: 1, - excluded: { platform: 0, scope: 0, input: 1 }, - reported: [expect.stringContaining('did not confirm the accessible input policy (it reported none)')], + generatedFiles: [], + attachedOnly: true, + }); + }, + jestDeadlineMs, +); + +test( + 'selects grouped callbacks and runs them through the smoke control', + async () => { + const { output } = await runContract('smoke'); + const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); + expect(result).toMatchObject({ + grouped: ['fixture-button--pass', 'fixture-button--second'], + groupedSelections: ['fixture-button--pass', 'fixture-button--second'], + smoke: { + plans: ['fixture-button--pass', 'fixture-button--second'], + callbacks: ['fixture-button--pass', 'fixture-button--second'], + // The tests phase first waits for runtime readiness by selecting the manifest's first story. + selections: [ + 'fixture-button--crash', + 'fixture-button--pass', + 'fixture-button--pass', + 'fixture-button--second', + 'fixture-button--second', + ], + status: 'passed', }, - }, - receipts: { - // Compatibility mode delivers what the host has, and still holds each declared case to its declaration. - compatibility: [ - { testName: 'declared click', status: 'passed', input: { receipts: { 'physical-click': 1 } } }, - { - testName: 'undeclared click', - status: 'failed', - input: { receipts: { 'physical-click': 1 }, undeclared: ['physical-click'] }, - error: 'The test delivered input it does not declare: physical-click (declare physical-click or click).', - }, - { testName: 'unused keyboard', status: 'passed', input: { receipts: {}, unused: ['keyboard'] } }, - { - testName: 'typing', - status: 'failed', - input: { receipts: { keyboard: 1 }, undeclared: ['keyboard'] }, - error: 'The test delivered input it does not declare: keyboard (declare keyboard).', - }, + smokeFailureLogged: true, + generatedFiles: [], + attachedOnly: true, + }); + }, + jestDeadlineMs, +); + +test( + 'enforces input lanes and records delivered input in inline workers', + async () => { + const { output, diagnostics } = await runContract('lanes'); + expect(diagnostics).toContain('wdio "fixture-button--lane / accessible only" during input policy verification'); + const result: Record = JSON.parse(output.split('\nWDIO_RESULT ')[1]); + expect(result).toMatchObject({ + button: [ + { testName: 'exposes enabled button semantics', status: 'passed' }, + { testName: 'supports native pointer activation', status: 'passed' }, ], - // The accessible lane clicks semantically and refuses typing before it is delivered, so nothing is attributed to it. - accessible: [ - { testName: 'declared click', status: 'passed', input: { receipts: { 'accessibility-click': 1 } } }, - { - testName: 'undeclared click', - status: 'failed', - input: { receipts: { 'accessibility-click': 1 }, undeclared: ['accessibility-click'] }, - }, - { - testName: 'typing', + // The migrated case declares physical-click, so a host without it reports the case unsupported before execution. + restrictedButton: [ + { testName: 'exposes enabled button semantics', status: 'passed' }, + { testName: 'supports native pointer activation', status: 'unsupported' }, + ], + lane: { + accessible: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + ], + physical: [ + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'passed' }, + ], + // No lane, and a host that masks physical click: the physical case is unsupported, not a failed session. + compatibility: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + { testName: 'physical only', status: 'unsupported' }, + ], + // The driver holds the policy, so an accessible lane runs on a driver with physical input enabled. + unmasked: [ + { testName: 'accessible only', status: 'passed' }, + { testName: 'either lane', status: 'passed' }, + ], + // A driver that does not confirm the policy stops the lane at its first case, reported once. + unconfirmed: { + results: [{ testName: 'accessible only', status: 'failed' }], status: 'failed', - input: { receipts: {} }, - error: expect.stringContaining('never delivers'), + failed: 1, + excluded: { platform: 0, scope: 0, input: 1 }, + reported: [expect.stringContaining('did not confirm the accessible input policy (it reported none)')], }, - ], - }, - generatedFiles: [], - attachedOnly: true, - }); -}, 120_000); + }, + receipts: { + // Compatibility mode delivers what the host has, and still holds each declared case to its declaration. + compatibility: [ + { testName: 'declared click', status: 'passed', input: { receipts: { 'physical-click': 1 } } }, + { + testName: 'undeclared click', + status: 'failed', + input: { receipts: { 'physical-click': 1 }, undeclared: ['physical-click'] }, + error: 'The test delivered input it does not declare: physical-click (declare physical-click or click).', + }, + { testName: 'unused keyboard', status: 'passed', input: { receipts: {}, unused: ['keyboard'] } }, + { + testName: 'typing', + status: 'failed', + input: { receipts: { keyboard: 1 }, undeclared: ['keyboard'] }, + error: 'The test delivered input it does not declare: keyboard (declare keyboard).', + }, + ], + // The accessible lane clicks semantically and refuses typing before it is delivered, so nothing is attributed to it. + accessible: [ + { testName: 'declared click', status: 'passed', input: { receipts: { 'accessibility-click': 1 } } }, + { + testName: 'undeclared click', + status: 'failed', + input: { receipts: { 'accessibility-click': 1 }, undeclared: ['accessibility-click'] }, + }, + { + testName: 'typing', + status: 'failed', + input: { receipts: {} }, + error: expect.stringContaining('never delivers'), + }, + ], + }, + generatedFiles: [], + attachedOnly: true, + }); + }, + jestDeadlineMs, +); diff --git a/references/agents/repository/development.md b/references/agents/repository/development.md index a68a2fd349..c5c883d0a1 100644 --- a/references/agents/repository/development.md +++ b/references/agents/repository/development.md @@ -117,6 +117,39 @@ settles (R-VALID-004): - Do not call `jest` directly (R-VALID-001): it would run platform-suffixed test files under the package's base platform instead of their own. +## Reliable JavaScript tests + +- Assert observable state, not elapsed time. Await the render or event that + changes state. In race tests, signal when the operation reaches the point of + interest and hold it with a deferred promise. Release the gate in `finally` + and await settlement, even if an assertion fails. +- Use fake timers for clock-driven unit behavior. Keep real timers around + real sockets and child processes; use their readiness/exit events rather than + sleeps. If testing an actual deadline, scope the short budget to that operation + and restore it before teardown. Preserve the underlying cleanup error. +- Keep subprocess contracts focused. Use an owned watchdog that requests + cancellation and waits for cleanup before the outer Jest deadline. Budget + startup, execution, and teardown separately; never use Jest's deadline + as the mechanism that stops a child process. +- Use `testBudget(ms)` from `@fluentui-react-native/scripts/test-budget` + for explicit test or fixture watchdog budgets that need CI headroom. + `FURN_TEST_TIMEOUT_SCALE` accepts integers 1–3, defaults to 1, and also + scales the shared Jest default. It must not scale production, protocol, or + deliberately short fault-injection deadlines. Jest's `--testTimeout` only + overrides the default, not an explicit test or hook budget. + +The GitHub JS PR job starts with two concurrent Lage tasks and one Jest +worker per package (`FURN_JEST_MAX_WORKERS=1`), with a timeout scale of 2. +This lets Lage provide cross-package parallelism without multiplying worker +pools. The Jest worker setting accepts integers 1–8; CI defaults to 1 +and local runs keep Jest's own default unless explicitly configured. +Compare worker settings on the same runner and revision before raising either cap. +CI uploads `JS_PR_Jest__` with per-invocation settings and +per-file timings/outcomes, plus navigation, worker, and cleanup durations +from the inline WDIO contracts. Completed records survive a failed test +when the upload step can run. The artifact does not contain test output or +error bodies. + ## Validation discipline - During iteration, run the focused tests for the code you changed (R-VALID-004), and run package lint or build early diff --git a/scripts/configs/jest/jest.config.cjs b/scripts/configs/jest/jest.config.cjs index 2d2ba62946..2135e182cf 100644 --- a/scripts/configs/jest/jest.config.cjs +++ b/scripts/configs/jest/jest.config.cjs @@ -2,6 +2,7 @@ const fs = require('fs'); const path = require('path'); const { PLATFORM_ENV_VAR } = require('../../src/const.mts'); +const { testBudget, testWorkerLimit } = require('./test-budget.cjs'); const projectManifestPath = path.resolve(process.cwd(), 'package.json'); const foundProject = fs.existsSync(projectManifestPath); const projectManifest = foundProject ? require(projectManifestPath) : null; @@ -62,4 +63,21 @@ function configureReactNativeJest(platform) { return config; } -module.exports = platform === 'react' ? require('./jest.react.config.cjs') : configureReactNativeJest(platform); +const config = platform === 'react' ? require('./jest.react.config.cjs') : configureReactNativeJest(platform); +const maxWorkers = testWorkerLimit(); +module.exports = { + ...config, + testTimeout: testBudget(typeof config.testTimeout === 'number' ? config.testTimeout : 5_000), + ...(maxWorkers === undefined ? {} : { maxWorkers }), + ...(process.env.FURN_JEST_TIMINGS_DIR + ? { + reporters: [ + 'default', + [ + require.resolve('./timing-reporter.cjs'), + { directory: process.env.FURN_JEST_TIMINGS_DIR, platform, workspace: projectManifest?.name ?? '' }, + ], + ], + } + : {}), +}; diff --git a/scripts/configs/jest/test-budget.cjs b/scripts/configs/jest/test-budget.cjs new file mode 100644 index 0000000000..66c243537d --- /dev/null +++ b/scripts/configs/jest/test-budget.cjs @@ -0,0 +1,19 @@ +function readInteger(env, name, min, max, fallback) { + const raw = env[name]; + if (raw === undefined) return fallback; + const value = Number(raw); + if (!/^[1-9][0-9]*$/.test(raw) || !Number.isInteger(value) || value < min || value > max) { + throw new Error(`${name} must be an integer from ${min} to ${max}.`); + } + return value; +} + +function testBudget(baseMs, env = process.env) { + return baseMs * readInteger(env, 'FURN_TEST_TIMEOUT_SCALE', 1, 3, 1); +} + +function testWorkerLimit(env = process.env) { + return readInteger(env, 'FURN_JEST_MAX_WORKERS', 1, 8, env.CI === 'true' || env.TF_BUILD ? 1 : undefined); +} + +module.exports = { testBudget, testWorkerLimit }; diff --git a/scripts/configs/jest/test-budget.d.cts b/scripts/configs/jest/test-budget.d.cts new file mode 100644 index 0000000000..841d809f6a --- /dev/null +++ b/scripts/configs/jest/test-budget.d.cts @@ -0,0 +1,2 @@ +export function testBudget(baseMs: number, env?: NodeJS.ProcessEnv): number; +export function testWorkerLimit(env?: NodeJS.ProcessEnv): number | undefined; diff --git a/scripts/configs/jest/test-policy.test.cjs b/scripts/configs/jest/test-policy.test.cjs new file mode 100644 index 0000000000..e7222b13d3 --- /dev/null +++ b/scripts/configs/jest/test-policy.test.cjs @@ -0,0 +1,17 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { testBudget, testWorkerLimit } = require('./test-budget.cjs'); + +test('preserves local defaults and starts CI with one Jest worker', () => { + assert.equal(testBudget(5_000, {}), 5_000); + assert.equal(testWorkerLimit({ CI: 'false' }), undefined); + assert.equal(testWorkerLimit({ CI: 'true' }), 1); + assert.equal(testWorkerLimit({ TF_BUILD: 'True' }), 1); + assert.equal(testWorkerLimit({ CI: 'true', FURN_JEST_MAX_WORKERS: '2' }), 2); +}); + +test('accepts explicit bounded test-only overrides', () => { + assert.equal(testBudget(15_000, { FURN_TEST_TIMEOUT_SCALE: '2' }), 30_000); + assert.throws(() => testBudget(15_000, { FURN_TEST_TIMEOUT_SCALE: '4' }), /FURN_TEST_TIMEOUT_SCALE/); + assert.throws(() => testWorkerLimit({ FURN_JEST_MAX_WORKERS: '50%' }), /FURN_JEST_MAX_WORKERS/); +}); diff --git a/scripts/configs/jest/timing-reporter.cjs b/scripts/configs/jest/timing-reporter.cjs new file mode 100644 index 0000000000..9fa5bd4f65 --- /dev/null +++ b/scripts/configs/jest/timing-reporter.cjs @@ -0,0 +1,52 @@ +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { randomUUID } = require('node:crypto'); + +module.exports = class TimingReporter { + /** + * @param {{maxWorkers: number, testTimeout: number}} config + * @param {{directory: string, platform: string, workspace: string}} options + */ + constructor(config, { directory, platform, workspace }) { + fs.mkdirSync(directory, { recursive: true }); + this.file = path.join(directory, `${randomUUID()}.ndjson`); + this.write({ + kind: 'run', + workspace, + platform, + availableCPUs: os.availableParallelism(), + maxWorkers: config.maxWorkers, + testTimeoutMs: config.testTimeout, + }); + } + + /** + * @param {{path: string, context: {config: {rootDir: string}}}} test + * @param {{perfStats: {runtime?: number}, numPassingTests: number, numFailingTests: number, numPendingTests: number, numTodoTests: number, testExecError?: unknown, snapshot?: {failure?: boolean}, skipped: boolean}} result + */ + onTestResult(test, result) { + this.write({ + kind: 'suite', + file: path.relative(test.context.config.rootDir, test.path).split(path.sep).join('/'), + durationMs: result.perfStats.runtime, + passed: result.numPassingTests, + failed: result.numFailingTests, + skipped: result.numPendingTests + result.numTodoTests, + status: result.testExecError || result.numFailingTests || result.snapshot?.failure ? 'failed' : result.skipped ? 'skipped' : 'passed', + }); + } + + /** + * @param {unknown} _contexts + * @param {{wasInterrupted: boolean, numFailedTestSuites: number}} result + */ + onRunComplete(_contexts, result) { + this.write({ kind: 'summary', interrupted: result.wasInterrupted, failedSuites: result.numFailedTestSuites }); + } + + /** @param {Record} record */ + write(record) { + fs.appendFileSync(this.file, `${JSON.stringify(record)}\n`); + } +}; diff --git a/scripts/package.json b/scripts/package.json index c33805c3b5..e3eeb8bbb6 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -19,6 +19,7 @@ }, "./babel-config": "./configs/jest/babel.config.cjs", "./jest-config": "./configs/jest/jest.config.cjs", + "./test-budget": "./configs/jest/test-budget.cjs", "./kit-config": "./configs/rnx-kit/rnx-kit.config.cts", "./lint-config": "./configs/oxlint/oxlint.config.ts", "./package.json": "./package.json", @@ -29,7 +30,7 @@ "build": "tsc -b", "format": "node ./src/cli.ts format", "lint": "node ./src/cli.ts lint", - "test": "node --test src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/tasks/smokeDesktopTarballs.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/utils/securityPatches.test.ts src/utils/skillDistribution.test.ts src/codegen/file.test.ts src/codegen/valuePool.test.ts" + "test": "node --test configs/jest/test-policy.test.cjs src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/tasks/smokeDesktopTarballs.test.mjs src/utils/jestSelection.test.ts src/utils/runScript.test.ts src/utils/securityPatches.test.ts src/utils/skillDistribution.test.ts src/codegen/file.test.ts src/codegen/valuePool.test.ts" }, "dependencies": { "@babel/core": "catalog:", diff --git a/scripts/src/tasks/jest.ts b/scripts/src/tasks/jest.ts index 9acd468616..8b97917b8a 100644 --- a/scripts/src/tasks/jest.ts +++ b/scripts/src/tasks/jest.ts @@ -58,9 +58,6 @@ export class JestCommand extends Command { return; } const args = ['--passWithNoTests', '--runTestsByPath']; - if (process.env.TF_BUILD) { - args.push('--runInBand'); - } const defaultPlatform = getBaseJestPlatform(context); const targetFiles = getJestTargetFiles(context, defaultPlatform);