diff --git a/doc/api/test.md b/doc/api/test.md index 01703227f8cf..2fdb2878af0e 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -135,6 +135,9 @@ create it. the state file is a JSON file that contains an array of run attempts. Each run attempt is an object mapping successful tests to the attempt they have passed in. The key identifying a test in this map is the test file path, with the line and column where the test is defined. +When tests run with process isolation and a test is defined outside of the test file being run, +for example in a helper shared by several test files, +the key is prefixed with the path of that test file. in a case where a test defined in a specific location is run multiple times, for example within a function or a loop, a counter will be appended to the key, to disambiguate the test runs. diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 53f6771a0912..b143ac28dfdb 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -68,6 +68,7 @@ function createTestTree(rootTestOptions, globalOptions) { teardown: null, snapshotManager: null, previousRuns: null, + rerunEntryFile: null, isFilteringByName, isFilteringByOnly, isFilteringByTags, @@ -219,6 +220,10 @@ function setupFailureStateFile(rootTest, globalOptions) { if (!process.env.NODE_TEST_CONTEXT) { const reporter = reportReruns(rootTest.harness.previousRuns, globalOptions); compose(rootTest.reporter, reporter).pipe(new PassThrough()); + } else { + // The state is written by the parent process, which scopes the tests of a + // child process running a single file to that file. + rootTest.harness.rerunEntryFile = rootTest.entryFile; } } diff --git a/lib/internal/test_runner/reporter/rerun.js b/lib/internal/test_runner/reporter/rerun.js index ba2a1fe39141..fcb497a04ce0 100644 --- a/lib/internal/test_runner/reporter/rerun.js +++ b/lib/internal/test_runner/reporter/rerun.js @@ -5,8 +5,8 @@ const { ArrayPrototypePush, JSONStringify, } = primordials; -const { relative } = require('path'); const { writeFileSync } = require('fs'); +const { getRerunTestId } = require('internal/test_runner/utils'); function reportReruns(previousRuns, globalOptions) { return async function reporter(source) { @@ -16,7 +16,7 @@ function reportReruns(previousRuns, globalOptions) { const roots = []; function getTestId(data) { - return `${relative(globalOptions.cwd, data.file)}:${data.line}:${data.column}`; + return getRerunTestId(globalOptions.cwd, data, data.entryFile); } function startTest(data) { diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index b37198df6d76..6edb25fcc7ad 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -305,7 +305,7 @@ class FileTest extends Test { __proto__: null, line: 1, column: 1, - file: resolve(this.name), + file: resolve(this.config.cwd, this.name), }; this.timeout = null; } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 1b0956e4253c..1e45dfe2d994 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -70,6 +70,7 @@ const { createDeferredCallback, createSeededGenerator, countCompletedTest, + getRerunTestId, isTestFailureError, reporterScope, testChannel, @@ -97,7 +98,6 @@ const { } = require('timers'); const { TIMEOUT_MAX } = require('internal/timers'); const { fileURLToPath } = require('internal/url'); -const { relative } = require('path'); const { availableParallelism } = require('os'); const { innerOk } = require('internal/assert/utils'); const { bigint: hrtime } = process.hrtime; @@ -828,7 +828,7 @@ class Test extends AsyncResource { } if (this.loc != null && this.root.harness.previousRuns != null) { - const baseIdentifier = `${relative(this.config.cwd, this.loc.file)}:${this.loc.line}:${this.loc.column}`; + const baseIdentifier = getRerunTestId(this.config.cwd, this.loc, this.root.harness.rerunEntryFile); let testIdentifier = baseIdentifier; const disambiguator = this.root.testDisambiguator.get(baseIdentifier); if (disambiguator !== undefined) { diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index 4f3728fe0926..1624a357b4ac 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -197,6 +197,16 @@ function parsePreviousRuns(rerunFailuresFilePath) { return JSONParse(data); } +// A location outside of the entry file is shared by every test file that +// reaches it, so it is scoped to the entry file when there is one. +function getRerunTestId(cwd, { file, line, column }, entryFile) { + const id = `${relative(cwd, file)}:${line}:${column}`; + if (entryFile == null || entryFile === file) { + return id; + } + return `${relative(cwd, entryFile)}::${id}`; +} + async function getReportersMap(reporters, destinations) { return SafePromiseAllReturnArrayLike(reporters, async (name, i) => { const destination = kBuiltinDestinations.get(destinations[i]) ?? @@ -741,6 +751,7 @@ module.exports = { testStorage, shouldColorizeTestFiles, getCoverageReport, + getRerunTestId, setupGlobalSetupTeardownFunctions, parsePreviousRuns, testChannel, diff --git a/test/fixtures/test-runner/rerun-shared-helper-across-files/a.mjs b/test/fixtures/test-runner/rerun-shared-helper-across-files/a.mjs new file mode 100644 index 000000000000..2f45f6bb2626 --- /dev/null +++ b/test/fixtures/test-runner/rerun-shared-helper-across-files/a.mjs @@ -0,0 +1,4 @@ +import { describeEngine } from './helper.mjs'; + +describeEngine('a-1'); +describeEngine('a-2'); diff --git a/test/fixtures/test-runner/rerun-shared-helper-across-files/b.mjs b/test/fixtures/test-runner/rerun-shared-helper-across-files/b.mjs new file mode 100644 index 000000000000..13d3dc936581 --- /dev/null +++ b/test/fixtures/test-runner/rerun-shared-helper-across-files/b.mjs @@ -0,0 +1,6 @@ +import { describeEngine } from './helper.mjs'; + +describeEngine('b-1'); +describeEngine('b-2'); +describeEngine('b-3'); +describeEngine('b-4', { fails: true }); diff --git a/test/fixtures/test-runner/rerun-shared-helper-across-files/helper.mjs b/test/fixtures/test-runner/rerun-shared-helper-across-files/helper.mjs new file mode 100644 index 000000000000..3c4b0d8abd4c --- /dev/null +++ b/test/fixtures/test-runner/rerun-shared-helper-across-files/helper.mjs @@ -0,0 +1,13 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert'; + +// Every test file calling this registers its suites and tests at the same +// source location, which is outside of the test file itself. +export function describeEngine(name, { fails = false } = {}) { + describe(name, () => { + it('works', ({ attempt }) => { + assert.ok(!fails, `${name} always fails`); + assert.strictEqual(attempt, 0, `${name} already passed and must not run again`); + }); + }); +} diff --git a/test/parallel/test-runner-entry-file.mjs b/test/parallel/test-runner-entry-file.mjs index 94750e6a745e..9b630453825d 100644 --- a/test/parallel/test-runner-entry-file.mjs +++ b/test/parallel/test-runner-entry-file.mjs @@ -48,6 +48,19 @@ describe('entryFile attribution in reporter events', { concurrency: false }, () assert.strictEqual(backupB.data.entryFile, bPath); }); + it('resolves entryFile against the cwd option', async () => { + const events = await collectEvents({ + isolation: 'process', + cwd: fixtures.path('test-runner', 'entry-file'), + files: ['a.test.mjs', 'b.test.mjs'], + }); + const pass = events.filter(({ type }) => type === 'test:pass'); + const backupA = pass.find(({ data }) => data.name === 'backup A'); + const backupB = pass.find(({ data }) => data.name === 'backup B'); + assert.strictEqual(backupA.data.entryFile, aPath); + assert.strictEqual(backupB.data.entryFile, bPath); + }); + it('does not stamp entryFile with isolation none', async () => { const events = await collectEvents({ isolation: 'none' }); for (const { data } of events) { diff --git a/test/parallel/test-runner-test-rerun-failures.js b/test/parallel/test-runner-test-rerun-failures.js index 585406ec919c..cdaef0f1d6d3 100644 --- a/test/parallel/test-runner-test-rerun-failures.js +++ b/test/parallel/test-runner-test-rerun-failures.js @@ -232,6 +232,32 @@ test('failing test is not swallowed when siblings share its source location', as assertAttempt(state, 1); }); +{ + // Both fixtures register all of their suites and tests from the same + // location in a shared helper. In each mode, the suite that fails must keep + // failing on the next attempt, and the ones that passed must be replayed. + const a = fixtures.path('test-runner', 'rerun-shared-helper-across-files', 'a.mjs'); + const b = fixtures.path('test-runner', 'rerun-shared-helper-across-files', 'b.mjs'); + const modes = [ + { name: 'process isolation', args: ['--test', a, b], pass: 5 }, + { name: 'no isolation', args: ['--test', '--test-isolation=none', a, b], pass: 5 }, + { name: 'a file run without --test', args: [b], pass: 3 }, + ]; + + for (const { name, args, pass } of modes) { + test(`failing test in a helper shared by test files is not swallowed with ${name}`, async () => { + for (let attempt = 0; attempt < 2; attempt++) { + const { code, signal, stdout } = await common.spawnPromisified( + process.execPath, ['--test-rerun-failures', stateFile, ...args]); + assert.strictEqual(signal, null); + assert.match(stdout, new RegExp(`^ℹ pass ${pass}$`, 'm'), `attempt ${attempt}`); + assert.match(stdout, /^ℹ fail 1$/m, `attempt ${attempt}`); + assert.strictEqual(code, 1, `attempt ${attempt}`); + } + }); + } +} + test('rerun preserves the original duration on the replayed pass', async () => { const durationFixture = fixtures.path('test-runner', 'rerun-duration.js'); const args = ['--test-rerun-failures', stateFile, durationFixture]; diff --git a/test/parallel/test-runner-v8-deserializer.mjs b/test/parallel/test-runner-v8-deserializer.mjs index f376ca6edb86..b0ca5eb7d3f2 100644 --- a/test/parallel/test-runner-v8-deserializer.mjs +++ b/test/parallel/test-runner-v8-deserializer.mjs @@ -91,7 +91,7 @@ describe('v8 deserializer', common.mustCall(() => { reported = []; fileTest = new runner.FileTest({ name: 'filetest', - harness: { config: {} }, + harness: { config: { cwd: process.cwd() } }, }); fileTest.reporter.on('data', (data) => reported.push(data)); assert(fileTest.isClearToSend());