test_runner: add option to rerun only failed tests

Author: MoLow

doc/api/cli.md

@@ -2647,6 +2647,18 @@ changes:
 The destination for the corresponding test reporter. See the documentation on
 [test reporters][] for more details.
 
+### `--test-rerun=path`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+A path to a file storing the results of a previous test run. The test runner
+will use this file to determine which tests to re-run, allowing for
+re-running of failed tests without having to re-run the entire test suite.
+The test runner will create this file if it does not exist.
+
 ### `--test-shard`
 
 <!-- YAML
@@ -3508,6 +3520,7 @@ one is included in the list below.
 * `--test-only`
 * `--test-reporter-destination`
 * `--test-reporter`
+* `--test-rerun`
 * `--test-shard`
 * `--test-skip-pattern`
 * `--throw-deprecation`

doc/api/test.md

@@ -153,6 +153,17 @@ test('skip() method with message', (t) => {
 });
 ```
 
+## Rerunning failed tests
+
+The test runner supports rerunning failed tests by passing the [`--test-rerun`][]
+command-line option. This option takes a file path as an argument, which is used
+to store the state of the run, allowing the test runner to rerun only the failed
+tests.
+
+```bash
+node --test-rerun /path/to/state/file
+```
+
 ## TODO tests
 
 Individual tests can be marked as flaky or incomplete by passing the `todo`
@@ -3929,6 +3940,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
 [`--test-reporter`]: cli.md#--test-reporter
+[`--test-rerun`]: cli.md#--test-rerun
 [`--test-skip-pattern`]: cli.md#--test-skip-pattern
 [`--test-update-snapshots`]: cli.md#--test-update-snapshots
 [`--test`]: cli.md#--test

lib/internal/test_runner/reporter/rerun.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const {
+  ArrayPrototypePush,
+  JSONStringify,
+} = primordials;
+const { writeFileSync } = require('fs');
+
+function reportReruns(previousRuns, rerunFilePath) {
+  return async function reporter(source) {
+    const obj = { __proto__: null };
+
+    for await (const { type, data } of source) {
+      if (type === 'test:pass') {
+        obj[`${data.file}:${data.line}:${data.column}`] = data.details.passed_attempt ?? data.details.attempt;
+      }
+    }
+
+    ArrayPrototypePush(previousRuns, obj);
+    writeFileSync(rerunFilePath, JSONStringify(previousRuns, null, 2), 'utf8');
+  };
+};
+
+module.exports = {
+  __proto__: null,
+  reportReruns,
+};

lib/internal/test_runner/runner.js

@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit,
                             only,
                             argv: suppliedArgs,
                             execArgv,
+                            rerunFilePath,
                             root: { timeout },
                             cwd }) {
   const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
@@ -169,6 +170,9 @@ function getRunArgs(path, { forceExit,
   if (timeout != null) {
     ArrayPrototypePush(argv, `--test-timeout=${timeout}`);
   }
+  if (rerunFilePath != null) {
+    ArrayPrototypePush(argv, `--test-rerun=${rerunFilePath}`);
+  }
 
   ArrayPrototypePushApply(argv, execArgv);
 

lib/internal/test_runner/test.js

@@ -290,6 +290,10 @@ class TestContext {
     return this.#test.passed;
   }
 
+  get attempt() {
+    return this.#test.attempt ?? 0;
+  }
+
   diagnostic(message) {
     this.#test.diagnostic(message);
   }
@@ -646,6 +650,8 @@ class Test extends AsyncResource {
     this.endTime = null;
     this.passed = false;
     this.error = null;
+    this.attempt = undefined;
+    this.passedAttempt = undefined;
     this.message = typeof skip === 'string' ? skip :
       typeof todo === 'string' ? todo : null;
     this.activeSubtests = 0;
@@ -690,6 +696,16 @@ class Test extends AsyncResource {
         this.loc.file = fileURLToPath(this.loc.file);
       }
     }
+
+    if (this.loc != null && this.config.previousRuns != null) {
+      const testLocation = `${this.loc.file}:${this.loc.line}:${this.loc.column}`;
+      this.attempt = this.config.previousRuns.length;
+      const previousAttempt = this.config.previousRuns[this.attempt - 1]?.[testLocation];
+      if (previousAttempt != null) {
+        this.passedAttempt = previousAttempt;
+        this.fn = noop;
+      }
+    }
   }
 
   applyFilters() {
@@ -1329,6 +1345,12 @@ class Test extends AsyncResource {
     if (!this.passed) {
       details.error = this.error;
     }
+    if (this.attempt !== undefined) {
+      details.attempt = this.attempt;
+    }
+    if (this.passedAttempt !== undefined) {
+      details.passed_attempt = this.passedAttempt;
+    }
     return { __proto__: null, details, directive };
   }
 

lib/internal/test_runner/utils.js

@@ -8,6 +8,7 @@ const {
   ArrayPrototypePush,
   ArrayPrototypeReduce,
   ArrayPrototypeSome,
+  JSONParse,
   MathFloor,
   MathMax,
   MathMin,
@@ -28,9 +29,10 @@ const {
 
 const { AsyncResource } = require('async_hooks');
 const { relative, sep, resolve } = require('path');
-const { createWriteStream } = require('fs');
+const { createWriteStream, readFileSync } = require('fs');
 const { pathToFileURL } = require('internal/url');
 const { getOptionValue } = require('internal/options');
+const { reportReruns } = require('internal/test_runner/reporter/rerun');
 const { green, yellow, red, white, shouldColorize } = require('internal/util/colors');
 
 const {
@@ -40,7 +42,7 @@ const {
   },
   kIsNodeError,
 } = require('internal/errors');
-const { compose } = require('stream');
+const { compose, PassThrough } = require('stream');
 const {
   validateInteger,
   validateFunction,
@@ -150,6 +152,20 @@ function shouldColorizeTestFiles(destinations) {
   });
 }
 
+function parsePreviousRuns(rerunFilePath) {
+  let data;
+  try {
+    data = readFileSync(rerunFilePath, 'utf8');
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      data = '[]';
+    } else {
+      throw err;
+    }
+  }
+  return JSONParse(data);
+}
+
 async function getReportersMap(reporters, destinations) {
   return SafePromiseAllReturnArrayLike(reporters, async (name, i) => {
     const destination = kBuiltinDestinations.get(destinations[i]) ??
@@ -202,6 +218,7 @@ function parseCommandLine() {
   const updateSnapshots = getOptionValue('--test-update-snapshots');
   const watch = getOptionValue('--watch');
   const timeout = getOptionValue('--test-timeout') || Infinity;
+  const rerunFilePath = getOptionValue('--test-rerun');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let globalSetupPath;
@@ -308,8 +325,24 @@ function parseCommandLine() {
     validateInteger(functionCoverage, '--test-coverage-functions', 0, 100);
   }
 
+  let previousRuns;
+  if (rerunFilePath) {
+    validatePath(rerunFilePath, '--test-rerun');
+    previousRuns = parsePreviousRuns(rerunFilePath);
+    if (previousRuns === null) {
+      throw new ERR_INVALID_ARG_VALUE('--test-rerun', rerunFilePath, 'is not a valid rerun file');
+    }
+  }
+
   const setup = reporterScope.bind(async (rootReporter) => {
     const reportersMap = await getReportersMap(reporters, destinations);
+    if (previousRuns && rerunFilePath) {
+      ArrayPrototypePush(reportersMap, {
+        __proto__: null,
+        reporter: reportReruns(previousRuns, rerunFilePath),
+        destination: new PassThrough(),
+      });
+    }
 
     for (let i = 0; i < reportersMap.length; i++) {
       const { reporter, destination } = reportersMap[i];
@@ -343,6 +376,8 @@ function parseCommandLine() {
     timeout,
     updateSnapshots,
     watch,
+    rerunFilePath,
+    previousRuns,
   };
 
   return globalTestOptions;
@@ -637,4 +672,5 @@ module.exports = {
   shouldColorizeTestFiles,
   getCoverageReport,
   setupGlobalSetupTeardownFunctions,
+  parsePreviousRuns,
 };

src/node_options.cc

@@ -918,6 +918,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             &EnvironmentOptions::test_global_setup_path,
             kAllowedInEnvvar,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-rerun",
+            "report test output using the given reporter",
+            &EnvironmentOptions::test_rerun,
+            kAllowedInEnvvar,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-udp-no-try-send",
             "",  // For testing only.
             &EnvironmentOptions::test_udp_no_try_send,

src/node_options.h

@@ -198,6 +198,7 @@ class EnvironmentOptions : public Options {
   bool test_runner_update_snapshots = false;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
+  std::string test_rerun;
   std::vector<std::string> test_reporter_destination;
   std::string test_global_setup_path;
   bool test_only = false;

test/fixtures/test-runner/rerun.js

@@ -0,0 +1,13 @@
+const { test } = require('node:test')
+
+test('should fail on first two attempts', ({ attempt }) => {
+  if (attempt < 2) {
+    throw new Error('This test is expected to fail on the first two attempts');
+  }
+});
+
+test('ok', ({ attempt }) => {
+  if (attempt > 0) {
+      throw new Error('Test should not rerun once it has passed');
+  }
+});

test/parallel/test-runner-test-rerun.js

@@ -0,0 +1,37 @@
+'use strict';
+const common = require('../common');
+
+const fixtures = require('../common/fixtures');
+const assert = require('node:assert');
+const { rm } = require('node:fs/promises');
+const { test, beforeEach, afterEach } = require('node:test');
+
+const stateFile = fixtures.path('test-runner', 'rerun-state.json');
+
+beforeEach(() => rm(stateFile, { force: true }));
+afterEach(() => rm(stateFile, { force: true }));
+
+test('test should pass on third rerun', async () => {
+  const cwd = fixtures.path('test-runner');
+  const fixture = fixtures.path('test-runner', 'rerun.js');
+  const args = ['--test-rerun', stateFile, fixture];
+
+  let { code, stdout, signal } = await common.spawnPromisified(process.execPath, args);
+  assert.strictEqual(code, 1);
+  assert.strictEqual(signal, null);
+  assert.match(stdout, /pass 1/);
+  assert.match(stdout, /fail 1/);
+
+  ({ code, stdout, signal } = await common.spawnPromisified(process.execPath, args, { cwd }));
+  assert.strictEqual(code, 1);
+  assert.strictEqual(signal, null);
+  assert.match(stdout, /pass 1/);
+  assert.match(stdout, /fail 1/);
+
+
+  ({ code, stdout, signal } = await common.spawnPromisified(process.execPath, args, { cwd }));
+  assert.strictEqual(code, 0);
+  assert.strictEqual(signal, null);
+  assert.match(stdout, /pass 2/);
+  assert.match(stdout, /fail 0/);
+});