test_runner: support programmatically running --test

PR-URL: #44241
Backport-PR-URL: #44873
Fixes: #44023
Fixes: #43675
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Antoine du Hamel <[email protected]>

Author: MoLow

doc/api/test.md

@@ -316,6 +316,35 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## `run([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object} Configuration options for running tests. The following
+  properties are supported:
+  * `concurrency` {number|boolean} If a number is provided,
+    then that many files would run in parallel.
+    If truthy, it would run (number of cpu cores - 1)
+    files in parallel.
+    If falsy, it would only run one file at a time.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `true`.
+  * `files`: {Array} An array containing the list of files to run.
+    **Default** matching files from [test runner execution model][].
+  * `signal` {AbortSignal} Allows aborting an in-progress test execution.
+  * `timeout` {number} A number of milliseconds the test execution will
+    fail after.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `Infinity`.
+* Returns: {TapStream}
+
+```js
+run({ files: [path.resolve('./tests/test.js')] })
+  .pipe(process.stdout);
+```
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -560,6 +589,47 @@ describe('tests', async () => {
 });
 ```
 
+## Class: `TapStream`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Extends {ReadableStream}
+
+A successful call to [`run()`][] method will return a new {TapStream}
+object, streaming a [TAP][] output
+`TapStream` will emit events, in the order of the tests definition
+
+### Event: `'test:diagnostic'`
+
+* `message` {string} The diagnostic message.
+
+Emitted when [`context.diagnostic`][] is called.
+
+### Event: `'test:fail'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `error` {Error} The failure casing test to fail.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test fails.
+
+### Event: `'test:pass'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test passes.
+
 ## Class: `TestContext`
 
 <!-- YAML
@@ -825,6 +895,10 @@ added: v16.17.0
 [`--test`]: cli.md#--test
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`context.diagnostic`]: #contextdiagnosticmessage
+[`context.skip`]: #contextskipmessage
+[`context.todo`]: #contexttodomessage
+[`run()`]: #runoptions
 [`test()`]: #testname-options-fn
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn

lib/internal/main/test_runner.js

@@ -1,146 +1,14 @@
 'use strict';
-const {
-  ArrayFrom,
-  ArrayPrototypeFilter,
-  ArrayPrototypeIncludes,
-  ArrayPrototypeJoin,
-  ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSort,
-  SafePromiseAll,
-  SafeSet,
-} = primordials;
 const {
   prepareMainThreadExecution,
 } = require('internal/bootstrap/pre_execution');
-const { spawn } = require('child_process');
-const { readdirSync, statSync } = require('fs');
-const console = require('internal/console/global');
-const {
-  codes: {
-    ERR_TEST_FAILURE,
-  },
-} = require('internal/errors');
-const { test } = require('internal/test_runner/harness');
-const { kSubtestsFailed } = require('internal/test_runner/test');
-const {
-  isSupportedFileType,
-  doesPathMatchFilter,
-} = require('internal/test_runner/utils');
-const { basename, join, resolve } = require('path');
-const { once } = require('events');
-const kFilterArgs = ['--test'];
+const { run } = require('internal/test_runner/runner');
 
 prepareMainThreadExecution(false);
 markBootstrapComplete();
 
-// TODO(cjihrig): Replace this with recursive readdir once it lands.
-function processPath(path, testFiles, options) {
-  const stats = statSync(path);
-
-  if (stats.isFile()) {
-    if (options.userSupplied ||
-        (options.underTestDir && isSupportedFileType(path)) ||
-        doesPathMatchFilter(path)) {
-      testFiles.add(path);
-    }
-  } else if (stats.isDirectory()) {
-    const name = basename(path);
-
-    if (!options.userSupplied && name === 'node_modules') {
-      return;
-    }
-
-    // 'test' directories get special treatment. Recursively add all .js,
-    // .cjs, and .mjs files in the 'test' directory.
-    const isTestDir = name === 'test';
-    const { underTestDir } = options;
-    const entries = readdirSync(path);
-
-    if (isTestDir) {
-      options.underTestDir = true;
-    }
-
-    options.userSupplied = false;
-
-    for (let i = 0; i < entries.length; i++) {
-      processPath(join(path, entries[i]), testFiles, options);
-    }
-
-    options.underTestDir = underTestDir;
-  }
-}
-
-function createTestFileList() {
-  const cwd = process.cwd();
-  const hasUserSuppliedPaths = process.argv.length > 1;
-  const testPaths = hasUserSuppliedPaths ?
-    ArrayPrototypeSlice(process.argv, 1) : [cwd];
-  const testFiles = new SafeSet();
-
-  try {
-    for (let i = 0; i < testPaths.length; i++) {
-      const absolutePath = resolve(testPaths[i]);
-
-      processPath(absolutePath, testFiles, { userSupplied: true });
-    }
-  } catch (err) {
-    if (err?.code === 'ENOENT') {
-      console.error(`Could not find '${err.path}'`);
-      process.exit(1);
-    }
-
-    throw err;
-  }
-
-  return ArrayPrototypeSort(ArrayFrom(testFiles));
-}
-
-function filterExecArgv(arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg);
-}
-
-function runTestFile(path) {
-  return test(path, async (t) => {
-    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
-    ArrayPrototypePush(args, path);
-
-    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
-    let err;
-
-    child.on('error', (error) => {
-      err = error;
-    });
-
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      child.stdout.toArray({ signal: t.signal }),
-      child.stderr.toArray({ signal: t.signal }),
-    ]);
-
-    if (code !== 0 || signal !== null) {
-      if (!err) {
-        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
-        err.exitCode = code;
-        err.signal = signal;
-        err.stdout = ArrayPrototypeJoin(stdout, '');
-        err.stderr = ArrayPrototypeJoin(stderr, '');
-        // The stack will not be useful since the failures came from tests
-        // in a child process.
-        err.stack = undefined;
-      }
-
-      throw err;
-    }
-  });
-}
-
-(async function main() {
-  const testFiles = createTestFileList();
-
-  for (let i = 0; i < testFiles.length; i++) {
-    runTestFile(testFiles[i]);
-  }
-})();
+const tapStream = run();
+tapStream.pipe(process.stdout);
+tapStream.once('test:fail', () => {
+  process.exitCode = 1;
+});