test_runner: create flag --check-coverage to enforce code coverage

Author: marco-ippolito

doc/api/cli.md

@@ -892,6 +892,66 @@ generated as part of the test runner output. If no tests are run, a coverage
 report is not generated. See the documentation on
 [collecting code coverage from tests][] for more details.
 
+### `--check-coverage`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--check-coverage` CLI flag, used in conjunction with
+the `--experimental-test-coverage` commands, enforce
+that test coverage thresholds on specified checks
+(`--lines`, `--branches`, `--functions`) are respected.
+
+### `--lines`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--lines` CLI flag, used in conjunction with the `--check-coverage` flag,
+enforces coverage thresholds check on lines of code covered by test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold, the test will result in a failure.
+
+### `--branches`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--branches` CLI flag, used in conjunction with the `--check-coverage` flag,
+enforces coverage thresholds check of branches covered by test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold, the test will result in a failure.
+
+### `--functions`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--functions` CLI flag, used in conjunction
+with the `--check-coverage` flag,
+enforces coverage thresholds check of functions covered by test.
+It is expressed as a numerical value between `0` and `100`,
+representing the percentage (e.g., 80 for 80% coverage).
+If the coverage falls below the threshold, the test will result in a failure.
+
 ### `--experimental-vm-modules`
 
 <!-- YAML

doc/api/test.md

@@ -403,6 +403,21 @@ if (anAlwaysFalseCondition) {
 }
 ```
 
+By using the CLI flag [`--check-coverage`][]
+in conjunction with the `--experimental-test-coverage` command,
+it is possible to enforce a specific test coverage threshold checks
+(`--lines`, `--branches`, `--functions`).
+When enabled, it evaluates the test coverage achieved during
+the execution of tests and determines whether it meets
+or exceeds the specified coverage threshold checks.
+If the coverage checks falls below the specified threshold,
+the command will result in a failure,
+indicating that the desired test coverage has not been reached.
+
+```bash
+node --test --experimental-test-coverage --check-coverage --lines=100 --branches=100 --function=100
+```
+
 ### Coverage reporters
 
 The tap and spec reporters will print a summary of the coverage statistics.
@@ -2966,6 +2981,7 @@ added:
 
 [TAP]: https://testanything.org/
 [TTY]: tty.md
+[`--check-coverage`]: cli.md#--check-coverage
 [`--experimental-test-coverage`]: cli.md#--experimental-test-coverage
 [`--import`]: cli.md#--importmodule
 [`--test-concurrency`]: cli.md#--test-concurrency

doc/node.1

@@ -91,6 +91,24 @@ Allow spawning process when using the permission model.
 .It Fl -allow-worker
 Allow creating worker threads when using the permission model.
 .
+.It Fl -check-coverage
+Enforce a test coverage thresholds check
+when used with the
+.Fl -experimental-test-coverage
+flag.
+.
+.It Fl -lines
+Enforce a minimum threshold of
+lines of code covered by test coverage (0 - 100).
+.
+.It Fl -branches
+Enforce a minimum threshold of
+branches covered by test coverage (0 - 100).
+.
+.It Fl -functions
+Enforce a minimum threshold of
+functions covered by test coverage (0 - 100).
+.
 .It Fl -completion-bash
 Print source-able bash completion script for Node.js.
 .

lib/internal/test_runner/test.js

@@ -10,6 +10,7 @@ const {
   FunctionPrototype,
   MathMax,
   Number,
+  NumberPrototypeToFixed,
   ObjectSeal,
   PromisePrototypeThen,
   PromiseResolve,
@@ -58,6 +59,7 @@ const {
 const { setTimeout } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { availableParallelism } = require('os');
+const { exitCodes: { kGenericUserError } } = internalBinding('errors');
 const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
@@ -75,7 +77,7 @@ const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
 const kUnwrapErrors = new SafeSet()
   .add(kTestCodeFailure).add(kHookFailure)
   .add('uncaughtException').add('unhandledRejection');
-const { testNamePatterns, testOnlyFlag } = parseCommandLine();
+const { testNamePatterns, testOnlyFlag, coverageCheckThresholds } = parseCommandLine();
 let kResistStopPropagation;
 
 function stopTest(timeout, signal) {
@@ -753,13 +755,33 @@ class Test extends AsyncResource {
       reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
 
       if (coverage) {
-        reporter.coverage(nesting, loc, coverage);
+        if (coverageCheckThresholds) {
+          const { coveredLinesThreshold,
+                  coveredBranchesThreshold,
+                  coveredFunctionsThreshold } = coverageCheckThresholds;
+          const { coveredLinePercent, coveredBranchPercent, coveredFunctionPercent } = coverage.totals;
+          const cb = (msg) => reporter.stderr(loc, msg);
+          this.checkCoverageThreshold(coveredLinePercent, coveredLinesThreshold, 'Lines', cb);
+          this.checkCoverageThreshold(coveredBranchPercent, coveredBranchesThreshold, 'Branches', cb);
+          this.checkCoverageThreshold(coveredFunctionPercent, coveredFunctionsThreshold, 'Functions', cb);
+
+        }
+        reporter.coverage(nesting, loc, coverage, coverageCheckThresholds);
       }
 
       reporter.end();
     }
   }
 
+  checkCoverageThreshold(actual, expected, type, cb) {
+    if (actual < expected) {
+      const msg = `ERROR: ${type} coverage (${NumberPrototypeToFixed(actual, 2)}%) does not meet expected threshold (${expected}%)\n`;
+      process.exitCode = kGenericUserError;
+      cb(msg);
+    }
+  }
+
+
   isClearToSend() {
     return this.parent === null ||
       (

lib/internal/test_runner/tests_stream.js

@@ -114,11 +114,12 @@ class TestsStream extends Readable {
     this[kEmitMessage]('test:stdout', { __proto__: null, message, ...loc });
   }
 
-  coverage(nesting, loc, summary) {
+  coverage(nesting, loc, summary, coverageCheckThresholds) {
     this[kEmitMessage]('test:coverage', {
       __proto__: null,
       nesting,
       summary,
+      coverageCheckThresholds,
       ...loc,
     });
   }

lib/internal/test_runner/utils.js

@@ -193,6 +193,46 @@ function parseCommandLine() {
 
   const isTestRunner = getOptionValue('--test');
   const coverage = getOptionValue('--experimental-test-coverage');
+  const checkCoverage = getOptionValue('--check-coverage');
+  let coverageCheckThresholds;
+
+  if (checkCoverage) {
+    const coveredLinesThreshold = getOptionValue('--lines');
+
+    if (coveredLinesThreshold < 0 || coveredLinesThreshold > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--lines',
+        coveredLinesThreshold,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    const coveredBranchesThreshold = getOptionValue('--branches');
+    if (coveredBranchesThreshold < 0 || coveredBranchesThreshold > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--branches',
+        coveredBranchesThreshold,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    const coveredFunctionsThreshold = getOptionValue('--functions');
+    if (coveredFunctionsThreshold < 0 || coveredFunctionsThreshold > 100) {
+      throw new ERR_INVALID_ARG_VALUE(
+        '--functions',
+        coveredFunctionsThreshold,
+        'must be a value between 0 and 100',
+      );
+    }
+
+    coverageCheckThresholds = {
+      __proto__: null,
+      coveredLinesThreshold,
+      coveredBranchesThreshold,
+      coveredFunctionsThreshold,
+    };
+  }
+
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let destinations;
@@ -244,6 +284,7 @@ function parseCommandLine() {
     __proto__: null,
     isTestRunner,
     coverage,
+    coverageCheckThresholds,
     testOnlyFlag,
     testNamePatterns,
     reporters,
@@ -386,8 +427,8 @@ function getCoverageReport(pad, summary, symbol, color, table) {
   // Head
   if (table) report += addTableLine(prefix, tableWidth);
   report += `${prefix}${getCell('file', filePadLength, StringPrototypePadEnd, truncateEnd)}${kSeparator}` +
-            `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumns, (column, i) => getCell(column, columnPadLengths[i], StringPrototypePadStart)), kSeparator)}${kSeparator}` +
-            `${getCell('uncovered lines', uncoveredLinesPadLength, false, truncateEnd)}\n`;
+    `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumns, (column, i) => getCell(column, columnPadLengths[i], StringPrototypePadStart)), kSeparator)}${kSeparator}` +
+    `${getCell('uncovered lines', uncoveredLinesPadLength, false, truncateEnd)}\n`;
   if (table) report += addTableLine(prefix, tableWidth);
 
   // Body
@@ -404,14 +445,14 @@ function getCoverageReport(pad, summary, symbol, color, table) {
     fileCoverage /= kColumnsKeys.length;
 
     report += `${prefix}${getCell(relativePath, filePadLength, StringPrototypePadEnd, truncateStart, fileCoverage)}${kSeparator}` +
-              `${ArrayPrototypeJoin(ArrayPrototypeMap(coverages, (coverage, j) => getCell(NumberPrototypeToFixed(coverage, 2), columnPadLengths[j], StringPrototypePadStart, false, coverage)), kSeparator)}${kSeparator}` +
-              `${getCell(formatUncoveredLines(getUncoveredLines(file.lines), table), uncoveredLinesPadLength, false, truncateEnd)}\n`;
+      `${ArrayPrototypeJoin(ArrayPrototypeMap(coverages, (coverage, j) => getCell(NumberPrototypeToFixed(coverage, 2), columnPadLengths[j], StringPrototypePadStart, false, coverage)), kSeparator)}${kSeparator}` +
+      `${getCell(formatUncoveredLines(getUncoveredLines(file.lines), table), uncoveredLinesPadLength, false, truncateEnd)}\n`;
   }
 
   // Foot
   if (table) report += addTableLine(prefix, tableWidth);
   report += `${prefix}${getCell('all files', filePadLength, StringPrototypePadEnd, truncateEnd)}${kSeparator}` +
-            `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`;
+    `${ArrayPrototypeJoin(ArrayPrototypeMap(kColumnsKeys, (columnKey, j) => getCell(NumberPrototypeToFixed(summary.totals[columnKey], 2), columnPadLengths[j], StringPrototypePadStart, false, summary.totals[columnKey])), kSeparator)} |\n`;
   if (table) report += addTableLine(prefix, tableWidth);
 
   report += `${prefix}end of coverage report\n`;

src/node_options.cc

@@ -622,6 +622,18 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--experimental-test-coverage",
             "enable code coverage in the test runner",
             &EnvironmentOptions::test_runner_coverage);
+  AddOption("--check-coverage",
+            "check that coverage falls within the thresholds provided",
+            &EnvironmentOptions::test_runner_check_coverage);
+  AddOption("--lines",
+            "coverage threshold for lines of code",
+            &EnvironmentOptions::test_runner_check_coverage_lines);
+  AddOption("--branches",
+            "coverage threshold for code branches",
+            &EnvironmentOptions::test_runner_check_coverage_branches);
+  AddOption("--functions",
+            "coverage threshold for functions",
+            &EnvironmentOptions::test_runner_check_coverage_functions);
   AddOption("--test-name-pattern",
             "run tests whose name matches this regular expression",
             &EnvironmentOptions::test_name_pattern);

src/node_options.h

@@ -166,6 +166,10 @@ class EnvironmentOptions : public Options {
   uint64_t test_runner_concurrency = 0;
   uint64_t test_runner_timeout = 0;
   bool test_runner_coverage = false;
+  bool test_runner_check_coverage = false;
+  uint64_t test_runner_check_coverage_lines = 0;
+  uint64_t test_runner_check_coverage_branches = 0;
+  uint64_t test_runner_check_coverage_functions = 0;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
   std::vector<std::string> test_reporter_destination;

test/fixtures/test-runner/output/coverage_check.js

@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage --lines=100 --branches=100 --functions=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 100,
+            totalBranchCount: 100,
+            totalFunctionCount: 100,
+            coveredLineCount: 100,
+            coveredBranchCount: 100,
+            coveredFunctionCount: 100,
+            coveredLinePercent: 100,
+            coveredBranchPercent: 100,
+            coveredFunctionPercent: 100
+        }
+    }
+});
+
+test('ok');

test/fixtures/test-runner/output/coverage_check.snapshot

@@ -0,0 +1,23 @@
+TAP version 13
+# Subtest: ok
+ok 1 - ok
+  ---
+  duration_ms: *
+  ...
+1..1
+# tests 1
+# suites 0
+# pass 1
+# fail 0
+# cancelled 0
+# skipped 0
+# todo 0
+# duration_ms *
+# start of coverage report
+# -----------------------------------------------------
+# file | line % | branch % | funcs % | uncovered lines
+# -----------------------------------------------------
+# -----------------------------------------------------
+# all… | 100.00 |   100.00 |  100.00 |
+# -----------------------------------------------------
+# end of coverage report

test/fixtures/test-runner/output/coverage_insufficient.js

@@ -0,0 +1,25 @@
+// Flags: --expose-internals --experimental-test-coverage --check-coverage --lines=100 --branches=100 --functions=100
+
+'use strict';
+require('../../../common');
+const { TestCoverage } = require('internal/test_runner/coverage');
+const { test, mock } = require('node:test');
+
+mock.method(TestCoverage.prototype, 'summary', () => {
+    return {
+        files: [],
+        totals: {
+            totalLineCount: 0,
+            totalBranchCount: 0,
+            totalFunctionCount: 0,
+            coveredLineCount: 0,
+            coveredBranchCount: 0,
+            coveredFunctionCount: 0,
+            coveredLinePercent: 0,
+            coveredBranchPercent: 0,
+            coveredFunctionPercent: 0
+        }
+    }
+});
+
+test('ok');