test_runner: count nested tests

Author: MoLow

lib/internal/test_runner/reporter/tap.js

@@ -113,6 +113,7 @@ function reportDetails(nesting, data = kEmptyObject) {
   let details = `${_indent}  ---\n`;
 
   details += jsToYaml(_indent, 'duration_ms', duration_ms);
+  details += jsToYaml(_indent, 'type', data.type);
   details += jsToYaml(_indent, null, error);
   details += `${_indent}  ...\n`;
   return details;

lib/internal/test_runner/runner.js

@@ -13,7 +13,6 @@ const {
   FunctionPrototypeCall,
   Number,
   ObjectAssign,
-  ObjectKeys,
   PromisePrototypeThen,
   SafePromiseAll,
   SafePromiseAllReturnVoid,
@@ -151,10 +150,14 @@ function getRunArgs({ path, inspectPort }) {
 
 class FileTest extends Test {
   #buffer = [];
-  #counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
+  #reportedChildren = 0;
   failedSubtests = false;
+  omitFromTopLevelCounters = true;
   #skipReporting() {
-    return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
+    return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
+  }
+  get omitFromCounters() {
+    return this.#skipReporting();
   }
   #checkNestedComment({ comment }) {
     const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
@@ -204,11 +207,19 @@ class FileTest extends Test {
         const method = pass ? 'ok' : 'fail';
         this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
         if (nesting === 0) {
-          FunctionPrototypeCall(super.countSubtest,
-                                { finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled },
-                                this.#counters);
           this.failedSubtests ||= !pass;
         }
+        this.#reportedChildren++;
+        FunctionPrototypeCall(super.countCompleted, {
+          name: node.description,
+          finished: true,
+          skipped: skip,
+          isTodo: todo,
+          passed: pass,
+          cancelled,
+          nesting,
+          omitFromCounters: diagnostics.type === 'suite',
+        });
         break;
 
       }
@@ -233,14 +244,6 @@ class FileTest extends Test {
     this.reportStarted();
     this.#handleReportItem(ast);
   }
-  countSubtest(counters) {
-    if (this.#counters.all === 0) {
-      return super.countSubtest(counters);
-    }
-    ArrayPrototypeForEach(ObjectKeys(counters), (key) => {
-      counters[key] += this.#counters[key];
-    });
-  }
   reportStarted() {}
   report() {
     const skipReporting = this.#skipReporting();

lib/internal/test_runner/test.js

@@ -71,6 +71,8 @@ const kUnwrapErrors = new SafeSet()
   .add('uncaughtException').add('unhandledRejection');
 const { testNamePatterns, testOnlyFlag } = parseCommandLine();
 
+const completedTests = new SafeSet();
+
 function stopTest(timeout, signal) {
   if (timeout === kDefaultTimeout) {
     return once(signal, 'abort');
@@ -575,31 +577,11 @@ class Test extends AsyncResource {
     this.postRun();
   }
 
-  countSubtest(counters) {
-    // Check SKIP and TODO tests first, as those should not be counted as
-    // failures.
-    if (this.skipped) {
-      counters.skipped++;
-    } else if (this.isTodo) {
-      counters.todo++;
-    } else if (this.cancelled) {
-      counters.cancelled++;
-    } else if (!this.passed) {
-      counters.failed++;
-    } else {
-      counters.passed++;
-    }
-
-    if (!this.passed) {
-      counters.totalFailed++;
-    }
-    counters.all++;
+  countCompleted() {
+    completedTests.add(this);
   }
 
   postRun(pendingSubtestsError) {
-    const counters = {
-      __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0,
-    };
     // If the test was failed before it even started, then the end time will
     // be earlier than the start time. Correct that here.
     if (this.endTime < this.startTime) {
@@ -610,19 +592,22 @@ class Test extends AsyncResource {
     // The test has run, so recursively cancel any outstanding subtests and
     // mark this test as failed if any subtests failed.
     this.pendingSubtests = [];
+    let failed = 0;
     for (let i = 0; i < this.subtests.length; i++) {
       const subtest = this.subtests[i];
 
       if (!subtest.finished) {
         subtest.#cancel(pendingSubtestsError);
         subtest.postRun(pendingSubtestsError);
       }
-      subtest.countSubtest(counters);
+      if (!subtest.passed) {
+        failed++;
+      }
     }
 
-    if ((this.passed || this.parent === null) && counters.totalFailed > 0) {
-      const subtestString = `subtest${counters.totalFailed > 1 ? 's' : ''}`;
-      const msg = `${counters.totalFailed} ${subtestString} failed`;
+    if ((this.passed || this.parent === null) && failed > 0) {
+      const subtestString = `subtest${failed > 1 ? 's' : ''}`;
+      const msg = `${failed} ${subtestString} failed`;
 
       this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
     }
@@ -635,9 +620,43 @@ class Test extends AsyncResource {
       this.parent.addReadySubtest(this);
       this.parent.processReadySubtestRange(false);
       this.parent.processPendingSubtests();
+      this.countCompleted();
     } else if (!this.reported) {
       this.reported = true;
-      this.reporter.plan(this.nesting, kFilename, counters.all);
+
+      const counters = {
+        __proto__: null,
+        all: 0,
+        failed: 0,
+        passed: 0,
+        cancelled: 0,
+        skipped: 0,
+        todo: 0,
+        topLevel: 0,
+      };
+      completedTests.forEach((test) => {
+        if (test.nesting === this.nesting && !test.omitFromTopLevelCounters) {
+          counters.topLevel++;
+        }
+        if (test.omitFromCounters) {
+          return;
+        }
+        // Check SKIP and TODO tests first, as those should not be counted as
+        // failures.
+        if (test.skipped) {
+          counters.skipped++;
+        } else if (test.isTodo) {
+          counters.todo++;
+        } else if (test.cancelled) {
+          counters.cancelled++;
+        } else if (!test.passed) {
+          counters.failed++;
+        } else {
+          counters.passed++;
+        }
+        counters.all++;
+      });
+      this.reporter.plan(this.nesting, kFilename, counters.topLevel);
 
       for (let i = 0; i < this.diagnostics.length; i++) {
         this.reporter.diagnostic(this.nesting, kFilename, this.diagnostics[i]);
@@ -703,6 +722,10 @@ class Test extends AsyncResource {
       directive = this.reporter.getTodo(this.message);
     }
 
+    if (this.reportedType) {
+      details.type = this.reportedType;
+    }
+
     if (this.passed) {
       this.reporter.ok(this.nesting, kFilename, this.testNumber, this.name, details, directive);
     } else {
@@ -746,6 +769,8 @@ class TestHook extends Test {
 }
 
 class Suite extends Test {
+  omitFromCounters = true;
+  reportedType = 'suite';
   constructor(options) {
     super(options);
 

test/message/test_runner_abort.out

@@ -260,10 +260,10 @@ not ok 4 - callback abort signal
     *
   ...
 1..4
-# tests 4
-# pass 0
+# tests 22
+# pass 8
 # fail 0
-# cancelled 4
+# cancelled 14
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_abort_suite.out

@@ -64,6 +64,7 @@ TAP version 13
 not ok 1 - describe timeout signal
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'testAborted'
   error: 'The operation was aborted due to timeout'
   code: 23
@@ -78,6 +79,7 @@ not ok 1 - describe timeout signal
 not ok 2 - describe abort signal
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'testAborted'
   error: 'This operation was aborted'
   code: 20
@@ -94,10 +96,10 @@ not ok 2 - describe abort signal
     *
   ...
 1..2
-# tests 2
-# pass 0
+# tests 9
+# pass 4
 # fail 0
-# cancelled 2
+# cancelled 5
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_describe_it.out

@@ -210,6 +210,7 @@ ok 21 - immediate resolve pass
 not ok 22 - subtest sync throw fail
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'subtestsFailed'
   error: '1 subtest failed'
   code: 'ERR_TEST_FAILURE'
@@ -247,11 +248,13 @@ not ok 23 - sync throw non-error fail
 ok 24 - level 0a
   ---
   duration_ms: *
+  type: 'suite'
   ...
 # Subtest: invalid subtest - pass but subtest fails
 ok 25 - invalid subtest - pass but subtest fails
   ---
   duration_ms: *
+  type: 'suite'
   ...
 # Subtest: sync skip option
 ok 26 - sync skip option # SKIP
@@ -494,6 +497,7 @@ not ok 53 - custom inspect symbol that throws fail
 not ok 54 - subtest sync throw fails
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'subtestsFailed'
   error: '2 subtests failed'
   code: 'ERR_TEST_FAILURE'
@@ -511,6 +515,7 @@ not ok 54 - subtest sync throw fails
 not ok 55 - describe sync throw fails
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'testCodeFailure'
   error: 'thrown from describe'
   code: 'ERR_TEST_FAILURE'
@@ -539,6 +544,7 @@ not ok 55 - describe sync throw fails
 not ok 56 - describe async throw fails
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'testCodeFailure'
   error: 'thrown from describe'
   code: 'ERR_TEST_FAILURE'
@@ -587,6 +593,7 @@ not ok 56 - describe async throw fails
 not ok 57 - timeouts
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'subtestsFailed'
   error: '2 subtests failed'
   code: 'ERR_TEST_FAILURE'
@@ -612,6 +619,7 @@ not ok 57 - timeouts
 not ok 58 - successful thenable
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'subtestsFailed'
   error: '1 subtest failed'
   code: 'ERR_TEST_FAILURE'
@@ -620,6 +628,7 @@ not ok 58 - successful thenable
 not ok 59 - rejected thenable
   ---
   duration_ms: *
+  type: 'suite'
   failureType: 'testCodeFailure'
   error: 'custom error'
   code: 'ERR_TEST_FAILURE'
@@ -643,10 +652,10 @@ not ok 60 - invalid subtest fail
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 60
-# pass 23
-# fail 22
-# cancelled 0
+# tests 67
+# pass 29
+# fail 19
+# cancelled 4
 # skipped 10
 # todo 5
 # duration_ms *

test/message/test_runner_describe_nested.out

@@ -10,11 +10,13 @@ TAP version 13
     ok 1 - nested
       ---
       duration_ms: *
+      type: 'suite'
       ...
     1..1
 ok 1 - nested - no tests
   ---
   duration_ms: *
+  type: 'suite'
   ...
 1..1
 # tests 1