feat: report retried tests (#1)

Author: noomorph

.eslintrc

@@ -19,6 +19,7 @@
       "__fixtures__",
       "coverage",
       "dist",
+      "scripts",
       "website",
       "jest.config.js",
       "*.jest.config.js"
@@ -51,11 +52,12 @@
         "unicorn/prefer-node-protocol": "off",
         "unicorn/prefer-string-replace-all": "off",
         "unicorn/no-unused-properties": "off",
-        "unicorn/prefer-module": "off"
+        "unicorn/prefer-module": "off",
+        "jsdoc/require-jsdoc": "off"
     },
     "overrides": [
         {
-            "files": ["**/__tests__/*.ts", "*.test.ts"],
+            "files": ["**/__tests__/*.ts", "**/__utils__/*.ts", "*.test.ts"],
             "rules": {
               "@typescript-eslint/no-explicit-any": "off",
               "node/no-unpublished-import": "off",

.github/workflows/ci.yml

@@ -0,0 +1,50 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+    paths-ignore:
+      - '**/*.md'
+  pull_request:
+    branches: [master]
+    paths-ignore:
+      - '**/*.md'
+
+jobs:
+
+  unit:
+    name: Unit Tests
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        jest: ['27.x.x', '28.x.x', '29.x.x']
+        include:
+          - node: 14
+            jest: '27.x.x'
+          - node: 16
+            jest: '28.x.x'
+          - node: 18
+            jest: '29.x.x'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node }}
+      - uses: bahmutov/npm-install@v1
+        with:
+          useLockFile: false
+      - name: Lint
+        run: npm run lint:ci
+      - name: Record fixtures
+        run: npm run record
+        env:
+          JEST_VERSION: ${{ matrix.jest }}
+      - name: Unit tests
+        run: npm test -- --coverage
+      - name: Semantic release
+        if: github.ref == 'refs/heads/master'
+        run: npx --no-install semantic-release --debug
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -1,5 +1,6 @@
 /tmp
-/node_modules
+
+node_modules
 package-lock.json
 yarn.lock
 /npm-debug.log
@@ -17,5 +18,6 @@ yarn.lock
 /dist
 /jest-allure2-reporter-*.tgz
 
-/__fixtures__/allure-results
-/__fixtures__/results
+allure-results
+/__fixtures__
+!/__fixtures__/tests

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npm run lint:staged

CONTRIBUTING.md

@@ -32,7 +32,13 @@ npm install
 Generate fixtures:
 
 ```bash
-npm run fixtures:results
+npm run record
+```
+
+> If you want to check compatiblity with a specific Jest version, you can use the `JEST_VERSION` environment variable:
+
+```bash
+JEST_VERSION=27 npm run record
 ```
 
 Run tests:
@@ -41,16 +47,22 @@ Run tests:
 npm test
 ```
 
-To view the test results, generate the Allure report:
+To view the test results, regenerate the full version of the fixtures:
+
+```bash
+npm run start
+```
+
+> If you want to check compatiblity with a specific Jest version, you can use the `JEST_VERSION` environment variable:
 
 ```bash
-npm run fixtures:allure
+JEST_VERSION=27 npm run start
 ```
 
-Start the Allure server:
+To re-run the Allure CLI server, run:
 
 ```bash
-npm start
+npm run serve
 ```
 
 ### Checking your code
@@ -70,4 +82,4 @@ npx cz
 
 and follow the instructions. We adhere to Angular's [commit message guidelines](https://github.com/angular/angular/blob/main/CONTRIBUTING.md#commit).
 
-Thanks in advance for your contribution!
+Thanks in advance for your contribution!

README.md

@@ -50,10 +50,12 @@ module.exports = {
 
 Below is a list of all the available options:
 
-| Property      | Type | Default | Description |
-| ------------- | ---- | ------- | ----------- |
-| `resultsDir`  | `string` | `<rootDir>/allure-results` | The directory where the results will be written to. |
-| `packageName` | `string` | `require('./package.json').name` | The name of your package. |
+| Property                   | Type                     | Default                          | Description                                                                                                                                                                                                                                |
+|----------------------------|--------------------------|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `errorsAsFailedAssertions` | `boolean`                | `false`                          | Treat thrown errors as failed assertions. By default, the reporter distinguishes between failed assertions and thrown errors. The former are reported as FAILED tests, the latter as BROKEN tests.                                         |
+| `getEnvironmentInfo`       | `function` or `boolean`  | `true`                           | Can be customized with an async function to extract environment information from the test environment. By default, the environment information is extracted from the `process.env` object. Use `false` to disable environment information. |
+| `packageName`              | `string`                 | `require('./package.json').name` | Add an extra label to each test case with the package name. Helpful when running tests from multiple packages in a monorepo.                                                                                                               |
+| `resultsDir`               | `string`                 | `<rootDir>/allure-results`       | Path to the directory where the report will be generated.                                                                                                                                                                                  |
 
 ## Usage
 

__fixtures__/dump-reporter.js

@@ -0,0 +1,37 @@
+const path = require('path');
+
+// eslint-disable-next-line node/no-unpublished-import
+const fs = require('fs-extra');
+
+class DumpReporter {
+  constructor(globalConfig) {
+    this._rootDir = globalConfig.rootDir;
+    this._outFile = path.join(this._rootDir, 'recordings', require('jest/package.json').version + '.jsonl');
+  }
+  onRunStart(aggregatedResult, options) {
+    console.log('Starting dump: %s', this._outFile);
+    fs.ensureFileSync(this._outFile);
+    fs.writeFileSync(this._outFile, '');
+    this._append({ method: 'onRunStart', params: [aggregatedResult, options] });
+  }
+  onTestFileStart(test) {
+    this._append({ method: 'onTestFileStart', params: [test] });
+  }
+  onTestCaseResult(test, testCaseResult) {
+    this._append({ method: 'onTestCaseResult', params: [test, testCaseResult] });
+  }
+  onTestFileResult(test, testResult, aggregatedResult) {
+    this._append({ method: 'onTestFileResult', params: [test, testResult, aggregatedResult] });
+  }
+  onRunComplete(_testContexts, results) {
+    this._append({ method: 'onRunComplete', params: [{}, results] });
+  }
+  getLastError() { }
+  _append(payload) {
+    console.log('Append: %s', payload.method);
+    const timedPayload = { ...payload, time: Date.now() };
+    fs.appendFileSync(this._outFile, JSON.stringify(timedPayload) + '\n');
+  }
+}
+
+module.exports = DumpReporter;

__fixtures__/tests/simple/simple-suite-retry.test.js renamed to __fixtures__/tests/flaky.test.js

@@ -1,6 +1,6 @@
 jest.retryTimes(1);
 
-describe('Suite', () => {
+describe('flaky test', () => {
   let firstRun = true;
 
   test('should pass on the second attempt', async () => {

__fixtures__/tests/hook-errors.test.js

@@ -0,0 +1,24 @@
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+describe('hook errors', () => {
+  describe.each([
+    ['beforeAll'],
+    ['beforeEach'],
+    ['afterEach'],
+    ['afterAll'],
+  ])('%s', (hook) => {
+    global[hook](async () => {
+      await sleep(50);
+
+      throw new Error(`Simulated error in ${hook}(...)`);
+    });
+
+    test('Test #1', async () => {
+      // some test
+    });
+
+    test('Test #2', async () => {
+      // some test
+    });
+  });
+});

__fixtures__/tests/hooks/hooks-beforeAll-fail.test.js

@@ -32,7 +32,7 @@ describe('Grand parent suite', () => {
 
     afterAll(async () => {
       console.log('Parent suite: afterAll');
-      throw new Error('Parent suite error');
+      throw new Error('Parent suite: afterAll error');
     });
   });
 });

__fixtures__/tests/nested/nested-suite.test.js renamed to __fixtures__/tests/nesting.test.js

@@ -0,0 +1,25 @@
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+generateSimpleSuite('root');
+
+describe('Suite', () => {
+  generateSimpleSuite('inner');
+});
+
+function generateSimpleSuite(name) {
+  test.skip(`${name} skipped test`, async () => {
+    // empty test
+  });
+
+  test(`${name} broken test`, async () => {
+    throw new Error('Simulated error');
+  });
+
+  test(`${name} failed test`, async () => {
+    expect(2 + 2).toBe(5);
+  });
+
+  test(`${name} passed test (600ms)`, async () => {
+    await sleep(600);
+  });
+}

__fixtures__/tests/root-test-fail.test.js

@@ -2,5 +2,8 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
-  testMatch: ['<rootDir>/src/**/*.test.{js,ts}'],
+  testMatch: [
+    '<rootDir>/src/**/*.test.{js,ts}',
+    '<rootDir>/src/__tests__/**/*.{js,ts}'
+  ],
 };