ci: split CI aggregator and generates markdown

bin/ncu-ci

@@ -2,10 +2,6 @@
 
 'use strict';
 
-const {
-  parsePRFromURL
-} = require('../lib/links');
-
 const {
   JobParser,
   parseJobFromURL,
@@ -15,7 +11,7 @@ const {
 } = require('../lib/ci/ci_type_parser');
 
 const {
-  PRBuild, BenchmarkRun, CommitBuild, listBuilds
+  PRBuild, BenchmarkRun, CommitBuild, listBuilds, FailureAggregator
   // , jobCache
 } = require('../lib/ci/ci_result_parser');
 const clipboardy = require('clipboardy');
@@ -26,8 +22,6 @@ const auth = require('../lib/auth');
 const Request = require('../lib/request');
 const CLI = require('../lib/cli');
 const yargs = require('yargs');
-const _ = require('lodash');
-const chalk = require('chalk');
 
 // This is used for testing
 // Default cache dir is ${ncu-source-dir}/.ncu/cache
@@ -49,12 +43,20 @@ const argv = yargs
   })
   .command({
     command: 'walk <type>',
-    desc: 'Walk the CI and store the failures',
+    desc: 'Walk the CI and display the failures',
     builder: (yargs) => {
       yargs
         .positional('type', {
           describe: 'type of CI',
           choices: ['commit', 'pr']
+        })
+        .option('stats', {
+          default: false,
+          describe: 'Aggregate the results'
+        })
+        .option('limit', {
+          default: 99,
+          describe: 'Maximum number of CIs to get data from'
         });
     },
     handler
@@ -160,19 +162,10 @@ async function runQueue(queue, cli, request, argv) {
     dataToJson = dataToJson.concat(build.formatAsJson());
   }
 
-  if (argv.copy) {
-    clipboardy.writeSync(dataToCopy);
-    cli.separator('');
-    cli.log(`Written markdown to clipboard`);
-  }
-
-  if (argv.json) {
-    writeJson(argv.json, dataToJson);
-    cli.separator('');
-    cli.log(`Written JSON to ${argv.json}`);
-  }
-
-  return dataToJson;
+  return {
+    json: dataToJson,
+    copy: dataToCopy
+  };
 }
 
 function pad(any, length) {
@@ -199,55 +192,6 @@ function displayHealth(builds, cli) {
   cli.log(result);
 }
 
-function getHighlight(f) {
-  return f.reason.split('\n')[f.highlight]
-    .replace(/not ok \d+ /, '')
-    .replace(
-      /'JNLP4-connect connection from .+?'/, 'JNLP4-connect connection from ...'
-    )
-    .replace(/FATAL: Could not checkout \w+/, 'FATAL: Could not checkout ...');
-}
-
-function aggregateFailures(cli, failures) {
-  const grouped = _.chain(failures)
-    .groupBy(getHighlight)
-    .toPairs()
-    .sortBy()
-    .value();
-  let results = [];
-  for (const item of grouped) {
-    const [ key, failures ] = item;
-    const cleaned = _.chain(failures)
-      .uniqBy('source')
-      .sortBy((f) => parseJobFromURL(f.upstream).jobid)
-      .value();
-    results.push([ key, failures, cleaned ]);
-  };
-
-  results = _.sortBy(results, r => 0 - (r[2].length));
-
-  cli.separator(chalk.bold('Stats'));
-  for (const item of results) {
-    const [ key, failures, cleaned ] = item;
-    const machines = _.uniq(failures.map(f => f.builtOn)).join(', ');
-    cli.table('Reason', key);
-    cli.table('Type', failures[0].type);
-    const prs = cleaned
-      .map(f => {
-        const parsed = parsePRFromURL(f.source);
-        return parsed ? `#${parsed.prid}` : f.source;
-      })
-      .join(', ');
-    cli.table('Failed PR', `${cleaned.length} (${prs})`);
-    cli.table('Appeared', machines);
-    if (cleaned.length > 1) {
-      cli.table('First CI', `${cleaned[0].upstream}`);
-    }
-    cli.table('Last CI', `${cleaned[cleaned.length - 1].upstream}`);
-    cli.separator();
-  }
-}
-
 async function main(command, argv) {
   const cli = new CLI();
   const credentials = await auth({
@@ -267,7 +211,7 @@ async function main(command, argv) {
     const type = commandToType[argv.type];
     const builds = await listBuilds(cli, request, type);
     if (command === 'walk') {
-      for (const build of builds.failed) {
+      for (const build of builds.failed.slice(0, argv.limit)) {
         queue.push(build);
       }
     } else {
@@ -304,8 +248,29 @@ async function main(command, argv) {
 
   if (queue.length > 0) {
     const data = await runQueue(queue, cli, request, argv);
+
     if (command === 'walk' && argv.stats) {
-      aggregateFailures(cli, data);
+      const aggregator = new FailureAggregator(cli, data.json);
+      data.json = aggregator.aggregate();
+      cli.log('');
+      cli.separator('Stats');
+      cli.log('');
+      aggregator.display();
+      if (argv.copy) {
+        data.copy = aggregator.formatAsMarkdown();
+      }
+    }
+
+    if (argv.copy) {
+      clipboardy.writeSync(data.copy);
+      cli.separator('');
+      cli.log(`Written markdown to clipboard`);
+    }
+
+    if (argv.json) {
+      writeJson(argv.json, data.json);
+      cli.separator('');
+      cli.log(`Written JSON to ${argv.json}`);
     }
   }
 }

lib/ci/ci_failure_parser.js

@@ -250,4 +250,11 @@ CIFailureParser.FAILURE_CONSTRUCTORS = {
   GIT_FAILURE: GitFailure
 };
 CIFailureParser.CIResult = CIResult;
+CIFailureParser.FAILURE_TYPES_NAME = {
+  BUILD_FAILURE: 'Build Failure',
+  JENKINS_FAILURE: 'Jenkins Failure',
+  JS_TEST_FAILURE: 'JSTest Failure',
+  CC_TEST_FAILURE: 'CCTest Failure',
+  GIT_FAILURE: 'Git Failure'
+};
 module.exports = CIFailureParser;

lib/ci/ci_result_parser.js

@@ -1,5 +1,8 @@
 'use strict';
 
+const {
+  parsePRFromURL
+} = require('../links');
 const Cache = require('../cache');
 const CIFailureParser = require('./ci_failure_parser');
 const {
@@ -9,14 +12,16 @@ const {
   FAILURE_CONSTRUCTORS: {
     [BUILD_FAILURE]: BuildFailure
   },
-  CIResult
+  CIResult,
+  FAILURE_TYPES_NAME
 } = CIFailureParser;
 const {
   CI_DOMAIN,
   parseJobFromURL,
   CI_TYPES
 } = require('./ci_type_parser');
 const qs = require('querystring');
+const _ = require('lodash');
 const chalk = require('chalk');
 
 const SUCCESS = 'SUCCESS';
@@ -325,6 +330,125 @@ class TestBuild extends Job {
   }
 }
 
+function getHighlight(f) {
+  return f.reason.split('\n')[f.highlight]
+    .replace(/not ok \d+ /, '')
+    .replace(
+      /'JNLP4-connect connection from .+?'/, 'JNLP4-connect connection from ...'
+    )
+    .replace(/FATAL: Could not checkout \w+/, 'FATAL: Could not checkout ...');
+}
+
+function markdownRow(...args) {
+  let result = '';
+  for (const item of args) {
+    result += `| ${item} `;
+  }
+  return result + '|\n';
+}
+
+class FailureAggregator {
+  constructor(cli, failures) {
+    this.cli = cli;
+    this.failures = failures;
+    this.aggregates = null;
+  }
+
+  aggregate() {
+    const failures = this.failures;
+    const groupedByReason = _.chain(failures)
+      .groupBy(getHighlight)
+      .toPairs()
+      .sortBy(0)
+      .value();
+    const data = [];
+    for (const item of groupedByReason) {
+      const [ reason, failures ] = item;
+      // If multiple sub builds of one PR are failed by the same reason,
+      // we'll only take one of those builds, as that might be a genuine failure
+      const prs = _.chain(failures)
+        .uniqBy('source')
+        .sortBy((f) => parseJobFromURL(f.upstream).jobid)
+        .map((item) => ({ source: item.source, upstream: item.upstream }))
+        .value();
+      const machines = _.uniq(failures.map(f => f.builtOn));
+      data.push({
+        reason, type: failures[0].type, failures, prs, machines
+      });
+    };
+
+    const groupedByType = _.groupBy(data, 'type');
+    for (const type of Object.keys(groupedByType)) {
+      groupedByType[type] =
+        _.sortBy(groupedByType[type], r => 0 - (r.prs.length));
+    }
+    this.aggregates = groupedByType;
+    return groupedByType;
+  }
+
+  formatAsMarkdown() {
+    let { aggregates } = this;
+    if (!aggregates) {
+      aggregates = this.aggregates = this.aggregate();
+    }
+
+    let output = '';
+    for (const type of Object.keys(aggregates)) {
+      output += `\n### ${FAILURE_TYPES_NAME[type]}\n\n`;
+      for (const item of aggregates[type]) {
+        const { reason, type, prs, failures, machines } = item;
+        if (prs.length < 2) { continue; }
+        output += markdownRow('Reason', `\`${reason}\``);
+        output += markdownRow('-', ':-');
+        output += markdownRow('Type', type);
+        const source = prs.map(f => f.source);
+        output += markdownRow(
+          'Failed PR', `${source.length} (${source.join(', ')})`
+        );
+        output += markdownRow('Appeared', machines.join(', '));
+        if (prs.length > 1) {
+          output += markdownRow('First CI', `${prs[0].upstream}`);
+        }
+        output += markdownRow('Last CI', `${prs[prs.length - 1].upstream}`);
+        output += '\n' + fold('Example', failures[0].reason) + '\n';
+        output += '\n-------\n\n';
+      }
+    }
+    return output;
+  }
+
+  display() {
+    let { cli, aggregates } = this;
+    if (!aggregates) {
+      aggregates = this.aggregates = this.aggregate();
+    }
+
+    for (const type of Object.keys(aggregates)) {
+      cli.separator(type);
+      for (const item of aggregates[type]) {
+        const { reason, type, prs, failures, machines } = item;
+        cli.table('Reason', reason);
+        cli.table('Type', type);
+        const source = prs
+          .map(f => {
+            const parsed = parsePRFromURL(f.source);
+            return parsed ? `#${parsed.prid}` : f.source;
+          });
+        cli.table('Failed PR', `${source.length} (${source.join(', ')})`);
+        cli.table('Appeared', machines.join(', '));
+        if (prs.length > 1) {
+          cli.table('First CI', `${prs[0].upstream}`);
+        }
+        cli.table('Last CI', `${prs[prs.length - 1].upstream}`);
+        cli.log('\n' + chalk.bold('Example:') + '\n');
+        const example = failures[0].reason;
+        cli.log(example.length > 512 ? example.slice(0, 512) + '...' : example);
+        cli.separator();
+      }
+    }
+  }
+}
+
 class CommitBuild extends TestBuild {
   constructor(cli, request, id) {
     const path = `job/node-test-commit/${id}/`;
@@ -714,6 +838,7 @@ class BenchmarkRun extends Job {
 }
 
 module.exports = {
+  FailureAggregator,
   PRBuild,
   BenchmarkRun,
   CommitBuild,