ci: detect resume failures, tweak markdown output

Author: joyeecheung

lib/ci/ci_failure_parser.js

@@ -33,10 +33,11 @@ const CC_TEST_FAILURE = 'CC_TEST_FAILURE';
 const JENKINS_FAILURE = 'JENKINS_FAILURE';
 const GIT_FAILURE = 'GIT_FAILURE';
 const NCU_FAILURE = 'NCU_FAILURE';
+const RESUME_FAILURE = 'RESUME_FAILURE';
 
 const FAILURE_TYPES = {
   BUILD_FAILURE, JS_TEST_FAILURE, CC_TEST_FAILURE,
-  JENKINS_FAILURE, GIT_FAILURE, NCU_FAILURE
+  JENKINS_FAILURE, GIT_FAILURE, NCU_FAILURE, RESUME_FAILURE
 };
 
 class CIResult {
@@ -103,6 +104,14 @@ class NCUFailure extends CIResult {
   }
 }
 
+// Refs: https://github.com/nodejs/build/issues/1496
+class ResumeFailure extends CIResult {
+  constructor(ctx, reason) {
+    super(ctx, reason);
+    this.type = RESUME_FAILURE;
+  }
+}
+
 function failureMatcher(Failure, patterns, ctx, text) {
   for (const pattern of patterns) {
     const matches = text.match(pattern.pattern);
@@ -277,7 +286,8 @@ CIFailureParser.FAILURE_CONSTRUCTORS = {
   JS_TEST_FAILURE: JSTestFailure,
   CC_TEST_FAILURE: CCTestFailure,
   GIT_FAILURE: GitFailure,
-  NCU_FAILURE: NCUFailure
+  NCU_FAILURE: NCUFailure,
+  RESUME_FAILURE: ResumeFailure
 };
 CIFailureParser.CIResult = CIResult;
 CIFailureParser.FAILURE_TYPES_NAME = {
@@ -286,6 +296,7 @@ CIFailureParser.FAILURE_TYPES_NAME = {
   JS_TEST_FAILURE: 'JSTest Failure',
   CC_TEST_FAILURE: 'CCTest Failure',
   GIT_FAILURE: 'Git Failure',
-  NCU_FAILURE: 'node-core-utils failure'
+  NCU_FAILURE: 'node-core-utils failure',
+  RESUME_FAILURE: 'resume failure'
 };
 module.exports = CIFailureParser;

lib/ci/ci_result_parser.js

@@ -8,11 +8,14 @@ const CIFailureParser = require('./ci_failure_parser');
 const {
   FAILURE_TYPES: {
     BUILD_FAILURE,
-    NCU_FAILURE
+    NCU_FAILURE,
+    GIT_FAILURE,
+    RESUME_FAILURE
   },
   FAILURE_CONSTRUCTORS: {
     [BUILD_FAILURE]: BuildFailure,
-    [NCU_FAILURE]: NCUFailure
+    [NCU_FAILURE]: NCUFailure,
+    [RESUME_FAILURE]: ResumeFailure
   },
   CIResult,
   FAILURE_TYPES_NAME
@@ -46,11 +49,12 @@ const COMMIT_TREE =
   `subBuilds[${BUILD_FIELDS}]`;
 // com.tikal.jenkins.plugins.multijob.MultiJobBuild
 const FANNED_TREE =
-  `result,url,number,subBuilds[phaseName,${BUILD_FIELDS}],builtOn`;
+  `result,url,number,subBuilds[phaseName,${BUILD_FIELDS}]`;
 // hudson.matrix.MatrixBuild
 const BUILD_TREE = 'result,runs[url,number,result],builtOn';
 const LINTER_TREE = 'result,url,number,builtOn';
-const RUN_TREE = 'actions[causes[upstreamBuild,upstreamProject]],builtOn';
+const CAUSE_TREE = 'upstreamBuild,upstreamProject,shortDescription,_class';
+const RUN_TREE = `actions[causes[${CAUSE_TREE}]],builtOn`;
 
 function getPath(url) {
   return url.replace(`https://${CI_DOMAIN}/`, '').replace('api/json', '');
@@ -128,6 +132,13 @@ class Job {
     return data;
   }
 
+  getCause(actions) {
+    if (actions && actions.find(item => item.causes)) {
+      const action = actions.find(item => item.causes);
+      return action.causes[0];
+    }
+  }
+
   async getAPIData() {
     const { cli, request, path } = this;
     const url = this.apiUrl;
@@ -355,7 +366,9 @@ function getHighlight(f) {
     )
     .replace(
       /fatal: loose object \w+ \(stored in .git\/objects\/.+\) is corrupt/,
-      'fatal: loose object ... (stored in .git/objects/...) is corrupt');
+      'fatal: loose object ... (stored in .git/objects/...) is corrupt')
+    .replace(/hudson\.plugins\.git\.GitException: /, '')
+    .replace(/java\.io\.IOException: /, '');
 }
 
 function markdownRow(...args) {
@@ -427,13 +440,15 @@ class FailureAggregator {
     output += `[${jobName}/${first.jobid}](${first.link}) to `;
     output += `[${jobName}/${last.jobid}](${last.link}) `;
     output += `that failed more than 2 PRs\n`;
+    output += `(Generated with \`ncu-ci `;
+    output += `${process.argv.slice(2).join(' ')}\`)\n`;
     const todo = [];
     for (const type of Object.keys(aggregates)) {
       output += `\n### ${FAILURE_TYPES_NAME[type]}\n\n`;
       for (const item of aggregates[type]) {
         const { reason, type, prs, failures, machines } = item;
         if (prs.length < 2) { continue; }
-        todo.push(reason);
+        todo.push({ count: prs.length, reason });
         output += markdownRow('Reason', `<code>${reason}</code>`);
         output += markdownRow('-', ':-');
         output += markdownRow('Type', type);
@@ -449,16 +464,19 @@ class FailureAggregator {
         }
         output += markdownRow('Last CI', `${prs[prs.length - 1].upstream}`);
         output += '\n';
+        const example = failures[0].reason;
         output += fold(
           `<a href="${failures[0].url}">Example</a>`,
-          failures[0].reason
+          (example.length > 1024 ? example.slice(0, 1024) + '...' : example)
         );
         output += '\n\n-------\n\n';
       }
     }
 
     output += '### Progress\n\n';
-    output += todo.map(i => `- \`${i}\``).join('\n');
+    output += todo.map(
+      ({count, reason}) => `- \`${reason}\` (${count})`).join('\n'
+    );
     return output + '\n';
   }
 
@@ -559,7 +577,10 @@ class CommitBuild extends TestBuild {
     cli.startSpinner(`Querying failures of ${path}`);
     const promises = builds.failed.map(({jobName, buildNumber, url}) => {
       if (jobName.includes('fanned')) {
-        return new FannedBuild(cli, request, jobName, buildNumber).getResults();
+        const cause = this.getCause(data.actions);
+        const isResumed = cause && cause._class.includes('ResumeCause');
+        return new FannedBuild(cli, request, jobName, buildNumber, isResumed)
+          .getResults();
       } else if (jobName.includes('linter')) {
         return new LinterBuild(cli, request, jobName, buildNumber).getResults();
       } else if (jobName.includes('freestyle')) {
@@ -622,6 +643,7 @@ class PRBuild extends TestBuild {
     const allBuilds = commitBuild.build.subBuilds;
     // TODO: fetch result, builtOn, timestamp in the commit build's own data
     // ..or maybe they do not worth an additional API call?
+    // Note that we have to pass the actions down to detect resume builds.
     const buildData = {
       result, subBuilds: allBuilds, changeSet, actions, timestamp
     };
@@ -682,14 +704,15 @@ async function listBuilds(cli, request, type) {
 }
 
 class FannedBuild extends Job {
-  constructor(cli, request, jobName, id) {
+  constructor(cli, request, jobName, id, isResumed) {
     // assert(jobName.includes('fanned'));
     const path = `job/${jobName}/${id}/`;
     const tree = FANNED_TREE;
     super(cli, request, path, tree);
 
     this.failures = [];
     this.builtOn = undefined;
+    this.isResumed = isResumed;
   }
 
   // Get the failures and their reasons of this build
@@ -724,17 +747,33 @@ class FannedBuild extends Job {
         !failedPhase.phaseName.toLowerCase().includes('compilation')) {
       this.failures = [
         new BuildFailure(
-          { url: failedPhase.url, builtOn: failedPhase.builtOn },
-          `Failed in ${failedPhase.phaseName} phase`
+          { url: failedPhase.url },
+          `Failed in ${failedPhase.phaseName} phase (${failedPhase.jobName})`
+          // TODO: parse console text for the failed phase
         )
       ];
       return this.failures;
     }
 
     const { jobName, buildNumber } = failedPhase;
     const build = new NormalBuild(cli, request, jobName, buildNumber);
-    const failures = await build.getResults();
-    this.failures = flatten(failures);
+    let failures = await build.getResults();
+    failures = flatten(failures);
+
+    if (this.isResumed) {
+      // XXX: if it's a resumed build, we replace the build/git failures
+      // with resume failures. Probably just a random guess, though
+      for (let i = 0; i < failures.length; ++i) {
+        const item = failures[i];
+        if (item.type === BUILD_FAILURE || item.type === GIT_FAILURE) {
+          failures[i] = new ResumeFailure(
+            item,
+            `Possible resume failure\n${item.reason}`
+          );
+        }
+      }
+    }
+    this.failures = failures;
     return this.failures;
   }
 }
@@ -853,10 +892,7 @@ class TestRun extends Job {
       ];
       return this.failures;
     }
-    if (data.actions && data.actions.find(item => item.causes)) {
-      const actions = data.actions.find(item => item.causes);
-      this.cause = actions.causes[0];
-    }
+    this.causes = this.getCause(data.actions) || {};
     this.builtOn = data.builtOn;
   }