platform-helpers/report-generator.js

/**
 * Report Generator functions. Helps add context to the generated HTML reports of the test results.
 *
 * NOTE: These methods only work inside tests!
 * @module ReportGenerator
 */

'use strict';
const showdown = require('showdown').setOption('tables', 'true');
const converter = new showdown.Converter();
const Moment = require('moment');
const HtmlGenerator = require('./wdio-html-generator.js');
const rootRequire = require('rpcm-root-require');
const fse = require('fs-extra');
const log4js = require('log4js');
const Utils = rootRequire('/platform-helpers/utils');
const logger = log4js.getLogger();
rootRequire('/platform-helpers/string-extensions');

/**
 * Used for our summary report generation.  It's basically the same as the Class of the same name within the html-reporter package that we use for the individual reports, but with some tweaks to make it work the way we need it to.
 *
 * @private
 * @class
 */
class ReportGenerator {
  constructor (opts) {
    // make sure provided arguments that are invalid get deleted otherwise they override defaults (skip booleans)
    Object.keys(opts).forEach(key => { if (typeof opts[key] === 'boolean') return; if (key) delete opts[key]; });
    // merge provided arguments with defaults. defaults get overrided
    this.options = Object.assign({}, {
      outputDir: 'reports/html-reports/',
      fileName: 'index.html',
      summaryReportTitle: 'Summary Report',
      reportTitle: 'Summary Report',
      showInBrowser: false,
      templateFilename: '{0}/report-templates/aggregated-template.hbs'.format(rootRequire.rootPath),
      templateFuncs: {},
      browserName: 'not specified',
      LOG: logger,
      reportContext: converter.makeHtml(('This report has been generated by Tealium\'s automated testing platform.{0}' +
        'These tests will have been created specifically for your project by a Tealium Implementation Engineer to verify a configuration or use case.{0}' +
        'The intent is to document the tests run on the completed configuration (acceptance tests). It can also serve as a limited regression test as work continues, to ensure that future projects don\'t break previous ones.{1}' +
        'For more information, see the [developer documentation](https://tealium.github.io/customer-success-testing-platform/).')
        .format(Utils.getNewLine(2), Utils.getNewLine()))
    }, opts);
    this.options.reportFile = '{0}/{1}{2}'.format(rootRequire.rootPath, this.options.outputDir, this.options.fileName);
    this.startTime = new Moment();
    this.reports = [];
  }

  calculateDuration (startTime, endTime) {
    const isMoment = (moment) => {
      return (moment && typeof moment.diff === 'function');
    };
    if (isMoment(startTime) && isMoment(endTime)) {
      return endTime.diff(startTime);
    }
  }

  moveFilesBetweenRuns (index) {
    try {
      // move the files to the 'runs' folder
      const allJsonPaths = Utils.getFilesRecursively(this.options.outputDir, ['.json', '.html']);
      const folderPaths = [];
      allJsonPaths.forEach((path) => {
        const re = /\/runs\//;
        const alreadyMoved = re.test(path);
        const pathParts = path.split('/');
        const fileName = pathParts[pathParts.length - 1];
        const folderPath = pathParts.slice(0, pathParts.length - 2).join('/');
        // don't try to move the files we already moved (which are in the same output dir)
        if (!alreadyMoved) {
          fse.moveSync(path, `${this.options.outputDir}/runs/${index}/${fileName}`);
          folderPaths.push(folderPath);
        }
      });

      folderPaths.forEach((folderPath) => {
        // wait for both the .json and the .html to be moved before we delete the folder
        // will fail silently when we try to remove the same file twice - that's fine.
        fse.removeSync(folderPath);
      });
    } catch (e) {}
  }

  clean () {
    // leave the output dir intact, but empty it
    fse.emptyDirSync(this.options.outputDir);
    // remove the entire 'tmp' folder on start if present (should have been moved already anyway, if the previous report completed successfully)
    fse.removeSync('tmp');
  }

  readJsonFiles () {
    return Utils.getFilesRecursively(this.options.outputDir, ['.json']);
  }

  log (message, object) {
    if (this.options.LOG) {
      this.options.LOG.debug(message + object);
    }
  }

  async createReport (results) {
    const metrics = {
      passed: 0,
      skipped: 0,
      failed: 0,
      total: 0,
      start: this.startTime,
      end: 0,
      duration: 0
    };
    const suites = [];
    const specs = [];
    const files = this.readJsonFiles();

    // copy the static JS and CSS assets into the report output folder
    const staticSource = 'report-templates/static-assets';
    const staticTarget = '{0}/static-assets'.format(this.options.outputDir);
    fse.ensureDirSync(staticTarget);
    fse.copySync(staticSource, staticTarget);

    // move the 'logs' folder into the report output folder, to keep it all together for debugging
    fse.ensureDirSync('tmp/logs');
    fse.copySync('tmp/logs', '{0}/logs'.format(this.options.outputDir));
    fse.ensureDirSync('tmp/tests');
    fse.copySync('tmp/tests', '{0}/tests'.format(this.options.outputDir), { overwrite: true });
    fse.removeSync('tmp');

    const fileList = {};

    files.forEach((fileName) => {
      // keep track of the capabilities we've seen, so we can do something about the missing ones
      const file = JSON.parse(fse.readFileSync(fileName));
      const capNumber = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities['teal:capabilityNumber']) || 0;
      const specPath = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities['teal:specPath']) || '';
      const browserName = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.browserName) || '';
      const browserVersion = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.browserVersion) || '';
      const platformName = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.platformName) || '';
      const key = capNumber + ' | ' + specPath;
      fileList[key] = {
        browser: browserName,
        version: browserVersion,
        platform: platformName,
        reportFile: fileName,
        spec: specPath,
        capNumber: capNumber
      };
    });

    const fullRunList = JSON.parse(results['teal:fullRunList']);
    fullRunList.forEach((run) => {
      const capNumber = run.capability['teal:capabilityNumber'];
      const specPath = run.spec;
      if (specPath + capNumber) {
        const key = capNumber + ' | ' + specPath;
        if (typeof fileList[key] === 'undefined') {
          fileList[key] = {
            browser: run.capability.browserName,
            version: run.capability.browserVersion,
            platform: run.capability.platformName,
            reportFile: '',
            spec: specPath,
            capNumber: capNumber
          };
        }
      }
    });

    // push reports or placeholders for each run
    const allRuns = Object.keys(fileList);

    // sort by capability number
    allRuns.sort((a, b) => {
      const aCapNumber = parseInt(a.split(' | ')[0], 10);
      const bCapNumber = parseInt(b.split(' | ')[0], 10);
      if (aCapNumber < bCapNumber) {
        return -1;
      } else if (aCapNumber > bCapNumber) {
        return 1;
      } else {
        return 0;
      }
    });

    for (let i = 0; i < allRuns.length; i++) {
      try {
        let report;
        const key = allRuns[i];
        const fileName = fileList[key].reportFile;
        // if the file name is blank, that means the run failed (generate stub report for summary)
        if (typeof fileName === 'undefined' || fileName === '') {
          const dummyReport = {
            fileLink: '',
            metrics: {
              passed: 0,
              failed: 1,
              skipped: 0
            },
            info: {
              specs: [
                fileList[key].spec
              ],
              config: {
                capabilities: [{
                  browserName: fileList[key].browser,
                  browserVersion: fileList[key].version,
                  platformName: fileList[key].platform,
                  'teal:capabilityNumber': fileList[key].capNumber
                }]
              },
              // it's weird that this is plural but not an array, but that's how it is elsewhere
              capabilities: {
                browserName: fileList[key].browser,
                browserVersion: fileList[key].version,
                platformName: fileList[key].platformName,
                'teal:capabilityNumber': fileList[key].capNumber
              }
            },
            suites: []
          };
          report = dummyReport;
          // console.log(dummyReport)
          this.reports.push(dummyReport);
        } else {
          // run succeeded, generate the full report
          report = JSON.parse(fse.readFileSync(fileName));
          // remove the dependency on the parent folders for functional links in the summary report
          report.fileLink = fileName.replace('reports/html-reports/', '').replace('reports\\html-reports\\', '');

          report.info.specs.forEach(spec => {
            specs.push(spec);
          });
          this.reports.push(report);
        }

        metrics.passed += report.metrics.passed;
        metrics.total += report.metrics.passed;
        metrics.failed += report.metrics.failed;
        metrics.total += report.metrics.failed;
        metrics.skipped += report.metrics.skipped;
        metrics.total += report.metrics.skipped;

        for (let k = 0; k < report.suites.length; k++) {
          const suite = report.suites[k];
          const start = new Moment(suite.start);

          if (start.isSameOrBefore(metrics.start)) {
            // metrics.start = start
          }

          const end = new Moment(suite.end);

          if (end.isAfter(metrics.end)) {
            // metrics.end = end
          }

          suites.push(suite);
        }
      } catch (ex) {
        console.error(ex);
      }
    }

    metrics.end = new Moment();

    const duration = this.calculateDuration(metrics.start, metrics.end);
    metrics.duration = duration;

    metrics.prettyStart = metrics.start.format('ddd MMMM Do YYYY HH:mm:ss ZZ');
    metrics.prettyEnd = metrics.end.format('ddd MMMM Do YYYY HH:mm:ss ZZ');

    for (let i = 0; i < this.reports.length; i++) {
      const report = this.reports[i];
      report.info.specs[0] = report.info.specs[0] || '';
      const nameSplit = report.info.specs[0].split('/');
      const shortFileName = nameSplit[nameSplit.length - 1];

      // console.log('File ' + shortFileName)

      const filesFound = report && report.info && report.info.config && report.info.config.capabilities;
      if (!filesFound) {
        throw new Error('No files found, halting report generation!');
      }

      const reportSummaryObject = {
        metrics: report.metrics,
        title: report.title,
        browserName: '',
        browserVersion: '',
        platformName: '',
        sanitizedCapabilities: report.info.sanitizedCapabilities,
        capabilityNumber: report.info.config.capabilities['teal:capabilityNumber'] || '',
        failedTests: [],
        otherNonPasses: [],
        cid: report.info.cid,
        // browserNumber: report.info.capabilities['teal:capabilityNumber'] || "?",
        hasPassedTests: (report.metrics && report.metrics.passed > 0),
        hasSkippedTests: (report.metrics && report.metrics.skipped > 0),
        hasFailedTests: (report.metrics && report.metrics.failed > 0),
        shortFileName: shortFileName,
        reportLink: report.fileLink.replace('.json', '.html') // the files are right next to each other
      };

      if (report.info.capabilities) {
        const cap = report.info.capabilities;
        reportSummaryObject.browserName = cap.browserName || '';
        reportSummaryObject.browserVersion = cap.browserVersion || cap.version || '';
        reportSummaryObject.platformName = cap.platformName || cap.platform || '';
      }

      this.capabilities = results.capabilities.map(function (cap, index) {
        cap.browserNumber = index;
        return cap;
      });

      // reportSummaryObject.thisAsString = JSON.stringify(reportSummaryObject, null, 2)
      for (let j = 0; j < report.suites.length; j++) {
        for (let k = 0; k < report.suites[j].tests.length; k++) {
          const test = report.suites[j].tests[k];
          const testObject = {
            title: test.fullTitle,
            state: test.state
          };
          if (test.state === 'passed') continue;
          if (test.state === 'failed') {
            reportSummaryObject.failedTests.push(testObject);
            continue;
          }
          reportSummaryObject.otherNonPasses.push(testObject);
        }
      }

      this.specSummaries = this.specSummaries || {};
      this.specSummaries[shortFileName] = this.specSummaries[shortFileName] || {};

      this.specSummaries[shortFileName].summaries = this.specSummaries[shortFileName].summaries || [];

      this.specSummaries[shortFileName].testTitle = report.info.config.capabilities['teal:testTitle'] || false;
      this.specSummaries[shortFileName].testContext = report.info.config.capabilities['teal:testContext'] || false;

      this.specSummaries[shortFileName].summaries.push(reportSummaryObject);
    }

    metrics.passedPercentage = metrics.passed / metrics.total;
    metrics.failedPercentage = metrics.failed / metrics.total;
    metrics.skippedPercentage = metrics.skipped / metrics.total;

    const reportOptions = {
      data: {
        info: (this.reports[0] && this.reports[0].info) || {},
        specs: specs,
        metrics: metrics,
        suites: suites,
        title: this.options.reportTitle,
        // reports: this.reports,
        specSummaries: this.specSummaries,
        capabilities: this.capabilities
      },
      outputDir: this.options.outputDir,
      reportFile: this.options.reportFile,
      templateFilename: this.options.templateFilename,
      LOG: this.options.LOG,
      templateFuncs: this.options.templateFuncs,
      showInBrowser: this.options.showInBrowser
    };

    HtmlGenerator.htmlOutput(reportOptions);
  }
}

module.exports = ReportGenerator;