/**
* Report Generator functions. Helps add context to the generated HTML reports of the test results.
*
* NOTE: These methods only work inside tests!
* @module ReportGenerator
*/
'use strict';
const showdown = require('showdown').setOption('tables', 'true');
const converter = new showdown.Converter();
const Moment = require('moment');
const HtmlGenerator = require('./wdio-html-generator.js');
const rootRequire = require('rpcm-root-require');
const fse = require('fs-extra');
const log4js = require('log4js');
const Utils = rootRequire('/platform-helpers/utils');
const logger = log4js.getLogger();
rootRequire('/platform-helpers/string-extensions');
/**
* Used for our summary report generation. It's basically the same as the Class of the same name within the html-reporter package that we use for the individual reports, but with some tweaks to make it work the way we need it to.
*
* @private
* @class
*/
class ReportGenerator {
constructor (opts) {
// make sure provided arguments that are invalid get deleted otherwise they override defaults (skip booleans)
Object.keys(opts).forEach(key => { if (typeof opts[key] === 'boolean') return; if (key) delete opts[key]; });
// merge provided arguments with defaults. defaults get overrided
this.options = Object.assign({}, {
outputDir: 'reports/html-reports/',
fileName: 'index.html',
summaryReportTitle: 'Summary Report',
reportTitle: 'Summary Report',
showInBrowser: false,
templateFilename: '{0}/report-templates/aggregated-template.hbs'.format(rootRequire.rootPath),
templateFuncs: {},
browserName: 'not specified',
LOG: logger,
reportContext: converter.makeHtml(('This report has been generated by Tealium\'s automated testing platform.{0}' +
'These tests will have been created specifically for your project by a Tealium Implementation Engineer to verify a configuration or use case.{0}' +
'The intent is to document the tests run on the completed configuration (acceptance tests). It can also serve as a limited regression test as work continues, to ensure that future projects don\'t break previous ones.{1}' +
'For more information, see the [developer documentation](https://tealium.github.io/customer-success-testing-platform/).')
.format(Utils.getNewLine(2), Utils.getNewLine()))
}, opts);
this.options.reportFile = '{0}/{1}{2}'.format(rootRequire.rootPath, this.options.outputDir, this.options.fileName);
this.startTime = new Moment();
this.reports = [];
}
calculateDuration (startTime, endTime) {
const isMoment = (moment) => {
return (moment && typeof moment.diff === 'function');
};
if (isMoment(startTime) && isMoment(endTime)) {
return endTime.diff(startTime);
}
}
moveFilesBetweenRuns (index) {
try {
// move the files to the 'runs' folder
const allJsonPaths = Utils.getFilesRecursively(this.options.outputDir, ['.json', '.html']);
const folderPaths = [];
allJsonPaths.forEach((path) => {
const re = /\/runs\//;
const alreadyMoved = re.test(path);
const pathParts = path.split('/');
const fileName = pathParts[pathParts.length - 1];
const folderPath = pathParts.slice(0, pathParts.length - 2).join('/');
// don't try to move the files we already moved (which are in the same output dir)
if (!alreadyMoved) {
fse.moveSync(path, `${this.options.outputDir}/runs/${index}/${fileName}`);
folderPaths.push(folderPath);
}
});
folderPaths.forEach((folderPath) => {
// wait for both the .json and the .html to be moved before we delete the folder
// will fail silently when we try to remove the same file twice - that's fine.
fse.removeSync(folderPath);
});
} catch (e) {}
}
clean () {
// leave the output dir intact, but empty it
fse.emptyDirSync(this.options.outputDir);
// remove the entire 'tmp' folder on start if present (should have been moved already anyway, if the previous report completed successfully)
fse.removeSync('tmp');
}
readJsonFiles () {
return Utils.getFilesRecursively(this.options.outputDir, ['.json']);
}
log (message, object) {
if (this.options.LOG) {
this.options.LOG.debug(message + object);
}
}
async createReport (results) {
const metrics = {
passed: 0,
skipped: 0,
failed: 0,
total: 0,
start: this.startTime,
end: 0,
duration: 0
};
const suites = [];
const specs = [];
const files = this.readJsonFiles();
// copy the static JS and CSS assets into the report output folder
const staticSource = 'report-templates/static-assets';
const staticTarget = '{0}/static-assets'.format(this.options.outputDir);
fse.ensureDirSync(staticTarget);
fse.copySync(staticSource, staticTarget);
// move the 'logs' folder into the report output folder, to keep it all together for debugging
fse.ensureDirSync('tmp/logs');
fse.copySync('tmp/logs', '{0}/logs'.format(this.options.outputDir));
fse.ensureDirSync('tmp/tests');
fse.copySync('tmp/tests', '{0}/tests'.format(this.options.outputDir), { overwrite: true });
fse.removeSync('tmp');
const fileList = {};
files.forEach((fileName) => {
// keep track of the capabilities we've seen, so we can do something about the missing ones
const file = JSON.parse(fse.readFileSync(fileName));
const capNumber = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities['teal:capabilityNumber']) || 0;
const specPath = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities['teal:specPath']) || '';
const browserName = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.browserName) || '';
const browserVersion = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.browserVersion) || '';
const platformName = (file && file.info && file.info.config && file.info.config.capabilities && file.info.config.capabilities.platformName) || '';
const key = capNumber + ' | ' + specPath;
fileList[key] = {
browser: browserName,
version: browserVersion,
platform: platformName,
reportFile: fileName,
spec: specPath,
capNumber: capNumber
};
});
const fullRunList = JSON.parse(results['teal:fullRunList']);
fullRunList.forEach((run) => {
const capNumber = run.capability['teal:capabilityNumber'];
const specPath = run.spec;
if (specPath + capNumber) {
const key = capNumber + ' | ' + specPath;
if (typeof fileList[key] === 'undefined') {
fileList[key] = {
browser: run.capability.browserName,
version: run.capability.browserVersion,
platform: run.capability.platformName,
reportFile: '',
spec: specPath,
capNumber: capNumber
};
}
}
});
// push reports or placeholders for each run
const allRuns = Object.keys(fileList);
// sort by capability number
allRuns.sort((a, b) => {
const aCapNumber = parseInt(a.split(' | ')[0], 10);
const bCapNumber = parseInt(b.split(' | ')[0], 10);
if (aCapNumber < bCapNumber) {
return -1;
} else if (aCapNumber > bCapNumber) {
return 1;
} else {
return 0;
}
});
for (let i = 0; i < allRuns.length; i++) {
try {
let report;
const key = allRuns[i];
const fileName = fileList[key].reportFile;
// if the file name is blank, that means the run failed (generate stub report for summary)
if (typeof fileName === 'undefined' || fileName === '') {
const dummyReport = {
fileLink: '',
metrics: {
passed: 0,
failed: 1,
skipped: 0
},
info: {
specs: [
fileList[key].spec
],
config: {
capabilities: [{
browserName: fileList[key].browser,
browserVersion: fileList[key].version,
platformName: fileList[key].platform,
'teal:capabilityNumber': fileList[key].capNumber
}]
},
// it's weird that this is plural but not an array, but that's how it is elsewhere
capabilities: {
browserName: fileList[key].browser,
browserVersion: fileList[key].version,
platformName: fileList[key].platformName,
'teal:capabilityNumber': fileList[key].capNumber
}
},
suites: []
};
report = dummyReport;
// console.log(dummyReport)
this.reports.push(dummyReport);
} else {
// run succeeded, generate the full report
report = JSON.parse(fse.readFileSync(fileName));
// remove the dependency on the parent folders for functional links in the summary report
report.fileLink = fileName.replace('reports/html-reports/', '').replace('reports\\html-reports\\', '');
report.info.specs.forEach(spec => {
specs.push(spec);
});
this.reports.push(report);
}
metrics.passed += report.metrics.passed;
metrics.total += report.metrics.passed;
metrics.failed += report.metrics.failed;
metrics.total += report.metrics.failed;
metrics.skipped += report.metrics.skipped;
metrics.total += report.metrics.skipped;
for (let k = 0; k < report.suites.length; k++) {
const suite = report.suites[k];
const start = new Moment(suite.start);
if (start.isSameOrBefore(metrics.start)) {
// metrics.start = start
}
const end = new Moment(suite.end);
if (end.isAfter(metrics.end)) {
// metrics.end = end
}
suites.push(suite);
}
} catch (ex) {
console.error(ex);
}
}
metrics.end = new Moment();
const duration = this.calculateDuration(metrics.start, metrics.end);
metrics.duration = duration;
metrics.prettyStart = metrics.start.format('ddd MMMM Do YYYY HH:mm:ss ZZ');
metrics.prettyEnd = metrics.end.format('ddd MMMM Do YYYY HH:mm:ss ZZ');
for (let i = 0; i < this.reports.length; i++) {
const report = this.reports[i];
report.info.specs[0] = report.info.specs[0] || '';
const nameSplit = report.info.specs[0].split('/');
const shortFileName = nameSplit[nameSplit.length - 1];
// console.log('File ' + shortFileName)
const filesFound = report && report.info && report.info.config && report.info.config.capabilities;
if (!filesFound) {
throw new Error('No files found, halting report generation!');
}
const reportSummaryObject = {
metrics: report.metrics,
title: report.title,
browserName: '',
browserVersion: '',
platformName: '',
sanitizedCapabilities: report.info.sanitizedCapabilities,
capabilityNumber: report.info.config.capabilities['teal:capabilityNumber'] || '',
failedTests: [],
otherNonPasses: [],
cid: report.info.cid,
// browserNumber: report.info.capabilities['teal:capabilityNumber'] || "?",
hasPassedTests: (report.metrics && report.metrics.passed > 0),
hasSkippedTests: (report.metrics && report.metrics.skipped > 0),
hasFailedTests: (report.metrics && report.metrics.failed > 0),
shortFileName: shortFileName,
reportLink: report.fileLink.replace('.json', '.html') // the files are right next to each other
};
if (report.info.capabilities) {
const cap = report.info.capabilities;
reportSummaryObject.browserName = cap.browserName || '';
reportSummaryObject.browserVersion = cap.browserVersion || cap.version || '';
reportSummaryObject.platformName = cap.platformName || cap.platform || '';
}
this.capabilities = results.capabilities.map(function (cap, index) {
cap.browserNumber = index;
return cap;
});
// reportSummaryObject.thisAsString = JSON.stringify(reportSummaryObject, null, 2)
for (let j = 0; j < report.suites.length; j++) {
for (let k = 0; k < report.suites[j].tests.length; k++) {
const test = report.suites[j].tests[k];
const testObject = {
title: test.fullTitle,
state: test.state
};
if (test.state === 'passed') continue;
if (test.state === 'failed') {
reportSummaryObject.failedTests.push(testObject);
continue;
}
reportSummaryObject.otherNonPasses.push(testObject);
}
}
this.specSummaries = this.specSummaries || {};
this.specSummaries[shortFileName] = this.specSummaries[shortFileName] || {};
this.specSummaries[shortFileName].summaries = this.specSummaries[shortFileName].summaries || [];
this.specSummaries[shortFileName].testTitle = report.info.config.capabilities['teal:testTitle'] || false;
this.specSummaries[shortFileName].testContext = report.info.config.capabilities['teal:testContext'] || false;
this.specSummaries[shortFileName].summaries.push(reportSummaryObject);
}
metrics.passedPercentage = metrics.passed / metrics.total;
metrics.failedPercentage = metrics.failed / metrics.total;
metrics.skippedPercentage = metrics.skipped / metrics.total;
const reportOptions = {
data: {
info: (this.reports[0] && this.reports[0].info) || {},
specs: specs,
metrics: metrics,
suites: suites,
title: this.options.reportTitle,
// reports: this.reports,
specSummaries: this.specSummaries,
capabilities: this.capabilities
},
outputDir: this.options.outputDir,
reportFile: this.options.reportFile,
templateFilename: this.options.templateFilename,
LOG: this.options.LOG,
templateFuncs: this.options.templateFuncs,
showInBrowser: this.options.showInBrowser
};
HtmlGenerator.htmlOutput(reportOptions);
}
}
module.exports = ReportGenerator;