/* global rootDir */
'use strict';
const Promise = require('bluebird');
const fse = require('fs-extra');
const _ = require('lodash');
const rootDir = require('app-root-path').path;
const stepFilePath = `${rootDir}/ready/steps.json`;
const Logger = require(`${rootDir}/platform-helpers/logger.js`);
const logger = new Logger();
// experimental, see https://stackoverflow.com/questions/39538473/using-settimeout-on-promise-chain
function delay (t, v) {
return new Promise(function (resolve) {
setTimeout(resolve.bind(null, v), t);
});
}
/**
* Delays the return of Promise if chained after it (executes Promise immediately).
*
* @private
* @param {number} t time to delay chained Promise return (in milliseconds)
* @return {Promise} whatever promise is before it in the chain
*/
Promise.prototype.delay = function (t) {
return this.then(function (v) {
return delay(t, v);
});
};
/**
* [mitmproxy](https://mitmproxy.org/) helper functions.
*
* mitmproxy writes to a file in realtime, so you can just check the file whenever you want.
* @module proxyHelper
*/
// Stores a JSON object in a file to make sure it's persisted
exports.startNewStep = function (name) {
fse.ensureFileSync(stepFilePath);
let stepFileContents = fse.readFileSync(stepFilePath);
try {
stepFileContents = JSON.parse(stepFileContents);
} catch (e) {
stepFileContents = { stepsSoFar: 0 };
}
const startTime = new Date().toISOString();
stepFileContents.stepsSoFar++;
stepFileContents.stepInfo = stepFileContents.stepInfo || {};
stepFileContents.stepInfo[stepFileContents.stepsSoFar] = {
start: startTime,
name: name || ''
};
if (stepFileContents.stepsSoFar > 1) {
stepFileContents.stepInfo[stepFileContents.stepsSoFar - 1].end = startTime;
}
fse.writeFileSync(stepFilePath, JSON.stringify(stepFileContents, null, 2));
};
/**
* Represents network requests captured by [mitmproxy](https://mitmproxy.org/).
*
* Returned by [proxyHelper.getLogs]{@link module:proxyHelper.getLogs}
*
* @hideconstructor
*/
class ProxyLogs {
// no JSDoc here, only relevant for developers working on the tool
constructor (har, steps) {
/**
* The raw HAR log information from the network capture.
*
* @type {object}
* @example
*{
"log": {
"version": "1.2",
"creator": {
"name": "mitmproxy har_dump",
"version": "0.1",
"comment": "mitmproxy version mitmproxy 6.0.2"
},
"entries": [
{
"startedDateTime": "2021-01-25T13:29:55.020851+00:00",
"time": 640,
"request": {
"method": "GET",
"url": "https://tags.tiqcdn.com/utag/tiqapp/utag.v.js?a=tealium-solutions/test-example/202005291402&cb=1611581394205",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [
{
"name": "User-Agent",
"value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0"
},
{
"name": "Accept",
"value": "*\/*"
},
{
"name": "Accept-Language",
"value": "en-US,en;q=0.5"
},
{
"name": "Accept-Encoding",
"value": "gzip, deflate, br"
},
{
"name": "Referer",
"value": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html"
},
{
"name": "Host",
"value": "tags.tiqcdn.com"
},
{
"name": "Via",
"value": "1.1 maki231 (squid/4.6)"
},
{
"name": "Cache-Control",
"value": "max-age=0"
},
{
"name": "Connection",
"value": "keep-alive"
},
{
"name": "X-SL-Job-ID",
"value": "3e756637232241df8514810fc6da95dc"
},
{
"name": "X-SL-Tunnel-ID",
"value": "a985b541abf04f18a3ef447451c81fc1"
},
{
"name": "X-SL-Chef-IP",
"value": "10.113.8.9"
}
],
"queryString": [
{
"name": "a",
"value": "tealium-solutions/test-example/202005291402"
},
{
"name": "cb",
"value": "1611581394205"
}
],
"headersSize": 599,
"bodySize": 0
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": [
{
"name": "Accept-Ranges",
"value": "bytes"
},
{
"name": "Content-Type",
"value": "application/x-javascript"
},
{
"name": "ETag",
"value": "\"7bc0ee636b3b83484fc3b9348863bd22:1460653071\""
},
{
"name": "Last-Modified",
"value": "Thu, 14 Apr 2016 16:57:51 GMT"
},
{
"name": "Server",
"value": "AkamaiNetStorage"
},
{
"name": "Content-Length",
"value": "2"
},
{
"name": "Cache-Control",
"value": "max-age=600"
},
{
"name": "Expires",
"value": "Mon, 25 Jan 2021 13:39:55 GMT"
},
{
"name": "Date",
"value": "Mon, 25 Jan 2021 13:29:55 GMT"
},
{
"name": "Connection",
"value": "keep-alive"
}
],
"content": {
"size": 2,
"compression": 0,
"mimeType": "application/x-javascript",
"text": "//"
},
"redirectURL": "",
"headersSize": 422,
"bodySize": 2
},
"cache": {},
"timings": {
"send": 15,
"receive": 8,
"wait": 318,
"connect": 56,
"ssl": 243
}
}
]
}
}
*/
this.rawLogs = har;
let logs = har.log.entries.map((entry) => {
const match = entry.startedDateTime.match(/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}).*$/);
entry.startedDateTime = match[1] + 'Z'; // intentionally fragile, in case the log format changes this should break
entry.request.urlWithoutQueryString = entry.request.url.split('?')[0];
return entry;
});
logs = addFeaturesToLogs(logs);
/**
* An object with detailed network request info, listed per step.
*
* Each request will appear in the appropriate step array AND the allSteps array, to help make assertions simpler.
*
* Collect requests will also have more advanced payload parsing (parsed, see example.).
*
* In the interest of brevity, the example below doesn't show the same request in 'allSteps'. The 'Accept' headers in this example have been modified to avoid breaking the JSDoc comments.
*
* @type {object}
* @example
* {
"allSteps": [ '(omitted for brevity in this example)' ],
"step1": [{
"startedDateTime": "2021-01-28T13:04:06.982Z",
"time": 2225,
"request": {
"method": "POST",
"url": "https://collect-eu-central-1.tealiumiq.com/tealium-solutions/test-example/2/i.gif",
"httpVersion": "HTTP/1.1",
"cookies": [],
"headers": {
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:85.0) Gecko/20100101 Firefox/85.0",
"accept": "*\/*",
"accept-language": "en-US,en;q=0.5",
"accept-encoding": "gzip, deflate, br",
"content-type": "multipart/form-data; boundary=---------------------------1787141617783135763328970803",
"content-length": "1902",
"origin": "https://solutions.tealium.net",
"referer": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html",
"host": "collect-eu-central-1.tealiumiq.com",
"via": "1.1 maki3615 (squid/4.6)",
"cache-control": "max-age=900",
"connection": "keep-alive",
"x-sl-job-id": "6c043688c72a4b93b67246927cf7527a",
"x-sl-tunnel-id": "d85beb87f4d74aee9e1fefc7f0038523",
"x-sl-chef-ip": "10.129.1.183"
},
"queryString": {},
"headersSize": 813,
"bodySize": 1902,
"postData": {
"mimeType": "multipart/form-data; boundary=---------------------------1787141617783135763328970803",
"text": "-----------------------------1787141617783135763328970803\r\nContent-Disposition: form-data; name=\"data\"\r\n\r\n{\"loader.cfg\":{\"2\":{\"load\":4,\"send\":1,\"v\":202005291402,\"wait\":1,\"tid\":20064,\"id\":\"2\",\"executed\":1}},\"data\":{\"page_type\":\"first_test\",\"cp.utag_main_v_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"cp.utag_main__pn\":\"1\",\"cp.utag_main_ses_id\":\"1611839047103\",\"cp.utag_main__ss\":\"1\",\"cp.utag_main__se\":\"1\",\"cp.utag_main__sn\":\"1\",\"cp.utag_main__st\":\"1611840847103\",\"dom.referrer\":\"\",\"dom.title\":\"Integration Test\",\"dom.domain\":\"solutions.tealium.net\",\"dom.query_string\":\"\",\"dom.hash\":\"\",\"dom.url\":\"https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html\",\"dom.pathname\":\"/hosted/webdriver-testing/standard-integration-test.html\",\"dom.viewport_height\":671,\"dom.viewport_width\":1024,\"ut.domain\":\"tealium.net\",\"ut.version\":\"ut4.46.202005291402\",\"ut.event\":\"view\",\"ut.visitor_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"ut.session_id\":\"1611839047103\",\"ut.account\":\"tealium-solutions\",\"ut.profile\":\"test-example\",\"ut.env\":\"prod\",\"tealium_event\":\"first_test\",\"tealium_visitor_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"tealium_session_id\":\"1611839047103\",\"tealium_session_number\":\"1\",\"tealium_session_event_number\":\"1\",\"tealium_datasource\":\"7hpfk3\",\"tealium_account\":\"tealium-solutions\",\"tealium_profile\":\"test-example\",\"tealium_environment\":\"prod\",\"tealium_random\":\"7091299497493456\",\"tealium_library_name\":\"utag.js\",\"tealium_library_version\":\"4.46.0\",\"tealium_timestamp_epoch\":1611839047,\"tealium_timestamp_utc\":\"2021-01-28T13:04:07.106Z\",\"tealium_timestamp_local\":\"2021-01-28T13:04:07.106\",\"cp.utag_main_dc_visit\":\"1\",\"cp.utag_main_dc_event\":\"1\"},\"browser\":{\"height\":671,\"width\":1024,\"screen_height\":768,\"screen_width\":1024,\"timezone_offset\":0},\"event\":\"view\",\"post_time\":1611839047115}\r\n-----------------------------1787141617783135763328970803--\r\n",
"params": [],
"parsed": {
"loader.cfg": {
"2": {
"load": 4,
"send": 1,
"v": 202005291402,
"wait": 1,
"tid": 20064,
"id": "2",
"executed": 1
}
},
"data": {
"page_type": "first_test",
"cp.utag_main_v_id": "0177491805bd00100897c23c7fad00052005500f00718",
"cp.utag_main__pn": "1",
"cp.utag_main_ses_id": "1611839047103",
"cp.utag_main__ss": "1",
"cp.utag_main__se": "1",
"cp.utag_main__sn": "1",
"cp.utag_main__st": "1611840847103",
"dom.referrer": "",
"dom.title": "Integration Test",
"dom.domain": "solutions.tealium.net",
"dom.query_string": "",
"dom.hash": "",
"dom.url": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html",
"dom.pathname": "/hosted/webdriver-testing/standard-integration-test.html",
"dom.viewport_height": 671,
"dom.viewport_width": 1024,
"ut.domain": "tealium.net",
"ut.version": "ut4.46.202005291402",
"ut.event": "view",
"ut.visitor_id": "0177491805bd00100897c23c7fad00052005500f00718",
"ut.session_id": "1611839047103",
"ut.account": "tealium-solutions",
"ut.profile": "test-example",
"ut.env": "prod",
"tealium_event": "first_test",
"tealium_visitor_id": "0177491805bd00100897c23c7fad00052005500f00718",
"tealium_session_id": "1611839047103",
"tealium_session_number": "1",
"tealium_session_event_number": "1",
"tealium_datasource": "7hpfk3",
"tealium_account": "tealium-solutions",
"tealium_profile": "test-example",
"tealium_environment": "prod",
"tealium_random": "7091299497493456",
"tealium_library_name": "utag.js",
"tealium_library_version": "4.46.0",
"tealium_timestamp_epoch": 1611839047,
"tealium_timestamp_utc": "2021-01-28T13:04:07.106Z",
"tealium_timestamp_local": "2021-01-28T13:04:07.106",
"cp.utag_main_dc_visit": "1",
"cp.utag_main_dc_event": "1"
},
"browser": {
"height": 671,
"width": 1024,
"screen_height": 768,
"screen_width": 1024,
"timezone_offset": 0
},
"event": "view",
"post_time": 1611839047115
}
},
"urlWithoutQueryString": "https://collect-eu-central-1.tealiumiq.com/tealium-solutions/test-example/2/i.gif"
},
"response": {
"status": 200,
"statusText": "OK",
"httpVersion": "HTTP/1.1",
"cookies": [
{
"name": "TAPID",
"value": "tealium-solutions/test-example>0177491805bd00100897c23c7fad00052005500f00718|",
"path": "/",
"domain": ".tealiumiq.com",
"httpOnly": true,
"secure": true,
"expires": "2022-01-28T13:04:08+00:00"
}
],
"headers": {
"date": "Thu, 28 Jan 2021 13:04:08 GMT",
"content-type": "image/gif",
"content-length": "43",
"connection": "keep-alive",
"x-acc": "tealium-solutions:test-example:2:datacloud",
"x-did": "0177491805bd00100897c23c7fad00052005500f00718",
"x-region": "eu-central-1",
"access-control-allow-origin": "https://solutions.tealium.net",
"x-serverid": "uconnect_i-06054a44183803790",
"pragma": "no-cache",
"p3p": "policyref=\"/w3c/p3p.xml\", CP=\"NOI DSP COR NID CUR ADM DEV OUR BUS\"",
"access-control-expose-headers": "X-Region",
"cache-control": "no-transform,private,no-cache,no-store,max-age=0,s-maxage=0",
"x-tid": "0177491805bd00100897c23c7fad00052005500f00718",
"access-control-allow-credentials": "true",
"x-ulver": "ed533b75a08fa8f6edbe6695d0295a01b07dd99c-SNAPSHOT",
"vary": "Origin",
"expires": "Thu, 28 Jan 2021 13:04:08 GMT",
"x-uuid": "24ba2778-6a42-412b-9dbc-94dacff69bed",
"set-cookie": "TAPID=tealium-solutions/test-example>0177491805bd00100897c23c7fad00052005500f00718|; Path=/; Domain=.tealiumiq.com; Expires=Fri, 28-Jan-2022 13:04:08 GMT; Max-Age=31536000; Secure; HttpOnly; SameSite=None"
},
"content": {
"size": 43,
"compression": 0,
"mimeType": "image/gif",
"text": "R0lGODlhAQABAPAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==",
"encoding": "base64"
},
"redirectURL": "",
"headersSize": 1184,
"bodySize": 43
},
"cache": {},
"timings": {
"send": 121,
"receive": 33,
"wait": 1061,
"connect": 831,
"ssl": 179
},
"stepNumber": 1,
"stepName": "step1"
}]
}
*/
this.logs = sortLogsByStep(logs, steps);
/**
* The steps that were recognized and used to split the network capture.
*
* @example
{
"stepsSoFar": 4,
"stepInfo": {
"1": {
"start": "2021-01-25T13:29:53.042Z",
"name": "STEP 1 - initial page visit, set (and verify) Trace cookie, confirm some globals and helpers",
"end": "2021-01-25T13:29:59.378Z"
},
"2": {
"start": "2021-01-25T13:29:59.378Z",
"name": "STEP 2 - reload the page, then increment the counter 4 times",
"end": "2021-01-25T13:30:23.820Z"
},
"3": {
"start": "2021-01-25T13:30:23.820Z",
"name": "STEP 3 - decrement the counter 5 times",
"end": "2021-01-25T13:30:45.128Z"
},
"4": {
"start": "2021-01-25T13:30:45.128Z",
"name": "STEP 4 - spoof a login and file import, setting the counter to 42"
}
}
}
*/
this.steps = steps;
/**
* Returns a filtered subset of the provided 'logs' object (based on the request URL)
*
* Expects double-escaping because of the string conversion, like `\\w+`,
* see [doc](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/RegExp)
*
* @param {string} filterStringForRegex A double-escaped string that will be used to create a RegExp, uses as a filter
* @returns A filtered 'logs' object with only matching entries, as in [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
* @example
* it('should find a single TiQ session counter, in the first step', function () {
* chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
* chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
* })
*/
this.getFilteredLogs = function (filterStringForRegex) {
const filtered = {};
const logObject = this.logs;
// make sure all steps have an array, even if it's empty
const highestStep = this.steps.stepsSoFar;
for (let i = 1; i <= highestStep; i++) {
filtered[`step${i}`] = [];
}
filtered.allSteps = [];
const re = new RegExp(filterStringForRegex);
logObject.allSteps.forEach((logEntry) => {
const key = logEntry.stepName;
const match = re.test(logEntry.request.url);
const isSslHandshake = /.*:443$/.test(logEntry.request.url);
if (match && !isSslHandshake) {
filtered[key].push(logEntry);
filtered.allSteps.push(logEntry);
}
});
return filtered;
};
function sortLogsByStep (inputLogArray, steps) {
// make a deep copy
const grouped = {};
const logArray = _.cloneDeep(inputLogArray);
const renamed = {};
// ensure all stteps are defined with empty arrays at least
renamed.allSteps = [];
for (let i = 1; i <= steps.stepsSoFar; i++) {
renamed[`step${i}`] = [];
}
function findStepNumber (entry) {
const entryTime = Date.parse(entry.startedDateTime);
// stop short of the last one
for (let i = 1; i < steps.stepsSoFar; i++) {
const step = steps.stepInfo[i];
const start = Date.parse(step.start);
const end = Date.parse(step.end);
const inRange = entryTime >= start && entryTime < end;
if (inRange) {
return i;
}
}
// greater or equal to the last step start means it's part of the last step
if (entryTime >= Date.parse(steps.stepInfo[steps.stepsSoFar].start)) {
return steps.stepsSoFar;
}
// otherwise, it must be stuff like browser setup before the start of the first step, put that as part of the first step
return 1;
}
logArray.forEach((logEntry, i) => {
const step = findStepNumber(logEntry);
logEntry.stepNumber = step;
logEntry.stepName = `step${logEntry.stepNumber}`;
grouped[step] = grouped[step] || [];
grouped[step].push(logEntry); // whole entry for now.
});
const sortedStepIds = Object.keys(grouped);
sortedStepIds.forEach((id, index) => {
const stepNumber = index + 1;
const stepName = 'step' + stepNumber;
renamed[stepName] = grouped[id];
renamed[stepName].forEach((stepEntry) => {
renamed.allSteps.push(stepEntry);
});
});
return renamed;
}
function addFeaturesToLogs (inputLogArray) {
// avoid side effects with a deep copy
const logArray = _.cloneDeep(inputLogArray);
logArray.forEach((entry, i) => {
// reformat into more useful objects (and lowercase the keys)
// [{ propName: 'Protocol', propValue: 'https' }] becomes
// { 'protocol': 'https'}
entry.request.headers = reformatObject(entry.request.headers);
entry.response.headers = reformatObject(entry.response.headers);
entry.request.queryString = reformatObject(entry.request.queryString);
const isMultipartFormData = entry.request.headers['content-type'] && entry.request.headers['content-type'].indexOf('multipart/form-data;') !== -1;
const isCollect = isMultipartFormData && /^https?:\/\/collect.*\/2\/i\.gif$/.test(entry.request.url);
// hack to parse the payload for the Collect tag specifically, ignoring other tags for now because mulitpart form data is
// hard to parse generally (and all the libraries expect buffers, received from file uploads) - Collect forms are pretty simple
if (isCollect) {
const boundary = entry.request.headers['content-type'].split('boundary=')[1];
const body = entry.request.postData.text;
const lines = body.split('\r\n');
let collectBody;
lines.forEach((line) => {
if (line.indexOf(boundary) === -1 && /^{.*}$/.test(line)) {
try {
collectBody = JSON.parse(line);
} catch (e) {
collectBody = { failed: true };
}
}
});
entry.request.postData.parsed = collectBody;
}
// Add /events endpoint parsing as well for POST requests
if (/^https:\/\/collect\.tealiumiq\.com\/event[/]?$/.test(entry.request.url)) {
try {
entry.request.postData.parsed = JSON.parse(entry.request.postData.text);
} catch (e) {
console.log(`Failed to parse ${entry.request.postData.text}\n\n${e}`);
}
}
logArray[i] = entry;
});
return logArray;
}
/**
* Reformat objects to make them easier to use.
*
* Input:
* [
* {
* 'name': 'Status',
* 'value': '200'
* }
* ]
*
* Output:
* {
* 'Status': '200'
* }
* @private
* @param {array} inputArray an array of objects with keys 'propName' and 'propValue'
* @returns {object} the reformatted array in classic key/value style
*/
function reformatObject (inputArray) {
const outputObject = {};
inputArray.forEach((entry) => {
outputObject[entry.name.toLowerCase()] = entry.value;
});
return outputObject;
}
}
}
// export for tests
exports.ProxyLogs = ProxyLogs;
// const reporter = rootRequire('platform-helpers/reporter-helper.js')
/**
* Retrieves and parses the network request logs captured by [mitmproxy](https://mitmproxy.org/).
*
* @public
* @returns {Promise}
* @yields instance of [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
* @deprecated This method can still be used but its being refactored. Alternatively start using the new method "require(./mitmproxy-helper).getLogsV2()"
* @example
// Define this variable outside the tests to allow us to split the assertions and the retrieval itself into different steps.
let proxyLogs
describe('FINISH - retrieve, validate, and check the proxy logs for for the run', async function () {
it('should fetch network logs (from mitmproxy in the Docker container)', function () {
// this helper returns a promise which resolves into the proxy output for the run
proxyLogs = await proxyHelper.getLogs()
})
// now you have an instance of ProxyLogs
it('should have the expected properties on the returned ProxyLogs object', async function () {
// logs
chai.expect(proxyLogs.logs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
chai.expect(proxyLogs.logs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
// raw logs
chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
// steps
chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepsSoFar').that.is.a('number').greaterThan(0)
chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepInfo').that.is.an('object').with.property('1').that.is.an('object').with.keys(['name', 'start', 'end'])
// getFilteredLogs function
chai.expect(proxyLogs.getFilteredLogs).to.be.an('function')
})
it('should find a single TiQ session counter, in the first step', async function () {
chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
})
})
*/
exports.getLogs = function () {
return new Promise(function (resolve, reject) {
console.log('[Proxy] Converting captured files...');
fse.removeSync(`${rootDir}/ready/logs.har`);
const { exec } = require('child_process');
exec('docker compose exec -T sauce_connect mitmdump -n -r ready/logs.mitm -s scripts/har_dump.py --set hardump=ready/logs.har', (error, stdout, stderr) => {
if (error) {
reject(error);
}
// console.log(`stdout: ${stdout}`)
if (stderr) {
console.error(`stderr: ${stderr}`);
}
const output = fse.readJsonSync(`${rootDir}/ready/logs.har`);
const steps = fse.readJsonSync(`${rootDir}/ready/steps.json`);
const proxyLogs = new ProxyLogs(output, steps);
logger.log('proxy_logs', proxyLogs, Logger.Outputs.FILE);
resolve(proxyLogs);
});
});
};
/**
* Retrieves and parses the network request logs captured by [mitmproxy](https://mitmproxy.org/).
*
* @public
* @returns {Promise}
* @yields instance of [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
* @example
// Define this variable outside the tests to allow us to split the assertions and the retrieval itself into different steps.
let proxyLogs
describe('FINISH - retrieve, validate, and check the proxy logs for for the run', async function () {
it('should fetch network logs (from mitmproxy in the Docker container)', function () {
// this helper returns a promise which resolves into the proxy output for the run
proxyLogs = await proxyHelper.getLogsV2()
})
// now you have an instance of ProxyLogs
it('should have the expected properties on the returned ProxyLogs object', async function () {
// logs
chai.expect(proxyLogs.logs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
chai.expect(proxyLogs.logs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
// raw logs
chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
// steps
chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepsSoFar').that.is.a('number').greaterThan(0)
chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepInfo').that.is.an('object').with.property('1').that.is.an('object').with.keys(['name', 'start', 'end'])
// getFilteredLogs function
chai.expect(proxyLogs.getFilteredLogs).to.be.an('function')
})
it('should find a single TiQ session counter, in the first step', async function () {
chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
})
})
*/
exports.getLogsV2 = async function () {
console.log('[Proxy] Converting captured files...');
fse.removeSync(`${rootDir}/ready/logs.har`);
const util = require('util');
const exec = util.promisify(require('child_process').exec);
try {
await exec('docker compose exec -T sauce_connect mitmdump -n -r ready/logs.mitm -s scripts/har_dump.py --set hardump=ready/logs.har');
const output = fse.readJsonSync(`${rootDir}/ready/logs.har`);
const steps = fse.readJsonSync(`${rootDir}/ready/steps.json`);
const proxyLogs = new ProxyLogs(output, steps);
return proxyLogs;
} catch (error) {
console.error(`stderr: ${error}`);
}
};
/**
* Adds Trace cookie writing to the provided extraProxyCommands, only meant to be used internally.
*
* Rudimentary approach so far - we
*
* @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
* @param {string} traceAccount - the Tealium CDH account associated with the traceId
* @param {string} traceProfile - the Tealium CDH account associated with the traceId
* @param {string} traceId - the traceId to add to outgoing requests from the proxy
* @returns {string} the provided extraProxyCommands (if any) plus the necessary Trace cookie proxy commands
* @private
*/
exports.addTraceCookie = function (extraProxyCommands, traceAccount, traceProfile, traceId) {
if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';
if (traceAccount && traceProfile && traceId) {
// lots of escaping, but it's correct to make the quotes work
const collectTagBodyRewrite = `--modify-body @~q@"tealium_account@"cp.trace_id":"${traceId}","tealium_account`.replace(/"/g, '\\"');
const pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http
def request(flow: http.HTTPFlow) -> None:
if flow.request.pretty_url.startswith("https://datacloud.tealiumiq.com/vdata/i.gif") and flow.request.query["tealium_account"] == "${traceAccount}" and flow.request.query["tealium_profile"] == "${traceProfile}":
flow.request.query["tealium_trace_id"] = "${traceId}"
`;
fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/auto_trace_cookie.py`, pythonScript);
extraProxyCommands = `${collectTagBodyRewrite} -s scripts/auto_trace_cookie.py ${extraProxyCommands}`;
}
return extraProxyCommands;
};
/**
* Adds request blocking functionality, meant to be used internally.
*
* @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
* @param {array} domainList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties
* @returns {string} the provided extraProxyCommands (if any) plus any necessary request blocking proxy commands
* @private
*/
exports.addRequestBlocking = function (extraProxyCommands, domainList) {
if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';
let foundValidEntry = false;
let pythonScript = `"""auto-generated in wdio.conf.js, sends a reply from the proxy without sending any data to the remote server for specified urls"""
from mitmproxy import http
import re
def request(flow: http.HTTPFlow) -> None:
`;
domainList = domainList || [];
domainList.forEach(function (obj, i) {
if (typeof obj.pattern !== 'string') return;
if (obj.type !== 'string' && obj.type !== 'regex') return;
obj.responseCode = obj.responseCode || 403;
obj.responseBody = obj.responseBody || 'b"Request blocked by the Tealium testing tool."';
obj.responseContentType = obj.responseContentType || 'text/html';
foundValidEntry = true;
let filterConditional = '';
if (obj.type === 'string') {
filterConditional = ` if '${obj.pattern}' in flow.request.pretty_url:`;
} else {
filterConditional = ` pattern = re.compile("${obj.pattern}")
if re.search(pattern, flow.request.pretty_url):`;
}
pythonScript += `${filterConditional}
flow.response = http.Response.make(
${obj.responseCode},
${obj.responseBody},
{"Content-Type": "${obj.responseContentType}"}
)
`;
});
if (foundValidEntry) {
fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/block_requests.py`, pythonScript);
extraProxyCommands = `-s scripts/block_requests.py ${extraProxyCommands}`;
}
return extraProxyCommands;
};
/**
* Adds URL rewriting functionality, intended to be used for internal functionality
*
* @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
* @param {array} rewritesList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties, as well as (string) 'target' and 'replacement' keys
* @returns {string} the provided extraProxyCommands (if any) plus the necessary url-rewriting proxy commands
* @private
*/
exports.addUrlRewrites = function (extraProxyCommands, rewritesList) {
if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';
let pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http
import re
def request(flow: http.HTTPFlow) -> None:
`;
let foundValidEntry = false;
rewritesList = rewritesList || [];
rewritesList.forEach(function (obj, i) {
if (typeof obj.pattern !== 'string') return;
if (obj.type !== 'string' && obj.type !== 'regex') return;
if (typeof obj.target !== 'string') return;
if (typeof obj.replacement !== 'string') return;
foundValidEntry = true;
let filterConditional = '';
if (obj.type === 'string') {
filterConditional = ` if '${obj.pattern}' in flow.request.pretty_url:`;
} else {
filterConditional = ` pattern = re.compile("${obj.pattern}")
if re.search(pattern, flow.request.pretty_url):`;
}
pythonScript += `${filterConditional}
flow.request.url = flow.request.url.replace("${obj.target}", "${obj.replacement}")
# flow.request.pretty_url = flow.request.pretty_url.replace("${obj.target}", "${obj.replacement}")
`;
});
if (foundValidEntry) {
fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/url_rewrites.py`, pythonScript);
extraProxyCommands = `-s scripts/url_rewrites.py ${extraProxyCommands}`;
}
return extraProxyCommands;
};