mitmproxy-helper.js

/* global rootDir */
'use strict';

const Promise = require('bluebird');
const fse = require('fs-extra');
const _ = require('lodash');
const rootDir = require('app-root-path').path;

const stepFilePath = `${rootDir}/ready/steps.json`;
const Logger = require(`${rootDir}/platform-helpers/logger.js`);

const logger = new Logger();

// experimental, see https://stackoverflow.com/questions/39538473/using-settimeout-on-promise-chain
function delay (t, v) {
  return new Promise(function (resolve) {
    setTimeout(resolve.bind(null, v), t);
  });
}
/**
 * Delays the return of Promise if chained after it (executes Promise immediately).
 *
 * @private
 * @param {number} t time to delay chained Promise return (in milliseconds)
 * @return {Promise} whatever promise is before it in the chain
 */
Promise.prototype.delay = function (t) {
  return this.then(function (v) {
    return delay(t, v);
  });
};

/**
 * [mitmproxy](https://mitmproxy.org/) helper functions.
 *
 * mitmproxy writes to a file in realtime, so you can just check the file whenever you want.
 * @module proxyHelper
 */

// Stores a JSON object in a file to make sure it's persisted
exports.startNewStep = function (name) {
  fse.ensureFileSync(stepFilePath);
  let stepFileContents = fse.readFileSync(stepFilePath);
  try {
    stepFileContents = JSON.parse(stepFileContents);
  } catch (e) {
    stepFileContents = { stepsSoFar: 0 };
  }

  const startTime = new Date().toISOString();

  stepFileContents.stepsSoFar++;
  stepFileContents.stepInfo = stepFileContents.stepInfo || {};
  stepFileContents.stepInfo[stepFileContents.stepsSoFar] = {
    start: startTime,
    name: name || ''
  };

  if (stepFileContents.stepsSoFar > 1) {
    stepFileContents.stepInfo[stepFileContents.stepsSoFar - 1].end = startTime;
  }

  fse.writeFileSync(stepFilePath, JSON.stringify(stepFileContents, null, 2));
};

/**
 * Represents network requests captured by [mitmproxy](https://mitmproxy.org/).
 *
 * Returned by [proxyHelper.getLogs]{@link module:proxyHelper.getLogs}
 *
 * @hideconstructor
*/
class ProxyLogs {
  // no JSDoc here, only relevant for developers working on the tool
  constructor (har, steps) {
    /**
     * The raw HAR log information from the network capture.
     *
     * @type {object}
     * @example
     *{
  "log": {
    "version": "1.2",
    "creator": {
      "name": "mitmproxy har_dump",
      "version": "0.1",
      "comment": "mitmproxy version mitmproxy 6.0.2"
    },
    "entries": [
      {
        "startedDateTime": "2021-01-25T13:29:55.020851+00:00",
        "time": 640,
        "request": {
          "method": "GET",
          "url": "https://tags.tiqcdn.com/utag/tiqapp/utag.v.js?a=tealium-solutions/test-example/202005291402&cb=1611581394205",
          "httpVersion": "HTTP/1.1",
          "cookies": [],
          "headers": [
            {
              "name": "User-Agent",
              "value": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0"
            },
            {
              "name": "Accept",
              "value": "*\/*"
            },
            {
              "name": "Accept-Language",
              "value": "en-US,en;q=0.5"
            },
            {
              "name": "Accept-Encoding",
              "value": "gzip, deflate, br"
            },
            {
              "name": "Referer",
              "value": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html"
            },
            {
              "name": "Host",
              "value": "tags.tiqcdn.com"
            },
            {
              "name": "Via",
              "value": "1.1 maki231 (squid/4.6)"
            },
            {
              "name": "Cache-Control",
              "value": "max-age=0"
            },
            {
              "name": "Connection",
              "value": "keep-alive"
            },
            {
              "name": "X-SL-Job-ID",
              "value": "3e756637232241df8514810fc6da95dc"
            },
            {
              "name": "X-SL-Tunnel-ID",
              "value": "a985b541abf04f18a3ef447451c81fc1"
            },
            {
              "name": "X-SL-Chef-IP",
              "value": "10.113.8.9"
            }
          ],
          "queryString": [
            {
              "name": "a",
              "value": "tealium-solutions/test-example/202005291402"
            },
            {
              "name": "cb",
              "value": "1611581394205"
            }
          ],
          "headersSize": 599,
          "bodySize": 0
        },
        "response": {
          "status": 200,
          "statusText": "OK",
          "httpVersion": "HTTP/1.1",
          "cookies": [],
          "headers": [
            {
              "name": "Accept-Ranges",
              "value": "bytes"
            },
            {
              "name": "Content-Type",
              "value": "application/x-javascript"
            },
            {
              "name": "ETag",
              "value": "\"7bc0ee636b3b83484fc3b9348863bd22:1460653071\""
            },
            {
              "name": "Last-Modified",
              "value": "Thu, 14 Apr 2016 16:57:51 GMT"
            },
            {
              "name": "Server",
              "value": "AkamaiNetStorage"
            },
            {
              "name": "Content-Length",
              "value": "2"
            },
            {
              "name": "Cache-Control",
              "value": "max-age=600"
            },
            {
              "name": "Expires",
              "value": "Mon, 25 Jan 2021 13:39:55 GMT"
            },
            {
              "name": "Date",
              "value": "Mon, 25 Jan 2021 13:29:55 GMT"
            },
            {
              "name": "Connection",
              "value": "keep-alive"
            }
          ],
          "content": {
            "size": 2,
            "compression": 0,
            "mimeType": "application/x-javascript",
            "text": "//"
          },
          "redirectURL": "",
          "headersSize": 422,
          "bodySize": 2
        },
        "cache": {},
        "timings": {
          "send": 15,
          "receive": 8,
          "wait": 318,
          "connect": 56,
          "ssl": 243
        }
      }
    ]
  }
}
    */
    this.rawLogs = har;

    let logs = har.log.entries.map((entry) => {
      const match = entry.startedDateTime.match(/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}).*$/);
      entry.startedDateTime = match[1] + 'Z'; // intentionally fragile, in case the log format changes this should break
      entry.request.urlWithoutQueryString = entry.request.url.split('?')[0];
      return entry;
    });

    logs = addFeaturesToLogs(logs);

    /**
     * An object with detailed network request info, listed per step.
     *
     * Each request will appear in the appropriate step array AND the allSteps array, to help make assertions simpler.
     *
     * Collect requests will also have more advanced payload parsing (parsed, see example.).
     *
     * In the interest of brevity, the example below doesn't show the same request in 'allSteps'.  The 'Accept' headers in this example have been modified to avoid breaking the JSDoc comments.
     *
     * @type {object}
     * @example
     * {
  "allSteps": [ '(omitted for brevity in this example)' ],
  "step1": [{
    "startedDateTime": "2021-01-28T13:04:06.982Z",
    "time": 2225,
    "request": {
      "method": "POST",
      "url": "https://collect-eu-central-1.tealiumiq.com/tealium-solutions/test-example/2/i.gif",
      "httpVersion": "HTTP/1.1",
      "cookies": [],
      "headers": {
        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:85.0) Gecko/20100101 Firefox/85.0",
        "accept": "*\/*",
        "accept-language": "en-US,en;q=0.5",
        "accept-encoding": "gzip, deflate, br",
        "content-type": "multipart/form-data; boundary=---------------------------1787141617783135763328970803",
        "content-length": "1902",
        "origin": "https://solutions.tealium.net",
        "referer": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html",
        "host": "collect-eu-central-1.tealiumiq.com",
        "via": "1.1 maki3615 (squid/4.6)",
        "cache-control": "max-age=900",
        "connection": "keep-alive",
        "x-sl-job-id": "6c043688c72a4b93b67246927cf7527a",
        "x-sl-tunnel-id": "d85beb87f4d74aee9e1fefc7f0038523",
        "x-sl-chef-ip": "10.129.1.183"
      },
      "queryString": {},
      "headersSize": 813,
      "bodySize": 1902,
      "postData": {
        "mimeType": "multipart/form-data; boundary=---------------------------1787141617783135763328970803",
        "text": "-----------------------------1787141617783135763328970803\r\nContent-Disposition: form-data; name=\"data\"\r\n\r\n{\"loader.cfg\":{\"2\":{\"load\":4,\"send\":1,\"v\":202005291402,\"wait\":1,\"tid\":20064,\"id\":\"2\",\"executed\":1}},\"data\":{\"page_type\":\"first_test\",\"cp.utag_main_v_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"cp.utag_main__pn\":\"1\",\"cp.utag_main_ses_id\":\"1611839047103\",\"cp.utag_main__ss\":\"1\",\"cp.utag_main__se\":\"1\",\"cp.utag_main__sn\":\"1\",\"cp.utag_main__st\":\"1611840847103\",\"dom.referrer\":\"\",\"dom.title\":\"Integration Test\",\"dom.domain\":\"solutions.tealium.net\",\"dom.query_string\":\"\",\"dom.hash\":\"\",\"dom.url\":\"https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html\",\"dom.pathname\":\"/hosted/webdriver-testing/standard-integration-test.html\",\"dom.viewport_height\":671,\"dom.viewport_width\":1024,\"ut.domain\":\"tealium.net\",\"ut.version\":\"ut4.46.202005291402\",\"ut.event\":\"view\",\"ut.visitor_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"ut.session_id\":\"1611839047103\",\"ut.account\":\"tealium-solutions\",\"ut.profile\":\"test-example\",\"ut.env\":\"prod\",\"tealium_event\":\"first_test\",\"tealium_visitor_id\":\"0177491805bd00100897c23c7fad00052005500f00718\",\"tealium_session_id\":\"1611839047103\",\"tealium_session_number\":\"1\",\"tealium_session_event_number\":\"1\",\"tealium_datasource\":\"7hpfk3\",\"tealium_account\":\"tealium-solutions\",\"tealium_profile\":\"test-example\",\"tealium_environment\":\"prod\",\"tealium_random\":\"7091299497493456\",\"tealium_library_name\":\"utag.js\",\"tealium_library_version\":\"4.46.0\",\"tealium_timestamp_epoch\":1611839047,\"tealium_timestamp_utc\":\"2021-01-28T13:04:07.106Z\",\"tealium_timestamp_local\":\"2021-01-28T13:04:07.106\",\"cp.utag_main_dc_visit\":\"1\",\"cp.utag_main_dc_event\":\"1\"},\"browser\":{\"height\":671,\"width\":1024,\"screen_height\":768,\"screen_width\":1024,\"timezone_offset\":0},\"event\":\"view\",\"post_time\":1611839047115}\r\n-----------------------------1787141617783135763328970803--\r\n",
        "params": [],
        "parsed": {
          "loader.cfg": {
            "2": {
              "load": 4,
              "send": 1,
              "v": 202005291402,
              "wait": 1,
              "tid": 20064,
              "id": "2",
              "executed": 1
            }
          },
          "data": {
            "page_type": "first_test",
            "cp.utag_main_v_id": "0177491805bd00100897c23c7fad00052005500f00718",
            "cp.utag_main__pn": "1",
            "cp.utag_main_ses_id": "1611839047103",
            "cp.utag_main__ss": "1",
            "cp.utag_main__se": "1",
            "cp.utag_main__sn": "1",
            "cp.utag_main__st": "1611840847103",
            "dom.referrer": "",
            "dom.title": "Integration Test",
            "dom.domain": "solutions.tealium.net",
            "dom.query_string": "",
            "dom.hash": "",
            "dom.url": "https://solutions.tealium.net/hosted/webdriver-testing/standard-integration-test.html",
            "dom.pathname": "/hosted/webdriver-testing/standard-integration-test.html",
            "dom.viewport_height": 671,
            "dom.viewport_width": 1024,
            "ut.domain": "tealium.net",
            "ut.version": "ut4.46.202005291402",
            "ut.event": "view",
            "ut.visitor_id": "0177491805bd00100897c23c7fad00052005500f00718",
            "ut.session_id": "1611839047103",
            "ut.account": "tealium-solutions",
            "ut.profile": "test-example",
            "ut.env": "prod",
            "tealium_event": "first_test",
            "tealium_visitor_id": "0177491805bd00100897c23c7fad00052005500f00718",
            "tealium_session_id": "1611839047103",
            "tealium_session_number": "1",
            "tealium_session_event_number": "1",
            "tealium_datasource": "7hpfk3",
            "tealium_account": "tealium-solutions",
            "tealium_profile": "test-example",
            "tealium_environment": "prod",
            "tealium_random": "7091299497493456",
            "tealium_library_name": "utag.js",
            "tealium_library_version": "4.46.0",
            "tealium_timestamp_epoch": 1611839047,
            "tealium_timestamp_utc": "2021-01-28T13:04:07.106Z",
            "tealium_timestamp_local": "2021-01-28T13:04:07.106",
            "cp.utag_main_dc_visit": "1",
            "cp.utag_main_dc_event": "1"
          },
          "browser": {
            "height": 671,
            "width": 1024,
            "screen_height": 768,
            "screen_width": 1024,
            "timezone_offset": 0
          },
          "event": "view",
          "post_time": 1611839047115
        }
      },
      "urlWithoutQueryString": "https://collect-eu-central-1.tealiumiq.com/tealium-solutions/test-example/2/i.gif"
    },
    "response": {
      "status": 200,
      "statusText": "OK",
      "httpVersion": "HTTP/1.1",
      "cookies": [
        {
          "name": "TAPID",
          "value": "tealium-solutions/test-example>0177491805bd00100897c23c7fad00052005500f00718|",
          "path": "/",
          "domain": ".tealiumiq.com",
          "httpOnly": true,
          "secure": true,
          "expires": "2022-01-28T13:04:08+00:00"
        }
      ],
      "headers": {
        "date": "Thu, 28 Jan 2021 13:04:08 GMT",
        "content-type": "image/gif",
        "content-length": "43",
        "connection": "keep-alive",
        "x-acc": "tealium-solutions:test-example:2:datacloud",
        "x-did": "0177491805bd00100897c23c7fad00052005500f00718",
        "x-region": "eu-central-1",
        "access-control-allow-origin": "https://solutions.tealium.net",
        "x-serverid": "uconnect_i-06054a44183803790",
        "pragma": "no-cache",
        "p3p": "policyref=\"/w3c/p3p.xml\", CP=\"NOI DSP COR NID CUR ADM DEV OUR BUS\"",
        "access-control-expose-headers": "X-Region",
        "cache-control": "no-transform,private,no-cache,no-store,max-age=0,s-maxage=0",
        "x-tid": "0177491805bd00100897c23c7fad00052005500f00718",
        "access-control-allow-credentials": "true",
        "x-ulver": "ed533b75a08fa8f6edbe6695d0295a01b07dd99c-SNAPSHOT",
        "vary": "Origin",
        "expires": "Thu, 28 Jan 2021 13:04:08 GMT",
        "x-uuid": "24ba2778-6a42-412b-9dbc-94dacff69bed",
        "set-cookie": "TAPID=tealium-solutions/test-example>0177491805bd00100897c23c7fad00052005500f00718|; Path=/; Domain=.tealiumiq.com; Expires=Fri, 28-Jan-2022 13:04:08 GMT; Max-Age=31536000; Secure; HttpOnly; SameSite=None"
      },
      "content": {
        "size": 43,
        "compression": 0,
        "mimeType": "image/gif",
        "text": "R0lGODlhAQABAPAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==",
        "encoding": "base64"
      },
      "redirectURL": "",
      "headersSize": 1184,
      "bodySize": 43
    },
    "cache": {},
    "timings": {
      "send": 121,
      "receive": 33,
      "wait": 1061,
      "connect": 831,
      "ssl": 179
    },
    "stepNumber": 1,
    "stepName": "step1"
  }]
}
    */
    this.logs = sortLogsByStep(logs, steps);

    /**
   * The steps that were recognized and used to split the network capture.
   *
   * @example
{
  "stepsSoFar": 4,
  "stepInfo": {
    "1": {
      "start": "2021-01-25T13:29:53.042Z",
      "name": "STEP 1 - initial page visit, set (and verify) Trace cookie, confirm some globals and helpers",
      "end": "2021-01-25T13:29:59.378Z"
    },
    "2": {
      "start": "2021-01-25T13:29:59.378Z",
      "name": "STEP 2 - reload the page, then increment the counter 4 times",
      "end": "2021-01-25T13:30:23.820Z"
    },
    "3": {
      "start": "2021-01-25T13:30:23.820Z",
      "name": "STEP 3 - decrement the counter 5 times",
      "end": "2021-01-25T13:30:45.128Z"
    },
    "4": {
      "start": "2021-01-25T13:30:45.128Z",
      "name": "STEP 4 - spoof a login and file import, setting the counter to 42"
    }
  }
}
   */
    this.steps = steps;

    /**
     * Returns a filtered subset of the provided 'logs' object (based on the request URL)
     *
     * Expects double-escaping because of the string conversion, like  `\\w+`,
     * see [doc](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/RegExp)
     *
     * @param {string} filterStringForRegex A double-escaped string that will be used to create a RegExp, uses as a filter
     * @returns A filtered 'logs' object with only matching entries, as in [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
     * @example
     * it('should find a single TiQ session counter, in the first step', function () {
     *   chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
     *   chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
     * })
     */
    this.getFilteredLogs = function (filterStringForRegex) {
      const filtered = {};
      const logObject = this.logs;
      // make sure all steps have an array, even if it's empty
      const highestStep = this.steps.stepsSoFar;
      for (let i = 1; i <= highestStep; i++) {
        filtered[`step${i}`] = [];
      }
      filtered.allSteps = [];

      const re = new RegExp(filterStringForRegex);
      logObject.allSteps.forEach((logEntry) => {
        const key = logEntry.stepName;
        const match = re.test(logEntry.request.url);
        const isSslHandshake = /.*:443$/.test(logEntry.request.url);
        if (match && !isSslHandshake) {
          filtered[key].push(logEntry);
          filtered.allSteps.push(logEntry);
        }
      });
      return filtered;
    };

    function sortLogsByStep (inputLogArray, steps) {
      // make a deep copy
      const grouped = {};
      const logArray = _.cloneDeep(inputLogArray);
      const renamed = {};

      // ensure all stteps are defined with empty arrays at least
      renamed.allSteps = [];
      for (let i = 1; i <= steps.stepsSoFar; i++) {
        renamed[`step${i}`] = [];
      }

      function findStepNumber (entry) {
        const entryTime = Date.parse(entry.startedDateTime);

        // stop short of the last one
        for (let i = 1; i < steps.stepsSoFar; i++) {
          const step = steps.stepInfo[i];
          const start = Date.parse(step.start);
          const end = Date.parse(step.end);
          const inRange = entryTime >= start && entryTime < end;
          if (inRange) {
            return i;
          }
        }

        // greater or equal to the last step start means it's part of the last step
        if (entryTime >= Date.parse(steps.stepInfo[steps.stepsSoFar].start)) {
          return steps.stepsSoFar;
        }

        // otherwise, it must be stuff like browser setup before the start of the first step, put that as part of the first step
        return 1;
      }

      logArray.forEach((logEntry, i) => {
        const step = findStepNumber(logEntry);
        logEntry.stepNumber = step;
        logEntry.stepName = `step${logEntry.stepNumber}`;
        grouped[step] = grouped[step] || [];
        grouped[step].push(logEntry); // whole entry for now.
      });

      const sortedStepIds = Object.keys(grouped);
      sortedStepIds.forEach((id, index) => {
        const stepNumber = index + 1;
        const stepName = 'step' + stepNumber;
        renamed[stepName] = grouped[id];
        renamed[stepName].forEach((stepEntry) => {
          renamed.allSteps.push(stepEntry);
        });
      });
      return renamed;
    }

    function addFeaturesToLogs (inputLogArray) {
      // avoid side effects with a deep copy
      const logArray = _.cloneDeep(inputLogArray);

      logArray.forEach((entry, i) => {
        // reformat into more useful objects (and lowercase the keys)
        // [{ propName: 'Protocol', propValue: 'https' }] becomes
        // { 'protocol': 'https'}
        entry.request.headers = reformatObject(entry.request.headers);
        entry.response.headers = reformatObject(entry.response.headers);

        entry.request.queryString = reformatObject(entry.request.queryString);

        const isMultipartFormData = entry.request.headers['content-type'] && entry.request.headers['content-type'].indexOf('multipart/form-data;') !== -1;
        const isCollect = isMultipartFormData && /^https?:\/\/collect.*\/2\/i\.gif$/.test(entry.request.url);
        // hack to parse the payload for the Collect tag specifically, ignoring other tags for now because mulitpart form data is
        // hard to parse generally (and all the libraries expect buffers, received from file uploads) - Collect forms are pretty simple
        if (isCollect) {
          const boundary = entry.request.headers['content-type'].split('boundary=')[1];
          const body = entry.request.postData.text;
          const lines = body.split('\r\n');
          let collectBody;
          lines.forEach((line) => {
            if (line.indexOf(boundary) === -1 && /^{.*}$/.test(line)) {
              try {
                collectBody = JSON.parse(line);
              } catch (e) {
                collectBody = { failed: true };
              }
            }
          });
          entry.request.postData.parsed = collectBody;
        }

        // Add /events endpoint parsing as well for POST requests
        if (/^https:\/\/collect\.tealiumiq\.com\/event[/]?$/.test(entry.request.url)) {
          try {
            entry.request.postData.parsed = JSON.parse(entry.request.postData.text);
          } catch (e) {
            console.log(`Failed to parse ${entry.request.postData.text}\n\n${e}`);
          }
        }

        logArray[i] = entry;
      });
      return logArray;
    }

    /**
     * Reformat objects to make them easier to use.
     *
     * Input:
     * [
     *   {
     *     'name': 'Status',
     *     'value': '200'
     *    }
     *  ]
     *
     * Output:
     * {
     *   'Status': '200'
     * }
     * @private
     * @param {array} inputArray an array of objects with keys 'propName' and 'propValue'
     * @returns {object} the reformatted array in classic key/value style
     */
    function reformatObject (inputArray) {
      const outputObject = {};
      inputArray.forEach((entry) => {
        outputObject[entry.name.toLowerCase()] = entry.value;
      });
      return outputObject;
    }
  }
}

// export for tests
exports.ProxyLogs = ProxyLogs;

// const reporter = rootRequire('platform-helpers/reporter-helper.js')

/**
 * Retrieves and parses the network request logs captured by [mitmproxy](https://mitmproxy.org/).
 *
 * @public
 * @returns {Promise}
 * @yields instance of [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
 * @deprecated This method can still be used but its being refactored. Alternatively start using the new method "require(./mitmproxy-helper).getLogsV2()"
 * @example
// Define this variable outside the tests to allow us to split the assertions and the retrieval itself into different steps.
let proxyLogs

describe('FINISH - retrieve, validate, and check the proxy logs for for the run', async function () {
  it('should fetch network logs (from mitmproxy in the Docker container)', function () {
    // this helper returns a promise which resolves into the proxy output for the run
    proxyLogs = await proxyHelper.getLogs()
  })

  // now you have an instance of ProxyLogs
  it('should have the expected properties on the returned ProxyLogs object', async function () {
    // logs
    chai.expect(proxyLogs.logs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
    chai.expect(proxyLogs.logs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)

    // raw logs
    chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
    chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)

    // steps
    chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepsSoFar').that.is.a('number').greaterThan(0)
    chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepInfo').that.is.an('object').with.property('1').that.is.an('object').with.keys(['name', 'start', 'end'])

    // getFilteredLogs function
    chai.expect(proxyLogs.getFilteredLogs).to.be.an('function')
  })

  it('should find a single TiQ session counter, in the first step', async function () {
    chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
    chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
  })
})

*/
exports.getLogs = function () {
  return new Promise(function (resolve, reject) {
    console.log('[Proxy] Converting captured files...');
    fse.removeSync(`${rootDir}/ready/logs.har`);
    const { exec } = require('child_process');
    exec('docker compose exec -T sauce_connect mitmdump -n -r ready/logs.mitm -s scripts/har_dump.py --set hardump=ready/logs.har', (error, stdout, stderr) => {
      if (error) {
        reject(error);
      }
      // console.log(`stdout: ${stdout}`)
      if (stderr) {
        console.error(`stderr: ${stderr}`);
      }

      const output = fse.readJsonSync(`${rootDir}/ready/logs.har`);
      const steps = fse.readJsonSync(`${rootDir}/ready/steps.json`);
      const proxyLogs = new ProxyLogs(output, steps);
      logger.log('proxy_logs', proxyLogs, Logger.Outputs.FILE);
      resolve(proxyLogs);
    });
  });
};

/**
 * Retrieves and parses the network request logs captured by [mitmproxy](https://mitmproxy.org/).
 *
 * @public
 * @returns {Promise}
 * @yields instance of [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
 * @example
// Define this variable outside the tests to allow us to split the assertions and the retrieval itself into different steps.
let proxyLogs

describe('FINISH - retrieve, validate, and check the proxy logs for for the run', async function () {
  it('should fetch network logs (from mitmproxy in the Docker container)', function () {
    // this helper returns a promise which resolves into the proxy output for the run
    proxyLogs = await proxyHelper.getLogsV2()
  })

  // now you have an instance of ProxyLogs
  it('should have the expected properties on the returned ProxyLogs object', async function () {
    // logs
    chai.expect(proxyLogs.logs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
    chai.expect(proxyLogs.logs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)

    // raw logs
    chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
    chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)

    // steps
    chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepsSoFar').that.is.a('number').greaterThan(0)
    chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepInfo').that.is.an('object').with.property('1').that.is.an('object').with.keys(['name', 'start', 'end'])

    // getFilteredLogs function
    chai.expect(proxyLogs.getFilteredLogs).to.be.an('function')
  })

  it('should find a single TiQ session counter, in the first step', async function () {
    chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
    chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
  })
})

*/
exports.getLogsV2 = async function () {
  console.log('[Proxy] Converting captured files...');
  fse.removeSync(`${rootDir}/ready/logs.har`);
  const util = require('util');
  const exec = util.promisify(require('child_process').exec);
  try {
    await exec('docker compose exec -T sauce_connect mitmdump -n -r ready/logs.mitm -s scripts/har_dump.py --set hardump=ready/logs.har');
    const output = fse.readJsonSync(`${rootDir}/ready/logs.har`);
    const steps = fse.readJsonSync(`${rootDir}/ready/steps.json`);

    const proxyLogs = new ProxyLogs(output, steps);
    return proxyLogs;
  } catch (error) {
    console.error(`stderr: ${error}`);
  }
};

/**
 * Adds Trace cookie writing to the provided extraProxyCommands, only meant to be used internally.
 *
 * Rudimentary approach so far - we
 *
 * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
 * @param {string} traceAccount - the Tealium CDH account associated with the traceId
 * @param {string} traceProfile  - the Tealium CDH account associated with the traceId
 * @param {string} traceId - the traceId to add to outgoing requests from the proxy
 * @returns {string} the provided extraProxyCommands (if any) plus the necessary Trace cookie proxy commands
 * @private
 */
exports.addTraceCookie = function (extraProxyCommands, traceAccount, traceProfile, traceId) {
  if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';

  if (traceAccount && traceProfile && traceId) {
    // lots of escaping, but it's correct to make the quotes work
    const collectTagBodyRewrite = `--modify-body @~q@"tealium_account@"cp.trace_id":"${traceId}","tealium_account`.replace(/"/g, '\\"');
    const pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http

def request(flow: http.HTTPFlow) -> None:
    if flow.request.pretty_url.startswith("https://datacloud.tealiumiq.com/vdata/i.gif") and flow.request.query["tealium_account"] == "${traceAccount}" and flow.request.query["tealium_profile"] == "${traceProfile}":
        flow.request.query["tealium_trace_id"] = "${traceId}"

`;

    fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/auto_trace_cookie.py`, pythonScript);
    extraProxyCommands = `${collectTagBodyRewrite} -s scripts/auto_trace_cookie.py ${extraProxyCommands}`;
  }
  return extraProxyCommands;
};

/**
 * Adds request blocking functionality, meant to be used internally.
 *
 * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
 * @param {array} domainList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties
 * @returns {string} the provided extraProxyCommands (if any) plus any necessary request blocking proxy commands
 * @private
 */
exports.addRequestBlocking = function (extraProxyCommands, domainList) {
  if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';

  let foundValidEntry = false;

  let pythonScript = `"""auto-generated in wdio.conf.js, sends a reply from the proxy without sending any data to the remote server for specified urls"""
from mitmproxy import http
import re

def request(flow: http.HTTPFlow) -> None:
`;

  domainList = domainList || [];
  domainList.forEach(function (obj, i) {
    if (typeof obj.pattern !== 'string') return;
    if (obj.type !== 'string' && obj.type !== 'regex') return;
    obj.responseCode = obj.responseCode || 403;
    obj.responseBody = obj.responseBody || 'b"Request blocked by the Tealium testing tool."';
    obj.responseContentType = obj.responseContentType || 'text/html';

    foundValidEntry = true;

    let filterConditional = '';
    if (obj.type === 'string') {
      filterConditional = `    if '${obj.pattern}' in flow.request.pretty_url:`;
    } else {
      filterConditional = `    pattern = re.compile("${obj.pattern}")
    if re.search(pattern, flow.request.pretty_url):`;
    }

    pythonScript += `${filterConditional}
        flow.response = http.Response.make(
            ${obj.responseCode},
            ${obj.responseBody},
            {"Content-Type": "${obj.responseContentType}"}
        )

`;
  });

  if (foundValidEntry) {
    fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/block_requests.py`, pythonScript);
    extraProxyCommands = `-s scripts/block_requests.py ${extraProxyCommands}`;
  }

  return extraProxyCommands;
};

/**
 * Adds URL rewriting functionality, intended to be used for internal functionality
 *
 * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
 * @param {array} rewritesList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties, as well as (string) 'target' and 'replacement' keys
 * @returns {string} the provided extraProxyCommands (if any) plus the necessary url-rewriting proxy commands
 * @private
 */
exports.addUrlRewrites = function (extraProxyCommands, rewritesList) {
  if (typeof extraProxyCommands !== 'string') extraProxyCommands = '';

  let pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http
import re

def request(flow: http.HTTPFlow) -> None:
`;

  let foundValidEntry = false;

  rewritesList = rewritesList || [];
  rewritesList.forEach(function (obj, i) {
    if (typeof obj.pattern !== 'string') return;
    if (obj.type !== 'string' && obj.type !== 'regex') return;
    if (typeof obj.target !== 'string') return;
    if (typeof obj.replacement !== 'string') return;

    foundValidEntry = true;

    let filterConditional = '';
    if (obj.type === 'string') {
      filterConditional = `    if '${obj.pattern}' in flow.request.pretty_url:`;
    } else {
      filterConditional = `    pattern = re.compile("${obj.pattern}")
    if re.search(pattern, flow.request.pretty_url):`;
    }

    pythonScript += `${filterConditional}
        flow.request.url = flow.request.url.replace("${obj.target}", "${obj.replacement}")
        # flow.request.pretty_url = flow.request.pretty_url.replace("${obj.target}", "${obj.replacement}")

`;
  });

  if (foundValidEntry) {
    fse.writeFileSync(`${rootDir}/platform-helpers/docker-scripts/url_rewrites.py`, pythonScript);
    extraProxyCommands = `-s scripts/url_rewrites.py ${extraProxyCommands}`;
  }

  return extraProxyCommands;
};