platform-helpers/mitmproxy-helper.js

'use strict';

const fse = require('fs-extra');
const rootRequire = require('rpcm-root-require');
const ProxyLogs = rootRequire('platform-helpers/proxy-logs');
const LoggerWrapper = rootRequire('/platform-helpers/log4js-wrapper');
const DockerHelper = rootRequire('platform-helpers/docker-helper');
const logger = LoggerWrapper.getLogger();
rootRequire('/platform-helpers/string-extensions');

/**
 * [mitmproxy](https://mitmproxy.org/) helper functions.
 *
 * mitmproxy writes to a file in realtime, so you can just check the file whenever you want.
 */

class ProxyHelper {
  static startNewStep = function (name) {
    const stepFilePath = '{0}/ready/steps.json'.format(rootRequire.rootPath);
    fse.ensureFileSync(stepFilePath);
    let stepFileContents = fse.readFileSync(stepFilePath);
    try {
      stepFileContents = JSON.parse(stepFileContents);
    } catch (e) {
      stepFileContents = { stepsSoFar: 0 };
    }

    const startTime = new Date().toISOString();

    stepFileContents.stepsSoFar++;
    stepFileContents.stepInfo = stepFileContents.stepInfo || {};
    stepFileContents.stepInfo[stepFileContents.stepsSoFar] = { start: startTime, name: name || '' };

    if (stepFileContents.stepsSoFar > 1) {
      stepFileContents.stepInfo[stepFileContents.stepsSoFar - 1].end = startTime;
    }

    fse.writeFileSync(stepFilePath, JSON.stringify(stepFileContents, null, 2));
  };

  /**
   * Retrieves and parses the network request logs captured by [mitmproxy](https://mitmproxy.org/).
   *
   * @public
   * @returns {Promise}
   * @yields instance of [ProxyLogs]{@link module:proxyHelper~ProxyLogs}
   * @example
   * // Define this variable outside the tests to allow us to split the assertions and the retrieval itself into different steps.
   * let proxyLogs
   *
   * describe('FINISH - retrieve, validate, and check the proxy logs for for the run', async function () {
   *   it('should fetch network logs (from mitmproxy in the Docker container)', function () {
   *     // this helper returns a promise which resolves into the proxy output for the run
   *     proxyLogs = await proxyHelper.getLogs()
   *   })
   *
   *   // now you have an instance of ProxyLogs
   *   it('should have the expected properties on the returned ProxyLogs object', async function () {
   *     // logs
   *     chai.expect(proxyLogs.logs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
   *     chai.expect(proxyLogs.logs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
   *
   *     // raw logs
   *     chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('allSteps').that.is.an('array').with.length.greaterThan(0)
   *     chai.expect(proxyLogs.rawLogs).to.be.an('object').with.property('step1').that.is.an('array').with.length.greaterThan(0)
   *
   *     // steps
   *     chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepsSoFar').that.is.a('number').greaterThan(0)
   *     chai.expect(proxyLogs.steps).to.be.an('object').with.property('stepInfo').that.is.an('object').with.property('1').that.is.an('object').with.keys(['name', 'start', 'end'])
   *
   *     // getFilteredLogs function
   *     chai.expect(proxyLogs.getFilteredLogs).to.be.an('function')
   *   })
   *
   *   it('should find a single TiQ session counter, in the first step', async function () {
   *     chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').allSteps).to.have.lengthOf(1)
   *     chai.expect(proxyLogs.getFilteredLogs('/utag.v.js?').step1).to.have.lengthOf(1)
   *   })
   * })
   *
   */
  static getLogs = async function () {
    logger.trace('[Proxy] Converting captured files...');
    fse.removeSync('{0}/ready/logs.har'.format(rootRequire.rootPath));
    try {
      await DockerHelper.dumpMitmProxyLogs();
      const output = fse.readJsonSync('{0}/ready/logs.har'.format(rootRequire.rootPath));
      const steps = fse.readJsonSync('{0}/ready/steps.json'.format(rootRequire.rootPath));

      const proxyLogs = new ProxyLogs(output, steps);
      return proxyLogs;
    } catch (error) {
      logger.error('stderr: {0}'.format(error));
    }
  };

  /**
 * Adds Trace cookie writing to the provided extraProxyCommands, only meant to be used internally.
 *
 * Rudimentary approach so far - we
 *
 * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
 * @param {string} traceAccount - the Tealium CDH account associated with the traceId
 * @param {string} traceProfile  - the Tealium CDH account associated with the traceId
 * @param {string} traceId - the traceId to add to outgoing requests from the proxy
 * @returns {string} the provided extraProxyCommands (if any) plus the necessary Trace cookie proxy commands
 * @private
 */
  static addTraceCookie = function (extraProxyCommands, traceAccount, traceProfile, traceId) {
    if (traceAccount && traceProfile && traceId) {
    // lots of escaping, but it's correct to make the quotes work
      const collectTagBodyRewrite = `--modify-body @~q@"tealium_account@"cp.trace_id":"${traceId}","tealium_account`.replace(/"/g, '\\"');
      const pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http

def request(flow: http.HTTPFlow) -> None:
    if flow.request.pretty_url.startswith("https://datacloud.tealiumiq.com/vdata/i.gif") and flow.request.query["tealium_account"] == "${traceAccount}" and flow.request.query["tealium_profile"] == "${traceProfile}":
        flow.request.query["tealium_trace_id"] = "${traceId}"

`;

      fse.writeFileSync('{0}/platform-helpers/docker-scripts/auto_trace_cookie.py'.format(rootRequire.rootPath), pythonScript);
      extraProxyCommands = '{0} -s scripts/auto_trace_cookie.py {1}'.format(collectTagBodyRewrite, extraProxyCommands);
    }
    return extraProxyCommands;
  };

  /**
   * Adds request blocking functionality, meant to be used internally.
   *
   * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
   * @param {array} domainList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties
   * @returns {string} the provided extraProxyCommands (if any) plus any necessary request blocking proxy commands
   * @private
   */
  static addRequestBlocking = function (extraProxyCommands, domainList) {
    let foundValidEntry = false;
    let pythonScript = `"""auto-generated in wdio.conf.js, sends a reply from the proxy without sending any data to the remote server for specified urls"""
from mitmproxy import http
import re

def request(flow: http.HTTPFlow) -> None:
`;

    domainList = domainList || [];
    domainList.forEach(function (obj, i) {
      if (typeof obj.pattern !== 'string') return;
      if (obj.type !== 'string' && obj.type !== 'regex') return;
      obj.responseCode = obj.responseCode || 403;
      obj.responseBody = obj.responseBody || 'b"Request blocked by the Tealium testing tool."';
      obj.responseContentType = obj.responseContentType || 'text/html';

      foundValidEntry = true;

      let filterConditional = '';
      if (obj.type === 'string') {
        filterConditional = `    if '${obj.pattern}' in flow.request.pretty_url:`;
      } else {
        filterConditional = `    pattern = re.compile("${obj.pattern}")
    if re.search(pattern, flow.request.pretty_url):`;
      }

      pythonScript += `${filterConditional}
        flow.response = http.Response.make(
            ${obj.responseCode},
            ${obj.responseBody},
            {"Content-Type": "${obj.responseContentType}"}
        )

`;
    });

    if (foundValidEntry) {
      fse.writeFileSync('{0}/platform-helpers/docker-scripts/block_requests.py'.format(rootRequire.rootPath), pythonScript);
      extraProxyCommands = `-s scripts/block_requests.py ${extraProxyCommands}`;
    }

    return extraProxyCommands;
  };

  /**
 * Adds URL rewriting functionality, intended to be used for internal functionality
 *
 * @param {string} [extraProxyCommands] - extraProxyCommands from the runner (defaults to an empty string if a non-string is provided)
 * @param {array} rewritesList - an array of objects, each with 'pattern' (string or regex pattern to match) and 'type' ('string' or 'regex') properties, as well as (string) 'target' and 'replacement' keys
 * @returns {string} the provided extraProxyCommands (if any) plus the necessary url-rewriting proxy commands
 * @private
 */
  static addUrlRewrites = function (extraProxyCommands, rewritesList) {
    let pythonScript = `"""auto-generated in wdio.conf.js"""
from mitmproxy import http
import re

def request(flow: http.HTTPFlow) -> None:
`;

    let foundValidEntry = false;

    rewritesList = rewritesList || [];
    rewritesList.forEach(function (obj, i) {
      if (typeof obj.pattern !== 'string') return;
      if (obj.type !== 'string' && obj.type !== 'regex') return;
      if (typeof obj.target !== 'string') return;
      if (typeof obj.replacement !== 'string') return;

      foundValidEntry = true;

      let filterConditional = '';
      if (obj.type === 'string') {
        filterConditional = `    if '${obj.pattern}' in flow.request.pretty_url:`;
      } else {
        filterConditional = `    pattern = re.compile("${obj.pattern}")
    if re.search(pattern, flow.request.pretty_url):`;
      }

      pythonScript += `${filterConditional}
        flow.request.url = flow.request.url.replace("${obj.target}", "${obj.replacement}")
        # flow.request.pretty_url = flow.request.pretty_url.replace("${obj.target}", "${obj.replacement}")

`;
    });

    if (foundValidEntry) {
      fse.writeFileSync('{0}/platform-helpers/docker-scripts/url_rewrites.py'.format(rootRequire.rootPath), pythonScript);
      extraProxyCommands = `-s scripts/url_rewrites.py ${extraProxyCommands}`;
    }

    return extraProxyCommands;
  };
}

module.exports = ProxyHelper;