/* global */
'use strict';
const qs = require('qs');
const Moment = require('moment');
const dotenv = require('dotenv');
const lodash = require('lodash');
const axios = require('axios');
const rootRequire = require('rpcm-root-require');
const reportHelper = rootRequire('/platform-helpers/report-helper.js');
const TiqHelper = rootRequire('/platform-helpers/tiq-helper');
const Utils = rootRequire('/platform-helpers/utils');
const TraceRecord = rootRequire('/platform-helpers/trace-record');
const TraceEvent = rootRequire('/platform-helpers/trace-event');
const TraceSummary = rootRequire('/platform-helpers/trace-summary');
const CdhConfigBeautifier = rootRequire('/platform-helpers/cdh-configuration-beautifier');
const CdhHelper = rootRequire('/platform-helpers/cdh-helper');
const visitorProfileBeautifier = rootRequire('/platform-helpers/visitor-profile-beautifier');
const objectHelper = rootRequire('/platform-helpers/object-helper');
const Constants = rootRequire('/platform-helpers/constants');
const LoggerWrapper = rootRequire('/platform-helpers/log4js-wrapper');
const logger = LoggerWrapper.getLogger();
rootRequire('/platform-helpers/string-extensions');
/**
* Adds the pretty names to events based on a provided map
*
* @private
* @param {object} event the raw event to base the returned pretty event object on
* @param {map} map a map of CDH attribute ids to enrich the provided raw event against
* @returns {object} the pretty event
*/
const prettifyEvent = (event, map) => {
map = map || {};
event.audiences_before = visitorProfileBeautifier.renameArrayFromMap(event.audiences_before, map.reverseAudiences, 'id');
event.audiences_left = visitorProfileBeautifier.renameArrayFromMap(event.audiences_left, map.reverseAudiences, 'id');
event.audiences_joined = visitorProfileBeautifier.renameArrayFromMap(event.audiences_joined, map.reverseAudiences, 'id');
event.audiences_after = visitorProfileBeautifier.renameArrayFromMap(event.audiences_after, map.reverseAudiences, 'id');
// turn ids into names
event.badges_before_pretty = visitorProfileBeautifier.renameArrayFromMap(event.badges_before, map.visitorAttributes, 'name');
event.badges_removed_pretty = visitorProfileBeautifier.renameArrayFromMap(event.badges_removed, map.visitorAttributes, 'name');
event.badges_added_pretty = visitorProfileBeautifier.renameArrayFromMap(event.badges_added, map.visitorAttributes, 'name');
event.badges_after_pretty = visitorProfileBeautifier.renameArrayFromMap(event.badges_after, map.visitorAttributes, 'name');
event.audiences_before_pretty = visitorProfileBeautifier.renameArrayFromMap(event.audiences_before, map.audiences, 'name');
event.audiences_left_pretty = visitorProfileBeautifier.renameArrayFromMap(event.audiences_left, map.audiences, 'name');
event.audiences_joined_pretty = visitorProfileBeautifier.renameArrayFromMap(event.audiences_joined, map.audiences, 'name');
event.audiences_after_pretty = visitorProfileBeautifier.renameArrayFromMap(event.audiences_after, map.audiences, 'name');
event.visitor_profile_before = visitorProfileBeautifier.beautify(event.visitor_profile_before || {}, map);
event.visitor_profile_after = visitorProfileBeautifier.beautify(event.visitor_profile_after || {}, map);
if (event.tealium_event && event.tealium_event !== '' && map.eventSpecs && Object.prototype.hasOwnProperty.call(map.eventSpecs, event.tealium_event)) {
event.event_spec_exists = true;
event.event_spec_missing_attributes = (() => {
const requiredAttributes = [];
const missingAttributes = [];
map.eventSpecs[event.tealium_event].eventAttributes.forEach((attributeInSpec) => {
if (attributeInSpec.required) {
const attributeName = /data\.udo\.(.*$)/.exec(attributeInSpec.attribute)[1];
requiredAttributes.push(attributeName);
}
});
requiredAttributes.forEach((attr) => {
if (Object.prototype.hasOwnProperty.call(event.raw_event_json.data.udo, attr) === false) missingAttributes.push(attr);
});
return missingAttributes;
})();
event.event_spec_is_valid = event.event_spec_missing_attributes.length === 0;
}
// mostly useful for debugging, remove to reduce clutter
delete event.entries;
// we only needed this to evaluate the event specs
delete event.raw_event_json;
// remove empty arrays and strings
const keys = Object.keys(event);
keys.forEach((key) => {
if (event[key] && event[key].length === 0) {
delete event[key];
}
});
// sort remaining keys
event = objectHelper.sortKeysByName(event);
return event;
};
/**
* Calculates audiences_after on the visitor profiles and adds some other nice-to-have properties
*
* @private
* @param parentEvent a parentEvent summary of the raw trace JSON
* @param lastPrettyEvent the previous event (needed to get audiences_before)
* @returns the finalized (but not yet prettified) parentEvent
*/
const finishParentEvent = (parentEvent, lastPrettyEvent) => {
// the Trace payload doesn't include a visitor_profile_after with the current audiences on it, so we
// have to build the current audiences array ourselves.
const audiencesLeft = parentEvent.audiences_left.slice(0);
const audiencesJoined = parentEvent.audiences_joined.slice(0);
let audiencesAfterLastEvent = [];
// if there's a previous event, use audiences_after, otherwise use the intial profile's audiences_before (which will be strange for events that include a stitch)
if (lastPrettyEvent) {
audiencesAfterLastEvent = (lastPrettyEvent.audiences_after && lastPrettyEvent.audiences_after.slice(0)) || [];
} else {
audiencesAfterLastEvent = (parentEvent.visitor_profile_before.audiences && parentEvent.visitor_profile_before.audiences.slice(0)) || [];
}
const calculatedCurrentAudiences = [];
audiencesAfterLastEvent.forEach((audience) => {
if (audience && audiencesLeft.indexOf(audience) === -1) {
calculatedCurrentAudiences.push(audience);
}
});
audiencesJoined.forEach((audience) => {
if (audience && audiencesLeft.indexOf(audience) === -1) {
calculatedCurrentAudiences.push(audience);
}
});
// filter out any duplicates
const alreadySeen = [];
const currentAudiencesNoDuplicates = calculatedCurrentAudiences.filter((audience) => {
if (alreadySeen.indexOf(audience) !== -1) {
return false;
}
alreadySeen.push(audience);
return true;
});
parentEvent.audiences_before = audiencesAfterLastEvent || [];
parentEvent.audiences_after = currentAudiencesNoDuplicates || [];
parentEvent.visitor_profile_after.audiences = currentAudiencesNoDuplicates || [];
parentEvent.badges_before = (parentEvent.visitor_profile_before && parentEvent.visitor_profile_before.badges) || [];
parentEvent.badges_after = (parentEvent.visitor_profile_after && parentEvent.visitor_profile_after.badges) || [];
return parentEvent;
};
const getFreshTraceRecord = async (account, profile, traceId, endVisit) => {
const random = Utils.getRandomString();
await Utils.sleep(10);
const traceOutput = await TraceHelper.getTraceRecord(account, profile, traceId);
if (!endVisit) {
const visitorId = traceOutput && traceOutput.traceJson[0] && traceOutput.traceJson[0].visitor_id;
const data = {
tealium_account: account,
tealium_profile: profile,
tealium_random: random,
tealium_event: 'update_random_to_ensure_freshness',
tealium_visitor_id: visitorId,
tealium_trace_id: traceId,
tealium_datasource: ''
};
await CdhHelper.sendCollectApiRequest(data);
}
if (endVisit) {
const allVisitorIdsObj = {};
traceOutput.eventGroups.forEach((event) => {
allVisitorIdsObj[event.visitor_profile_before._id] = true;
allVisitorIdsObj[event.visitor_profile_after._id] = true;
});
const allVisitorIds = Object.keys(allVisitorIdsObj);
const visitEndPayloads = allVisitorIds.map((visitorId) => {
return {
tealium_account: account,
tealium_profile: profile,
tealium_visitor_id: visitorId,
tealium_event: 'kill_visitor_session',
tealium_trace_id: traceId,
tealium_random: Utils.getRandomString(),
event: 'kill_visitor_session'
};
});
for (let index = 0; index < visitEndPayloads.length; index++) {
const item = visitEndPayloads[index];
reportHelper.logMessage('Sent Visit End request ({0}) {1}'.format(Moment(), JSON.stringify(item, null, 2)));
await CdhHelper.sendCollectApiRequest(item);
await Utils.sleep(400);
}
}
let newTraceOutput = await TraceHelper.getTraceRecord(account, profile, traceId);
let profileIsFresh = false;
let visitEndAttemptCounter = 0; // Trace needs a couple retries to be reliable after each event, this lets us fix it
const requiredVisitEndAttempts = 2; // Playing the odds really, there's definitely a more elegant approach out there for this
for (let tries = 0; tries < 20; tries++) {
if (tries > 20) {
throw new Error('[Trace] Failed to get fresh Trace, stopped after attempt 20 of 20');
}
const currentProfile = newTraceOutput && newTraceOutput.summary && newTraceOutput.summary.lastProfile;
const visitorId = currentProfile && currentProfile._id;
const lastEvent = currentProfile && currentProfile.current_visit && currentProfile.current_visit.last_event.data.udo;
const lastEventTealiumRandom = lastEvent && lastEvent.tealium_random;
// if we've ended the visit, we should be able to just see check if the visitor is live to confirm profile freshness, but it's not working reliably
if (endVisit) {
visitEndAttemptCounter++;
profileIsFresh = visitEndAttemptCounter >= requiredVisitEndAttempts;
} else {
profileIsFresh = lastEventTealiumRandom && lastEventTealiumRandom === random && typeof visitorId !== 'undefined';
if (profileIsFresh) {
reportHelper.logMessage('Confirmed fresh mid-session Trace on attempt {0} of 20 - {1} === {2}'
.format(tries, random, lastEventTealiumRandom));
}
}
logger.trace('trace-helper.js --> getFreshTraceRecord --> {0}_{1}_{2}_traceJson'.format(account, profile, traceId));
if (profileIsFresh) {
break;
}
await Utils.sleep(400);
newTraceOutput = await TraceHelper.getTraceRecord(account, profile, traceId);
}
const freshTraceRecord = new TraceRecord(newTraceOutput);
logger.trace('trace-helper.js --> getFreshTraceRecord --> {0}_{1}_{2}_traceJson'.format(account, profile, traceId));
return freshTraceRecord;
};
/**
* Trace helper class.
*
* @module TraceHelper
* @static
* @requires axios
* @requires string-extensions
*/
class TraceHelper {
/**
*
* @param {*} account
* @param {*} profile
* @param {*} tealSessionDetails
* @returns
*/
static getNewTraceId = async (account, profile, tealSessionDetails = null) => {
dotenv.config();
if (!tealSessionDetails) {
tealSessionDetails = await TiqHelper.getTiqSessionDetails(account, profile, process.env.TEALIUM_EMAIL, process.env.TEALIUM_PASS);
}
const traceCall = await axios.post('https://my.tealiumiq.com/urest/datacloud/{0}/{1}/trace?utk={2}'.format(account, profile, tealSessionDetails.utk),
{}, {
headers: {
Cookie: 'JSESSIONID={0}'.format(tealSessionDetails.jsessionId),
Accept: 'application/json'
}
});
return traceCall.data.trace_id;
};
/**
*
* @param {*} account
* @param {*} profile
* @param {*} traceId
* @param {*} tealSessionDetails
* @param {*} stepTimeGreaterThan
* @returns
*/
static getTraceById = async (account, profile, traceId, tealSessionDetails, stepTimeGreaterThan) => {
const t = { utk: tealSessionDetails.utk };
if (stepTimeGreaterThan) t.stgt = stepTimeGreaterThan;
return await axios.get('https://my.tealiumiq.com/urest/datacloud/{0}/{1}/trace/{2}/step?{3}'
.format(account, profile, traceId, qs.stringify(t)), {
headers: {
Authority: 'my.tealiumiq.com',
Cookie: 'JSESSIONID={0}'.format(tealSessionDetails.jsessionId),
Accept: 'application/json',
Referrer: 'https://my.tealiumiq.com/datacloud/en-US/?account={0}&profile={1}'.format(account, profile),
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
}
});
};
/**
*
* @param {*} account
* @param {*} profile
* @param {*} traceId
* @returns
*/
static getTraceRecord = async (account, profile, traceId) => {
dotenv.config();
logger.trace('trace-helper.js --> getTraceRecord --> getting trace for id '.format(traceId));
let traceJson = [];
logger.trace('trace-helper.js --> getTraceRecord --> fetching tiqSessionDetails');
const tiqSessionDetails = await TiqHelper.getTiqSessionDetails(account, profile, process.env.TEALIUM_EMAIL, process.env.TEALIUM_PASS);
logger.trace('trace-helper.js --> getTraceRecord --> fetching rawCdhProfileData');
const rawCdhProfileData = await CdhHelper.getRawCdhConfiguration(account, profile, tiqSessionDetails);
logger.trace('trace-helper.js --> getTraceRecord --> processing CdhConfigBeautifier.beautify');
const prettyCdhConfiguration = CdhConfigBeautifier.beautify(rawCdhProfileData);
logger.trace('trace-helper.js --> getTraceRecord --> fetching with getTraceById for 7 times with a timegap of 100 ms');
// not sure why 7, previously there before ownership takeover
for (let index = 0; index < 7; index++) {
// the last one in the array isn't necessarily the latest,
// but here we just take the last one, like Trace does in the AS UI version
const stepTimeGreaterThan = traceJson[traceJson.length - 1] && traceJson[traceJson.length - 1].step_time;
const axiosTraceRequest = await TraceHelper.getTraceById(account, profile, traceId, tiqSessionDetails, stepTimeGreaterThan);
traceJson = traceJson.concat(axiosTraceRequest.data);
logger.trace('trace-helper.js --> getTraceRecord --> fetch {0} returned {1} trace items'.format(index + 1, traceJson.length));
await Utils.sleep(100);
}
logger.trace('trace-helper.js --> getTraceRecord --> filter trace items by unique step_id');
traceJson = lodash.uniqBy(traceJson, obj => obj.step_id);
logger.trace('trace-helper.js --> getTraceRecord --> total unique trace items {0}'.format(traceJson.length));
logger.trace('trace-helper.js --> getTraceRecord --> processing TraceHelper.generateTraceEventArray');
const traceEventArray = TraceHelper.generateTraceEventArray(traceJson, prettyCdhConfiguration);
logger.trace('trace-helper.js --> getTraceRecord --> building a new TraceSummary object');
const summary = new TraceSummary(traceEventArray, rawCdhProfileData);
logger.trace('trace-helper.js --> getTraceRecord --> building and returning a new TraceRecord object');
return new TraceRecord({
prettyCdhConfiguration: prettyCdhConfiguration,
traceJson: traceJson,
eventGroups: traceEventArray,
summary: summary
});
};
/**
* Get the up-to-date Trace record for the specified (current) trace. Does not end the visit.
*
* Intended to enable mid-journey success checks from within tests.
*
* @public
* @param {string} account the CDH profile associated with the traceId
* @param {string} profile the CDH profile associated with the traceId
* @param {string} traceId the current traceId
* @returns {object}
* @yields an instance of [TraceRecord]{@link module:TraceRecord} containing fresh information about your trace.
* @example
* it('should have the correct customer_id and no purchases so far on the Visitor Profile', async function () {
* const output = await TraceHelper.getTraceInfo(traceAccount, traceProfile, traceId)
* const latestProfile = output.summary.lastProfile
*
* // it should be prefixed
* const expected = 'friend_' + customerId
* reporterHelper.logMessage(`Expected customer_id is ${expected}`)
* chai.expect(latestProfile.secondary_ids['5160']).to.equal(expected)
* chai.expect(latestProfile.metrics['5160']).to.equal(expected)
* })
*/
static getTraceInfo = async (account, profile, traceId) => {
logger.trace('trace-helper.js --> getTraceInfo --> Getting trace info...');
return await getFreshTraceRecord(account, profile, traceId, false);
};
/**
* End the current Visit in the current AudienceStream Trace, and return the final Trace Record
*
* @public
* @param {string} account the CDH profile associated with the traceId
* @param {string} profile the CDH profile associated with the traceId
* @param {string} traceId the current traceId
* @returns {Promise}
* @yields an instance of [TraceRecord]{@link module:TraceRecord} containing fresh information about your trace.
* @example
* describe('FINISH AND GET TRACE - End Trace visit and get Trace logs', function () {
* it('FINISH TRACE - end the AS visit and get the trace', async function () {
* const traceEndAndFetch = await traceHelper.endVisitAndGetTraceInfo(traceAccount, traceProfile, traceId)
* chai.expect(traceEndAndFetch).to.be.an('object')
*
* // these have been defined before the first 'describe' block, to make available in subsequent tests as well
* traceJson = traceEndAndFetch.traceJson
* traceEventGroups = traceEndAndFetch.eventGroups
* traceSummary = traceEndAndFetch.summary
* })
*
* it('Trace Validation', async function () {
* reporterHelper.logMessage(`# Trace Summary By Event\n\n${traceSummary.summaryString}`, true)
* reporterHelper.logMessage(`Visitor Profile: ${JSON.stringify(traceSummary.lastProfile, null, 2)}`)
* chai.expect(traceJson, 'traceJson').to.not.be.undefined()
* chai.expect(traceEventGroups, 'traceEventGroups').to.not.be.undefined()
* chai.expect(traceSummary, 'summary').to.not.be.undefined()
* })
* })
*/
static endVisitAndGetTraceInfo = async (account, profile, traceId) => {
logger.trace('trace-helper.js --> endVisitAndGetTraceInfo --> Ending visit and getting trace info...');
return await getFreshTraceRecord(account, profile, traceId, true);
};
/**
*
* @param {*} account
* @param {*} profile
* @param {*} traceId
* @see trace-helper-tests.spec.js test file
*/
static endVisit = async (account, profile, traceId) => {
logger.trace('trace-helper.js --> endVisit --> Start ending visit...');
logger.trace('trace-helper.js --> endVisit --> TraceHelper.getTraceRecord');
const traceRecord = await TraceHelper.getTraceRecord(account, profile, traceId);
logger.trace('trace-helper.js --> endVisit --> Building all visitor Ids available in the trace');
const allTraceVisitorIds = lodash.uniq([
...traceRecord.eventGroups.filter(obj => obj.visitor_profile_before._id).map(obj => obj.visitor_profile_before._id),
...traceRecord.eventGroups.filter(obj => obj.visitor_profile_after._id).map(obj => obj.visitor_profile_after._id)
]);
logger.trace('trace-helper.js --> endVisit --> Visitor Ids found: ', allTraceVisitorIds);
logger.trace('trace-helper.js --> endVisit --> Sending kill session event for each visitor id');
for (let index = 0; index < allTraceVisitorIds.length; index++) {
const id = allTraceVisitorIds[index];
await CdhHelper.sendCollectApiRequest({
tealium_account: account,
tealium_profile: profile,
tealium_visitor_id: id,
tealium_event: 'kill_visitor_session',
tealium_trace_id: traceId,
tealium_random: Utils.getRandomString(),
event: 'kill_visitor_session'
});
await Utils.sleep(400);
}
logger.trace('trace-helper.js --> endVisit --> Finished ending visit...');
};
/**
* Generates an instance of [TraceEvent]{}
*
* @private
*/
static generateTraceEventArray = (traceJson, map) => {
const grouped = {};
traceJson.forEach((el) => {
const key = el.event_id;
if (typeof grouped[key] === 'undefined') {
grouped[key] = {};
if (el && el.step_time) grouped[key].event_time_ms_ts = Date.parse(el.step_time);
}
grouped[key].entries = grouped[key].entries || [];
grouped[key].entries.push(el);
});
const simpleOutput = [];
const keys = Object.keys(grouped);
let lastPrettyEvent;
keys.forEach((key, index, allKeys) => {
let parentEvent = new TraceEvent();
const entries = grouped[key].entries;
entries.forEach((entry) => {
// a very simple summary to help with debugging mostly
const simple = {};
simple.component_type = entry.component_type;
simple.step_name = entry.step_type.name;
switch (entry.component_type) {
case 'MESSAGE_ROUTER':
simple.tealium_event = entry.event_json.data.udo.tealium_event;
parentEvent.tealium_event = entry.event_json.data.udo.tealium_event;
simple.event_id = entry.event_id;
parentEvent.event_id = entry.event_id;
simple.event_time = entry.step_time;
parentEvent.event_time = entry.step_time;
parentEvent.visitor_id = entry.visitor_id;
parentEvent.trace_event_type = entry.event_json.data.udo.tealium_event;
parentEvent.raw_event_json = entry.event_json;
break;
case 'DATA_DISTRIBUTOR':
// both AS and ES qualify here
if (entry.step_type.name === 'ACTION_EXECUTED') {
entry.executed_actions.forEach((action) => {
simple.executed_actions = simple.executed_actions || [];
simple.executed_actions.push(action);
parentEvent.executed_actions = parentEvent.executed_actions || [];
parentEvent.executed_actions.push(action);
});
}
break;
case 'DATA_DISTRIBUTOR_BATCH':
// these are just objects instead of arrays of object, unlike the other actions
if (entry.step_type.name === 'ACTION_QUEUED' || typeof entry.queued_action === 'object') {
simple.queued_actions = simple.queued_actions || [];
simple.queued_actions.push(entry.queued_action); // intentionally singular
parentEvent.queued_actions = parentEvent.queued_actions || [];
parentEvent.queued_actions.push(entry.queued_action); // intentionally singular
}
break;
case 'VISITOR_PROCESSOR':
if (entry.step_type && entry.step_type.name === 'ACTION_TRIGGERED') {
Constants.EVENT_ACTION_TYPES.forEach(type => {
const typeArray = entry[type] || [];
typeArray.forEach((action) => {
simple[type] = simple[type] || [];
simple[type].push(action);
parentEvent[type] = parentEvent[type] || [];
parentEvent[type].push(action);
});
});
}
if (entry.step_type && entry.step_type.name === 'EVENT_PROCESSING') {
if (entry.visitor_profile_after.current_visit.last_event.page_url) {
parentEvent.last_event_url = entry.visitor_profile_after.current_visit.last_event.page_url.full_url;
parentEvent.last_event_querystring = entry.visitor_profile_after.current_visit.last_event.page_url.querystring;
}
parentEvent.lifetime_visit_count = entry.visitor_profile_after.metrics['21'];
parentEvent.lifetime_event_count = entry.visitor_profile_after.metrics['22'];
parentEvent.visitor_profile_before = entry.visitor_profile_before;
parentEvent.visitor_profile_after = entry.visitor_profile_after;
}
if (entry.step_type && entry.step_type.name === 'VISITOR_STITCHED') {
// use the pre-stitch and stitched profiles as the before and after to avoid confusion
parentEvent.visitor_profile_before = entry.visitor_profile;
parentEvent.visitor_profile_after = entry.visitor_profile_stitched;
parentEvent.event_includes_stitch = true;
}
// the visitor profile isn't actually re-included in the Trace output after this stage,
// so we have to add the current audience info
if (entry.step_type && entry.step_type.name === 'AUDIENCE_PROCESSING') {
parentEvent.audiences_joined = parentEvent.audiences_joined.concat(entry.audiences_joined);
parentEvent.audiences_left = parentEvent.audiences_left.concat(entry.audiences_left);
}
break;
default:
simple.is_placeholder = true;
break;
}
// after a SESSION_ENDED pseudo-event, all following actions belong to that pseudo-event
if (entry.step_type.name === 'SESSION_ENDED') {
// finish and push everything collected in the parentEvent so far
const finishedParentEvent = finishParentEvent(parentEvent, lastPrettyEvent);
const prettyEvent = prettifyEvent(finishedParentEvent, map);
lastPrettyEvent = prettyEvent;
simpleOutput.push(prettyEvent);
// make a new parent, reset
parentEvent = new TraceEvent();
// use the defined string as the label for the end of the session
parentEvent.trace_event_type = 'visit_ended';
}
parentEvent.entries.push(simple);
// for debugging and verification
parentEvent.rawEntries.push(entry);
});
// filter out 'unknown' events - don't understand what they're meant to represent?
if (parentEvent.trace_event_type === 'unknown') {
return false;
}
const finishedParentEvent = finishParentEvent(parentEvent, lastPrettyEvent);
const prettyEvent = prettifyEvent(finishedParentEvent, map);
lastPrettyEvent = prettyEvent;
simpleOutput.push(prettyEvent);
});
return simpleOutput;
};
}
module.exports = TraceHelper;