127 lines
4.3 KiB
JavaScript
Executable File
127 lines
4.3 KiB
JavaScript
Executable File
// Sets a global variable to the current test spec
|
|
// ex: global.currentSpec.description
|
|
const { performance } = require('perf_hooks');
|
|
|
|
global.currentSpec = null;
|
|
|
|
/**
|
|
* Names of tests that fail randomly and are considered flaky. These tests will be retried
|
|
* a number of times to reduce the chance of false negatives. The test name must be the same
|
|
* as the one displayed in the CI log test output.
|
|
*/
|
|
const flakyTests = [
|
|
// Timeout
|
|
"ParseLiveQuery handle invalid websocket payload length",
|
|
// Unhandled promise rejection: TypeError: message.split is not a function
|
|
"rest query query internal field",
|
|
// TypeError: Cannot read properties of undefined (reading 'link')
|
|
"UserController sendVerificationEmail parseFrameURL not provided uses publicServerURL",
|
|
// TypeError: Cannot read properties of undefined (reading 'link')
|
|
"UserController sendVerificationEmail parseFrameURL provided uses parseFrameURL and includes the destination in the link parameter",
|
|
// Expected undefined to be defined
|
|
"Email Verification Token Expiration: sets the _email_verify_token_expires_at and _email_verify_token fields after user SignUp",
|
|
// Expected 0 to be 1.
|
|
"Email Verification Token Expiration: should send a new verification email when a resend is requested and the user is UNVERIFIED",
|
|
// Expected 0 to be 1.
|
|
"Email Verification Token Expiration: should match codes with emailVerifyTokenReuseIfValid",
|
|
];
|
|
|
|
/** The minimum execution time in seconds for a test to be considered slow. */
|
|
const slowTestLimit = 2;
|
|
|
|
/** The number of times to retry a flaky test. */
|
|
const retries = 5;
|
|
|
|
const timerMap = {};
|
|
const retryMap = {};
|
|
const duplicates = [];
|
|
class CurrentSpecReporter {
|
|
specStarted(spec) {
|
|
if (timerMap[spec.fullName]) {
|
|
console.log('Duplicate spec: ' + spec.fullName);
|
|
duplicates.push(spec.fullName);
|
|
}
|
|
timerMap[spec.fullName] = performance.now();
|
|
global.currentSpec = spec;
|
|
}
|
|
specDone(result) {
|
|
if (result.status === 'excluded') {
|
|
delete timerMap[result.fullName];
|
|
return;
|
|
}
|
|
timerMap[result.fullName] = (performance.now() - timerMap[result.fullName]) / 1000;
|
|
global.currentSpec = null;
|
|
}
|
|
}
|
|
|
|
global.displayTestStats = function() {
|
|
const times = Object.values(timerMap).sort((a,b) => b - a).filter(time => time >= slowTestLimit);
|
|
if (times.length > 0) {
|
|
console.log(`Slow tests with execution time >=${slowTestLimit}s:`);
|
|
}
|
|
times.forEach((time) => {
|
|
console.warn(`${time.toFixed(1)}s:`, Object.keys(timerMap).find(key => timerMap[key] === time));
|
|
});
|
|
console.log('\n');
|
|
duplicates.forEach((spec) => {
|
|
console.warn('Duplicate spec: ' + spec);
|
|
});
|
|
console.log('\n');
|
|
Object.keys(retryMap).forEach((spec) => {
|
|
console.warn(`Flaky test: ${spec} failed ${retryMap[spec]} times`);
|
|
});
|
|
console.log('\n');
|
|
};
|
|
|
|
global.retryFlakyTests = function() {
|
|
const originalSpecConstructor = jasmine.Spec;
|
|
|
|
jasmine.Spec = function(attrs) {
|
|
const spec = new originalSpecConstructor(attrs);
|
|
const originalTestFn = spec.queueableFn.fn;
|
|
const runOriginalTest = () => {
|
|
if (originalTestFn.length == 0) {
|
|
// handle async testing
|
|
return originalTestFn();
|
|
} else {
|
|
// handle done() callback
|
|
return new Promise((resolve) => {
|
|
originalTestFn(resolve);
|
|
});
|
|
}
|
|
};
|
|
spec.queueableFn.fn = async function() {
|
|
const isFlaky = flakyTests.includes(spec.result.fullName);
|
|
const runs = isFlaky ? retries : 1;
|
|
let exceptionCaught;
|
|
let returnValue;
|
|
|
|
for (let i = 0; i < runs; ++i) {
|
|
spec.result.failedExpectations = [];
|
|
returnValue = undefined;
|
|
exceptionCaught = undefined;
|
|
try {
|
|
returnValue = await runOriginalTest();
|
|
} catch (exception) {
|
|
exceptionCaught = exception;
|
|
}
|
|
const failed = !spec.markedPending &&
|
|
(exceptionCaught || spec.result.failedExpectations.length != 0);
|
|
if (!failed) {
|
|
break;
|
|
}
|
|
if (isFlaky) {
|
|
retryMap[spec.result.fullName] = (retryMap[spec.result.fullName] || 0) + 1;
|
|
await global.afterEachFn();
|
|
}
|
|
}
|
|
if (exceptionCaught) {
|
|
throw exceptionCaught;
|
|
}
|
|
return returnValue;
|
|
};
|
|
return spec;
|
|
};
|
|
}
|
|
|
|
module.exports = CurrentSpecReporter; |