joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci: Add test retry logic for flaky tests (#9218)

Browse Source
This commit is contained in:
Diamond Lewis
2024-08-11 07:24:50 -05:00
committed by GitHub
parent 453a9879ef
commit 9fd707092e
5 changed files with 129 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
spec/Idempotency.spec.js
View File

@@ -45,6 +45,10 @@ describe('Idempotency', () => {
}); });
}); });
afterEach(() => {
jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000;
});
// Tests // Tests
it_id('e25955fd-92eb-4b22-b8b7-38980e5cb223')('should enforce idempotency for cloud code function', async () => { it_id('e25955fd-92eb-4b22-b8b7-38980e5cb223')('should enforce idempotency for cloud code function', async () => {
let counter = 0; let counter = 0;

71
spec/RegexVulnerabilities.spec.js
View File

@@ -16,8 +16,13 @@ const emailAdapter = {
const appName = 'test'; const appName = 'test';
const publicServerURL = 'http://localhost:8378/1'; const publicServerURL = 'http://localhost:8378/1';
describe('Regex Vulnerabilities', function () { describe('Regex Vulnerabilities', () => {
beforeEach(async function () { let objectId;
let sessionToken;
let partialSessionToken;
let user;
beforeEach(async () => {
await reconfigureServer({ await reconfigureServer({
maintenanceKey: 'test2', maintenanceKey: 'test2',
verifyUserEmails: true, verifyUserEmails: true,
@@ -38,13 +43,13 @@ describe('Regex Vulnerabilities', function () {
email: 'someemail@somedomain.com', email: 'someemail@somedomain.com',
}), }),
}); });
this.objectId = signUpResponse.data.objectId; objectId = signUpResponse.data.objectId;
this.sessionToken = signUpResponse.data.sessionToken; sessionToken = signUpResponse.data.sessionToken;
this.partialSessionToken = this.sessionToken.slice(0, 3); partialSessionToken = sessionToken.slice(0, 3);
}); });
describe('on session token', function () { describe('on session token', () => {
it('should not work with regex', async function () { it('should not work with regex', async () => {
try { try {
await request({ await request({
url: `${serverURL}/users/me`, url: `${serverURL}/users/me`,
@@ -53,7 +58,7 @@ describe('Regex Vulnerabilities', function () {
body: JSON.stringify({ body: JSON.stringify({
...keys, ...keys,
_SessionToken: { _SessionToken: {
$regex: this.partialSessionToken, $regex: partialSessionToken,
}, },
_method: 'GET', _method: 'GET',
}), }),
@@ -65,43 +70,43 @@ describe('Regex Vulnerabilities', function () {
} }
}); });
it('should work with plain token', async function () { it('should work with plain token', async () => {
const meResponse = await request({ const meResponse = await request({
url: `${serverURL}/users/me`, url: `${serverURL}/users/me`,
method: 'POST', method: 'POST',
headers, headers,
body: JSON.stringify({ body: JSON.stringify({
...keys, ...keys,
_SessionToken: this.sessionToken, _SessionToken: sessionToken,
_method: 'GET', _method: 'GET',
}), }),
}); });
expect(meResponse.data.objectId).toEqual(this.objectId); expect(meResponse.data.objectId).toEqual(objectId);
expect(meResponse.data.sessionToken).toEqual(this.sessionToken); expect(meResponse.data.sessionToken).toEqual(sessionToken);
}); });
}); });
describe('on verify e-mail', function () { describe('on verify e-mail', () => {
beforeEach(async function () { beforeEach(async function () {
const userQuery = new Parse.Query(Parse.User); const userQuery = new Parse.Query(Parse.User);
this.user = await userQuery.get(this.objectId, { useMasterKey: true }); user = await userQuery.get(objectId, { useMasterKey: true });
}); });
it('should not work with regex', async function () { it('should not work with regex', async () => {
expect(this.user.get('emailVerified')).toEqual(false); expect(user.get('emailVerified')).toEqual(false);
await request({ await request({
url: `${serverURL}/apps/test/verify_email?username=someemail@somedomain.com&token[$regex]=`, url: `${serverURL}/apps/test/verify_email?username=someemail@somedomain.com&token[$regex]=`,
method: 'GET', method: 'GET',
}); });
await this.user.fetch({ useMasterKey: true }); await user.fetch({ useMasterKey: true });
expect(this.user.get('emailVerified')).toEqual(false); expect(user.get('emailVerified')).toEqual(false);
}); });
it_id('92bbb86d-bcda-49fa-8d79-aa0501078044')('should work with plain token', async function () { it_id('92bbb86d-bcda-49fa-8d79-aa0501078044')('should work with plain token', async () => {
expect(this.user.get('emailVerified')).toEqual(false); expect(user.get('emailVerified')).toEqual(false);
const current = await request({ const current = await request({
method: 'GET', method: 'GET',
url: `http://localhost:8378/1/classes/_User/${this.user.id}`, url: `http://localhost:8378/1/classes/_User/${user.id}`,
json: true, json: true,
headers: { headers: {
'X-Parse-Application-Id': 'test', 'X-Parse-Application-Id': 'test',
@@ -115,18 +120,18 @@ describe('Regex Vulnerabilities', function () {
url: `${serverURL}/apps/test/verify_email?username=someemail@somedomain.com&token=${current._email_verify_token}`, url: `${serverURL}/apps/test/verify_email?username=someemail@somedomain.com&token=${current._email_verify_token}`,
method: 'GET', method: 'GET',
}); });
await this.user.fetch({ useMasterKey: true }); await user.fetch({ useMasterKey: true });
expect(this.user.get('emailVerified')).toEqual(true); expect(user.get('emailVerified')).toEqual(true);
}); });
}); });
describe('on password reset', function () { describe('on password reset', () => {
beforeEach(async function () { beforeEach(async () => {
this.user = await Parse.User.logIn('someemail@somedomain.com', 'somepassword'); user = await Parse.User.logIn('someemail@somedomain.com', 'somepassword');
}); });
it('should not work with regex', async function () { it('should not work with regex', async () => {
expect(this.user.id).toEqual(this.objectId); expect(user.id).toEqual(objectId);
await request({ await request({
url: `${serverURL}/requestPasswordReset`, url: `${serverURL}/requestPasswordReset`,
method: 'POST', method: 'POST',
@@ -137,7 +142,7 @@ describe('Regex Vulnerabilities', function () {
email: 'someemail@somedomain.com', email: 'someemail@somedomain.com',
}), }),
}); });
await this.user.fetch({ useMasterKey: true }); await user.fetch({ useMasterKey: true });
const passwordResetResponse = await request({ const passwordResetResponse = await request({
url: `${serverURL}/apps/test/request_password_reset?username=someemail@somedomain.com&token[$regex]=`, url: `${serverURL}/apps/test/request_password_reset?username=someemail@somedomain.com&token[$regex]=`,
method: 'GET', method: 'GET',
@@ -162,8 +167,8 @@ describe('Regex Vulnerabilities', function () {
} }
}); });
it('should work with plain token', async function () { it('should work with plain token', async () => {
expect(this.user.id).toEqual(this.objectId); expect(user.id).toEqual(objectId);
await request({ await request({
url: `${serverURL}/requestPasswordReset`, url: `${serverURL}/requestPasswordReset`,
method: 'POST', method: 'POST',
@@ -176,7 +181,7 @@ describe('Regex Vulnerabilities', function () {
}); });
const current = await request({ const current = await request({
method: 'GET', method: 'GET',
url: `http://localhost:8378/1/classes/_User/${this.user.id}`, url: `http://localhost:8378/1/classes/_User/${user.id}`,
json: true, json: true,
headers: { headers: {
'X-Parse-Application-Id': 'test', 'X-Parse-Application-Id': 'test',
@@ -204,7 +209,7 @@ describe('Regex Vulnerabilities', function () {
}, },
}); });
const userAgain = await Parse.User.logIn('someemail@somedomain.com', 'newpassword'); const userAgain = await Parse.User.logIn('someemail@somedomain.com', 'newpassword');
expect(userAgain.id).toEqual(this.objectId); expect(userAgain.id).toEqual(objectId);
}); });
}); });
}); });

3
spec/helper.js
View File

@@ -14,6 +14,7 @@ if (dns.setDefaultResultOrder) {
jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000; jasmine.DEFAULT_TIMEOUT_INTERVAL = process.env.PARSE_SERVER_TEST_TIMEOUT || 10000;
jasmine.getEnv().addReporter(new CurrentSpecReporter()); jasmine.getEnv().addReporter(new CurrentSpecReporter());
jasmine.getEnv().addReporter(new SpecReporter()); jasmine.getEnv().addReporter(new SpecReporter());
global.retryFlakyTests();
global.on_db = (db, callback, elseCallback) => { global.on_db = (db, callback, elseCallback) => {
if (process.env.PARSE_SERVER_TEST_DB == db) { if (process.env.PARSE_SERVER_TEST_DB == db) {
@@ -287,7 +288,7 @@ afterEach(function (done) {
}); });
afterAll(() => { afterAll(() => {
global.displaySlowTests(); global.displayTestStats();
}); });
const TestObject = Parse.Object.extend({ const TestObject = Parse.Object.extend({

91
spec/support/CurrentSpecReporter.js
View File

@@ -1,13 +1,36 @@
// Sets a global variable to the current test spec // Sets a global variable to the current test spec
// ex: global.currentSpec.description // ex: global.currentSpec.description
const { performance } = require('perf_hooks'); const { performance } = require('perf_hooks');
global.currentSpec = null; global.currentSpec = null;
const timerMap = {}; /**
const duplicates = []; * Names of tests that fail randomly and are considered flaky. These tests will be retried
* a number of times to reduce the chance of false negatives. The test name must be the same
* as the one displayed in the CI log test output.
*/
const flakyTests = [
// Timeout
"ParseLiveQuery handle invalid websocket payload length",
// Unhandled promise rejection: TypeError: message.split is not a function
"rest query query internal field",
// TypeError: Cannot read properties of undefined (reading 'link')
"UserController sendVerificationEmail parseFrameURL not provided uses publicServerURL",
// TypeError: Cannot read properties of undefined (reading 'link')
"UserController sendVerificationEmail parseFrameURL provided uses parseFrameURL and includes the destination in the link parameter",
// Expected undefined to be defined
"Email Verification Token Expiration: sets the _email_verify_token_expires_at and _email_verify_token fields after user SignUp",
];
/** The minimum execution time in seconds for a test to be considered slow. */ /** The minimum execution time in seconds for a test to be considered slow. */
const slowTestLimit = 2; const slowTestLimit = 2;
/** The number of times to retry a flaky test. */
const retries = 5;
const timerMap = {};
const retryMap = {};
const duplicates = [];
class CurrentSpecReporter { class CurrentSpecReporter {
specStarted(spec) { specStarted(spec) {
if (timerMap[spec.fullName]) { if (timerMap[spec.fullName]) {
@@ -26,20 +49,74 @@ class CurrentSpecReporter {
global.currentSpec = null; global.currentSpec = null;
} }
} }
global.displaySlowTests = function() {
const times = Object.values(timerMap).sort((a,b) => b - a); global.displayTestStats = function() {
const times = Object.values(timerMap).sort((a,b) => b - a).filter(time => time >= slowTestLimit);
if (times.length > 0) { if (times.length > 0) {
console.log(`Slow tests with execution time >=${slowTestLimit}s:`); console.log(`Slow tests with execution time >=${slowTestLimit}s:`);
} }
times.forEach((time) => { times.forEach((time) => {
if (time >= slowTestLimit) { console.warn(`${time.toFixed(1)}s:`, Object.keys(timerMap).find(key => timerMap[key] === time));
console.warn(`${time.toFixed(1)}s:`, Object.keys(timerMap).find(key => timerMap[key] === time));
}
}); });
console.log('\n'); console.log('\n');
duplicates.forEach((spec) => { duplicates.forEach((spec) => {
console.warn('Duplicate spec: ' + spec); console.warn('Duplicate spec: ' + spec);
}); });
console.log('\n');
Object.keys(retryMap).forEach((spec) => {
console.warn(`Flaky test: ${spec} failed ${retryMap[spec]} times`);
});
console.log('\n');
}; };
global.retryFlakyTests = function() {
const originalSpecConstructor = jasmine.Spec;
jasmine.Spec = function(attrs) {
const spec = new originalSpecConstructor(attrs);
const originalTestFn = spec.queueableFn.fn;
const runOriginalTest = () => {
if (originalTestFn.length == 0) {
// handle async testing
return originalTestFn();
} else {
// handle done() callback
return new Promise((resolve) => {
originalTestFn(resolve);
});
}
};
spec.queueableFn.fn = async function() {
const isFlaky = flakyTests.includes(spec.result.fullName);
const runs = isFlaky ? retries : 1;
let exceptionCaught;
let returnValue;
for (let i = 0; i < runs; ++i) {
spec.result.failedExpectations = [];
returnValue = undefined;
exceptionCaught = undefined;
try {
returnValue = await runOriginalTest();
} catch (exception) {
exceptionCaught = exception;
}
const failed = !spec.markedPending &&
(exceptionCaught || spec.result.failedExpectations.length != 0);
if (!failed) {
break;
}
if (isFlaky) {
retryMap[spec.result.fullName] = (retryMap[spec.result.fullName] || 0) + 1;
}
}
if (exceptionCaught) {
throw exceptionCaught;
}
return returnValue;
};
return spec;
};
}
module.exports = CurrentSpecReporter; module.exports = CurrentSpecReporter;

2
spec/support/jasmine.json
View File

@@ -2,5 +2,5 @@
"spec_dir": "spec", "spec_dir": "spec",
"spec_files": ["*spec.js"], "spec_files": ["*spec.js"],
"helpers": ["helper.js"], "helpers": ["helper.js"],
"random": false "random": true
} }