Add idempotency (#6748)

* added idempotency router and middleware

* added idempotency rules for routes classes, functions, jobs, installaions, users

* fixed typo

* ignore requests without header

* removed unused var

* enabled feature only for MongoDB

* changed code comment

* fixed inconsistend storage adapter specification

* Trigger notification

* Travis CI trigger

* Travis CI trigger

* Travis CI trigger

* rebuilt option definitions

* fixed incorrect import path

* added new request ID header to allowed headers

* fixed typescript typos

* add new system class to spec helper

* fixed typescript typos

* re-added postgres conn parameter

* removed postgres conn parameter

* fixed incorrect schema for index creation

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* temporarily disabling index creation to fix postgres issue

* trying to fix postgres issue

* fixed incorrect auth when writing to _Idempotency

* trying to fix postgres issue

* Travis CI trigger

* added test cases

* removed number grouping

* fixed test description

* trying to fix postgres issue

* added Github readme docs

* added change log

* refactored tests; fixed some typos

* fixed test case

* fixed default TTL value

* Travis CI Trigger

* Travis CI Trigger

* Travis CI Trigger

* added test case to increase coverage

* Trigger Travis CI

* changed configuration syntax to use regex; added test cases

* removed unused vars

* removed IdempotencyRouter

* Trigger Travis CI

* updated docs

* updated docs

* updated docs

* updated docs

* update docs

* Trigger Travis CI

* fixed coverage

* removed code comments

src/Controllers/DatabaseController.js

@@ -244,6 +244,7 @@ const filterSensitiveData = (
 };
 
 import type { LoadSchemaOptions } from './types';
+import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
 
 // Runs an update on the database.
 // Returns a promise for an object with the new values for field
@@ -1736,6 +1737,12 @@ class DatabaseController {
         ...SchemaController.defaultColumns._Role,
       },
     };
+    const requiredIdempotencyFields = {
+      fields: {
+        ...SchemaController.defaultColumns._Default,
+        ...SchemaController.defaultColumns._Idempotency,
+      },
+    };
 
     const userClassPromise = this.loadSchema().then(schema =>
       schema.enforceClassExists('_User')
@@ -1743,6 +1750,9 @@ class DatabaseController {
     const roleClassPromise = this.loadSchema().then(schema =>
       schema.enforceClassExists('_Role')
     );
+    const idempotencyClassPromise = this.adapter instanceof MongoStorageAdapter
+      ? this.loadSchema().then((schema) => schema.enforceClassExists('_Idempotency'))
+      : Promise.resolve();
 
     const usernameUniqueness = userClassPromise
       .then(() =>
@@ -1807,6 +1817,43 @@ class DatabaseController {
         throw error;
       });
 
+    const idempotencyRequestIdIndex = this.adapter instanceof MongoStorageAdapter
+      ? idempotencyClassPromise
+        .then(() =>
+          this.adapter.ensureUniqueness(
+            '_Idempotency',
+            requiredIdempotencyFields,
+            ['reqId']
+          ))
+        .catch((error) => {
+          logger.warn(
+            'Unable to ensure uniqueness for idempotency request ID: ',
+            error
+          );
+          throw error;
+        })
+      : Promise.resolve();
+
+    const idempotencyExpireIndex = this.adapter instanceof MongoStorageAdapter
+      ? idempotencyClassPromise
+        .then(() =>
+          this.adapter.ensureIndex(
+            '_Idempotency',
+            requiredIdempotencyFields,
+            ['expire'],
+            'ttl',
+            false,
+            { ttl: 0 },
+          ))
+        .catch((error) => {
+          logger.warn(
+            'Unable to create TTL index for idempotency expire date: ',
+            error
+          );
+          throw error;
+        })
+      : Promise.resolve();
+
     const indexPromise = this.adapter.updateSchemaWithIndexes();
 
     // Create tables for volatile classes
@@ -1819,6 +1866,8 @@ class DatabaseController {
       emailUniqueness,
       emailCaseInsensitiveIndex,
       roleUniqueness,
+      idempotencyRequestIdIndex,
+      idempotencyExpireIndex,
       adapterInit,
       indexPromise,
     ]);

src/Controllers/SchemaController.js

@@ -144,6 +144,10 @@ const defaultColumns: { [string]: SchemaFields } = Object.freeze({
     lastUsed: { type: 'Date' },
     timesUsed: { type: 'Number' },
   },
+  _Idempotency: {
+    reqId: { type: 'String' },
+    expire: { type: 'Date' },
+  }
 });
 
 const requiredColumns = Object.freeze({
@@ -161,6 +165,7 @@ const systemClasses = Object.freeze([
   '_JobStatus',
   '_JobSchedule',
   '_Audience',
+  '_Idempotency'
 ]);
 
 const volatileClasses = Object.freeze([
@@ -171,6 +176,7 @@ const volatileClasses = Object.freeze([
   '_GraphQLConfig',
   '_JobSchedule',
   '_Audience',
+  '_Idempotency'
 ]);
 
 // Anything that start with role
@@ -660,6 +666,13 @@ const _AudienceSchema = convertSchemaToAdapterSchema(
     classLevelPermissions: {},
   })
 );
+const _IdempotencySchema = convertSchemaToAdapterSchema(
+  injectDefaultSchema({
+    className: '_Idempotency',
+    fields: defaultColumns._Idempotency,
+    classLevelPermissions: {},
+  })
+);
 const VolatileClassesSchemas = [
   _HooksSchema,
   _JobStatusSchema,
@@ -668,6 +681,7 @@ const VolatileClassesSchemas = [
   _GlobalConfigSchema,
   _GraphQLConfigSchema,
   _AudienceSchema,
+  _IdempotencySchema
 ];
 
 const dbTypeMatchesObjectType = (