feat: Disable index-field validation to create index for fields that don't yet exist (#8137)

spec/GridFSBucketStorageAdapter.spec.js

@@ -421,6 +421,14 @@ describe_only_db('mongo')('GridFSBucket', () => {
     expect(gfsResult.toString('utf8')).toBe(twoMegabytesFile);
   });
 
+  it('properly upload a file when disableIndexFieldValidation exist in databaseOptions', async () => {
+    const gfsAdapter = new GridFSBucketAdapter(databaseURI, { disableIndexFieldValidation: true });
+    const twoMegabytesFile = randomString(2048 * 1024);
+    await gfsAdapter.createFile('myFileName', twoMegabytesFile);
+    const gfsResult = await gfsAdapter.getFileData('myFileName');
+    expect(gfsResult.toString('utf8')).toBe(twoMegabytesFile);
+  });
+
   it('properly deletes a file from GridFS', async () => {
     const gfsAdapter = new GridFSBucketAdapter(databaseURI);
     await gfsAdapter.createFile('myFileName', 'a simple file');

spec/schemas.spec.js

@@ -3008,6 +3008,7 @@ describe('schemas', () => {
     beforeEach(async () => {
       await TestUtils.destroyAllDataPermanently(false);
       await config.database.adapter.performInitialization({ VolatileClassesSchemas: [] });
+      databaseAdapter.disableIndexFieldValidation = false;
     });
 
     it('cannot create index if field does not exist', done => {
@@ -3036,6 +3037,29 @@ describe('schemas', () => {
       });
     });
 
+    it('can create index if field does not exist with disableIndexFieldValidation true ', async () => {
+      databaseAdapter.disableIndexFieldValidation = true;
+      await request({
+        url: 'http://localhost:8378/1/schemas/NewClass',
+        method: 'POST',
+        headers: masterKeyHeaders,
+        json: true,
+        body: {},
+      });
+      const response = await request({
+        url: 'http://localhost:8378/1/schemas/NewClass',
+        method: 'PUT',
+        headers: masterKeyHeaders,
+        json: true,
+        body: {
+          indexes: {
+            name1: { aString: 1 },
+          },
+        },
+      });
+      expect(response.data.indexes.name1).toEqual({ aString: 1 });
+    });
+
     it('can create index on default field', done => {
       request({
         url: 'http://localhost:8378/1/schemas/NewClass',

src/Adapters/Files/GridFSBucketAdapter.js

@@ -37,7 +37,7 @@ export class GridFSBucketAdapter extends FilesAdapter {
     const defaultMongoOptions = {
     };
     const _mongoOptions = Object.assign(defaultMongoOptions, mongoOptions);
-    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS']) {
+    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS', 'disableIndexFieldValidation']) {
       delete _mongoOptions[key];
     }
     this._mongoOptions = _mongoOptions;

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -140,6 +140,7 @@ export class MongoStorageAdapter implements StorageAdapter {
   canSortOnJoinTables: boolean;
   enableSchemaHooks: boolean;
   schemaCacheTtl: ?number;
+  disableIndexFieldValidation: boolean;
 
   constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) {
     this._uri = uri;
@@ -152,7 +153,8 @@ export class MongoStorageAdapter implements StorageAdapter {
     this.canSortOnJoinTables = true;
     this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks;
     this.schemaCacheTtl = mongoOptions.schemaCacheTtl;
-    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS']) {
+    this.disableIndexFieldValidation = !!mongoOptions.disableIndexFieldValidation;
+    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'maxTimeMS', 'disableIndexFieldValidation']) {
       delete mongoOptions[key];
       delete this._mongoOptions[key];
     }
@@ -289,6 +291,7 @@ export class MongoStorageAdapter implements StorageAdapter {
       } else {
         Object.keys(field).forEach(key => {
           if (
+            !this.disableIndexFieldValidation &&
             !Object.prototype.hasOwnProperty.call(
               fields,
               key.indexOf('_p_') === 0 ? key.replace('_p_', '') : key

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -627,13 +627,11 @@ const buildWhereClause = ({ schema, query, index, caseInsensitive }): WhereClaus
       const distance = fieldValue.$maxDistance;
       const distanceInKM = distance * 6371 * 1000;
       patterns.push(
-        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${
+        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2
-          index + 2
         })::geometry) <= $${index + 3}`
       );
       sorts.push(
-        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${
+        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2
-          index + 2
         })::geometry) ASC`
       );
       values.push(fieldName, point.longitude, point.latitude, distanceInKM);
@@ -681,8 +679,7 @@ const buildWhereClause = ({ schema, query, index, caseInsensitive }): WhereClaus
       }
       const distanceInKM = distance * 6371 * 1000;
       patterns.push(
-        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${
+        `ST_DistanceSphere($${index}:name::geometry, POINT($${index + 1}, $${index + 2
-          index + 2
         })::geometry) <= $${index + 3}`
       );
       values.push(fieldName, point.longitude, point.latitude, distanceInKM);
@@ -862,19 +859,22 @@ export class PostgresStorageAdapter implements StorageAdapter {
   _stream: any;
   _uuid: any;
   schemaCacheTtl: ?number;
+  disableIndexFieldValidation: boolean;
 
   constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) {
     const options = { ...databaseOptions };
     this._collectionPrefix = collectionPrefix;
     this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks;
+    this.disableIndexFieldValidation = !!databaseOptions.disableIndexFieldValidation;
+
     this.schemaCacheTtl = databaseOptions.schemaCacheTtl;
-    for (const key of ['enableSchemaHooks', 'schemaCacheTtl']) {
+    for (const key of ['enableSchemaHooks', 'schemaCacheTtl', 'disableIndexFieldValidation']) {
       delete options[key];
     }
 
     const { client, pgp } = createClient(uri, options);
     this._client = client;
-    this._onchange = () => {};
+    this._onchange = () => { };
     this._pgp = pgp;
     this._uuid = uuidv4();
     this.canSortOnJoinTables = false;
@@ -991,7 +991,10 @@ export class PostgresStorageAdapter implements StorageAdapter {
         delete existingIndexes[name];
       } else {
         Object.keys(field).forEach(key => {
-          if (!Object.prototype.hasOwnProperty.call(fields, key)) {
+          if (
+            !this.disableIndexFieldValidation &&
+            !Object.prototype.hasOwnProperty.call(fields, key)
+          ) {
             throw new Parse.Error(
               Parse.Error.INVALID_QUERY,
               `Field ${key} does not exist, cannot add index.`
@@ -1006,8 +1009,22 @@ export class PostgresStorageAdapter implements StorageAdapter {
       }
     });
     await conn.tx('set-indexes-with-schema-format', async t => {
-      if (insertedIndexes.length > 0) {
+      try {
-        await self.createIndexes(className, insertedIndexes, t);
+        if (insertedIndexes.length > 0) {
+          await self.createIndexes(className, insertedIndexes, t);
+        }
+      } catch (e) {
+        // pg-promise use Batch error see https://github.com/vitaly-t/spex/blob/e572030f261be1a8e9341fc6f637e36ad07f5231/src/errors/batch.js#L59
+        const columnDoesNotExistError = e.getErrors && e.getErrors()[0] && e.getErrors()[0].code === '42703';
+        // Specific case when the column does not exist
+        if (columnDoesNotExistError) {
+          // If the disableIndexFieldValidation is true, we should ignore the error
+          if (!this.disableIndexFieldValidation) {
+            throw e;
+          }
+        } else {
+          throw e;
+        }
       }
       if (deletedIndexes.length > 0) {
         await self.dropIndexes(className, deletedIndexes, t);
@@ -1625,16 +1642,14 @@ export class PostgresStorageAdapter implements StorageAdapter {
         index += 2;
       } else if (fieldValue.__op === 'Remove') {
         updatePatterns.push(
-          `$${index}:name = array_remove(COALESCE($${index}:name, '[]'::jsonb), $${
+          `$${index}:name = array_remove(COALESCE($${index}:name, '[]'::jsonb), $${index + 1
-            index + 1
           }::jsonb)`
         );
         values.push(fieldName, JSON.stringify(fieldValue.objects));
         index += 2;
       } else if (fieldValue.__op === 'AddUnique') {
         updatePatterns.push(
-          `$${index}:name = array_add_unique(COALESCE($${index}:name, '[]'::jsonb), $${
+          `$${index}:name = array_add_unique(COALESCE($${index}:name, '[]'::jsonb), $${index + 1
-            index + 1
           }::jsonb)`
         );
         values.push(fieldName, JSON.stringify(fieldValue.objects));
@@ -1745,8 +1760,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
           updateObject = `COALESCE($${index}:name, '{}'::jsonb)`;
         }
         updatePatterns.push(
-          `$${index}:name = (${updateObject} ${deletePatterns} ${incrementPatterns} || $${
+          `$${index}:name = (${updateObject} ${deletePatterns} ${incrementPatterns} || $${index + 1 + keysToDelete.length
-            index + 1 + keysToDelete.length
           }::jsonb )`
         );
         values.push(fieldName, ...keysToDelete, JSON.stringify(fieldValue));
@@ -2185,8 +2199,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
                     groupByFields.push(`"${source}"`);
                   }
                   columns.push(
-                    `EXTRACT(${
+                    `EXTRACT(${mongoAggregateToPostgres[operation]
-                      mongoAggregateToPostgres[operation]
                     } FROM $${index}:name AT TIME ZONE 'UTC')::integer AS $${index + 1}:name`
                   );
                   values.push(source, alias);