feat: Full Text Search Support (#3904)

* Full Text Support

* invalid input test

* Support for sort

* index exist test

* clean up

* better error messaging

* postgres support

* error instructions for $diacritic and $case sensitivity

* nit

* nit

* nit

* separate test for full text

spec/ParseQuery.FullTextSearch.spec.js

@@ -0,0 +1,459 @@
+'use strict';
+
+const MongoStorageAdapter = require('../src/Adapters/Storage/Mongo/MongoStorageAdapter');
+const mongoURI = 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase';
+const PostgresStorageAdapter = require('../src/Adapters/Storage/Postgres/PostgresStorageAdapter');
+const postgresURI = 'postgres://localhost:5432/parse_server_postgres_adapter_test_database';
+const Parse = require('parse/node');
+const rp = require('request-promise');
+let databaseAdapter;
+
+const fullTextHelper = () => {
+  if (process.env.PARSE_SERVER_TEST_DB === 'postgres') {
+    if (!databaseAdapter) {
+      databaseAdapter = new PostgresStorageAdapter({ uri: postgresURI });
+    }
+  } else {
+    databaseAdapter = new MongoStorageAdapter({ uri: mongoURI });
+  }
+  const subjects = [
+    'coffee',
+    'Coffee Shopping',
+    'Baking a cake',
+    'baking',
+    'Café Con Leche',
+    'Сырники',
+    'coffee and cream',
+    'Cafe con Leche',
+  ];
+  const requests = [];
+  for (const i in subjects) {
+    const request = {
+      method: "POST",
+      body: {
+        subject: subjects[i]
+      },
+      path: "/1/classes/TestObject"
+    };
+    requests.push(request);
+  }
+  return reconfigureServer({
+    appId: 'test',
+    restAPIKey: 'test',
+    publicServerURL: 'http://localhost:8378/1',
+    databaseAdapter
+  }).then(() => {
+    if (process.env.PARSE_SERVER_TEST_DB === 'postgres') {
+      return Parse.Promise.as();
+    }
+    return databaseAdapter.createIndex('TestObject', {subject: 'text'});
+  }).then(() => {
+    return rp.post({
+      url: 'http://localhost:8378/1/batch',
+      body: {
+        requests
+      },
+      json: true,
+      headers: {
+        'X-Parse-Application-Id': 'test',
+        'X-Parse-REST-API-Key': 'test'
+      }
+    });
+  });
+}
+
+describe('Parse.Query Full Text Search testing', () => {
+  it('fullTextSearch: $search', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'coffee'
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(3);
+      done();
+    }, done.fail);
+  });
+
+  it('fullTextSearch: $search, sort', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'coffee'
+            }
+          }
+        }
+      };
+      const order = '$score';
+      const keys = '$score';
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, order, keys, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(3);
+      expect(resp.results[0].score);
+      expect(resp.results[1].score);
+      expect(resp.results[2].score);
+      done();
+    }, done.fail);
+  });
+
+  it('fullTextSearch: $language', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'leche',
+              $language: 'spanish'
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(2);
+      done();
+    }, done.fail);
+  });
+
+  it('fullTextSearch: $diacriticSensitive', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'CAFÉ',
+              $diacriticSensitive: true
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(1);
+      done();
+    }, done.fail);
+  });
+
+  it('fullTextSearch: $search, invalid input', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: true
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`no request should succeed: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+
+  it('fullTextSearch: $language, invalid input', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'leche',
+              $language: true
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`no request should succeed: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+
+  it('fullTextSearch: $caseSensitive, invalid input', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'Coffee',
+              $caseSensitive: 'string'
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`no request should succeed: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+
+  it('fullTextSearch: $diacriticSensitive, invalid input', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'CAFÉ',
+              $diacriticSensitive: 'string'
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`no request should succeed: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+});
+
+describe_only_db('mongo')('Parse.Query Full Text Search testing', () => {
+  it('fullTextSearch: $search, index not exist', (done) => {
+    return reconfigureServer({
+      appId: 'test',
+      restAPIKey: 'test',
+      publicServerURL: 'http://localhost:8378/1',
+      databaseAdapter: new MongoStorageAdapter({ uri: mongoURI })
+    }).then(() => {
+      return rp.post({
+        url: 'http://localhost:8378/1/batch',
+        body: {
+          requests: [
+            {
+              method: "POST",
+              body: {
+                subject: "coffee is java"
+              },
+              path: "/1/classes/TestObject"
+            },
+            {
+              method: "POST",
+              body: {
+                subject: "java is coffee"
+              },
+              path: "/1/classes/TestObject"
+            }
+          ]
+        },
+        json: true,
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'coffee'
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`Text Index should not exist: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INTERNAL_SERVER_ERROR);
+      done();
+    });
+  });
+
+  it('fullTextSearch: $diacriticSensitive - false', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'CAFÉ',
+              $diacriticSensitive: false
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(2);
+      done();
+    }, done.fail);
+  });
+
+  it('fullTextSearch: $caseSensitive', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'Coffee',
+              $caseSensitive: true
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      expect(resp.results.length).toBe(1);
+      done();
+    }, done.fail);
+  });
+});
+
+describe_only_db('postgres')('Parse.Query Full Text Search testing', () => {
+  it('fullTextSearch: $diacriticSensitive - false', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'CAFÉ',
+              $diacriticSensitive: false
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`$diacriticSensitive - false should not supported: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+
+  it('fullTextSearch: $caseSensitive', (done) => {
+    fullTextHelper().then(() => {
+      const where = {
+        subject: {
+          $text: {
+            $search: {
+              $term: 'Coffee',
+              $caseSensitive: true
+            }
+          }
+        }
+      };
+      return rp.post({
+        url: 'http://localhost:8378/1/classes/TestObject',
+        json: { where, '_method': 'GET' },
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-REST-API-Key': 'test'
+        }
+      });
+    }).then((resp) => {
+      fail(`$caseSensitive should not supported: ${JSON.stringify(resp)}`);
+      done();
+    }).catch((err) => {
+      expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
+      done();
+    });
+  });
+});

src/Adapters/Storage/Mongo/MongoCollection.js

@@ -14,6 +14,11 @@ export default class MongoCollection {
   // This could be improved a lot but it's not clear if that's a good
   // idea. Or even if this behavior is a good idea.
   find(query, { skip, limit, sort, keys, maxTimeMS } = {}) {
+    // Support for Full Text Search - $text
+    if(keys && keys.$score) {
+      delete keys.$score;
+      keys.score = {$meta: 'textScore'};
+    }
     return this._rawFind(query, { skip, limit, sort, keys, maxTimeMS })
       .catch(error => {
         // Check for "no geoindex" error

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -393,6 +393,11 @@ export class MongoStorageAdapter {
   performInitialization() {
     return Promise.resolve();
   }
+
+  createIndex(className, index) {
+    return this._adaptiveCollection(className)
+    .then(collection => collection._mongoCollection.createIndex(index));
+  }
 }
 
 export default MongoStorageAdapter;

src/Adapters/Storage/Mongo/MongoTransform.js

@@ -228,6 +228,9 @@ function transformQueryKeyValue(className, key, value, schema) {
   // Handle query constraints
   const transformedConstraint = transformConstraint(value, expectedTypeIsArray);
   if (transformedConstraint !== CannotTransform) {
+    if (transformedConstraint.$text) {
+      return {key: '$text', value: transformedConstraint.$text};
+    }
     return {key, value: transformedConstraint};
   }
 
@@ -576,6 +579,50 @@ function transformConstraint(constraint, inArray) {
       answer[key] = constraint[key];
       break;
 
+    case '$text': {
+      const search = constraint[key].$search;
+      if (typeof search !== 'object') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $search, should be object`
+        );
+      }
+      if (!search.$term || typeof search.$term !== 'string') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $term, should be string`
+        );
+      } else {
+        answer[key] = {
+          '$search': search.$term
+        }
+      }
+      if (search.$language && typeof search.$language !== 'string') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $language, should be string`
+        );
+      } else if (search.$language) {
+        answer[key].$language = search.$language;
+      }
+      if (search.$caseSensitive && typeof search.$caseSensitive !== 'boolean') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $caseSensitive, should be boolean`
+        );
+      } else if (search.$caseSensitive) {
+        answer[key].$caseSensitive = search.$caseSensitive;
+      }
+      if (search.$diacriticSensitive && typeof search.$diacriticSensitive !== 'boolean') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $diacriticSensitive, should be boolean`
+        );
+      } else if (search.$diacriticSensitive) {
+        answer[key].$diacriticSensitive = search.$diacriticSensitive;
+      }
+      break;
+    }
     case '$nearSphere':
       var point = constraint[key];
       answer[key] = [point.longitude, point.latitude];

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -324,6 +324,56 @@ const buildWhereClause = ({ schema, query, index }) => {
       index += 1;
     }
 
+    if (fieldValue.$text) {
+      const search = fieldValue.$text.$search;
+      let language = 'english';
+      if (typeof search !== 'object') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $search, should be object`
+        );
+      }
+      if (!search.$term || typeof search.$term !== 'string') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $term, should be string`
+        );
+      }
+      if (search.$language && typeof search.$language !== 'string') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $language, should be string`
+        );
+      } else if (search.$language) {
+        language = search.$language;
+      }
+      if (search.$caseSensitive && typeof search.$caseSensitive !== 'boolean') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $caseSensitive, should be boolean`
+        );
+      } else if (search.$caseSensitive) {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $caseSensitive not supported, please use $regex or create a separate lower case column.`
+        );
+      }
+      if (search.$diacriticSensitive && typeof search.$diacriticSensitive !== 'boolean') {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $diacriticSensitive, should be boolean`
+        );
+      } else if (search.$diacriticSensitive === false) {
+        throw new Parse.Error(
+          Parse.Error.INVALID_JSON,
+          `bad $text: $diacriticSensitive - false not supported, install Postgres Unaccent Extension`
+        );
+      }
+      patterns.push(`to_tsvector($${index}, $${index + 1}:name) @@ to_tsquery($${index + 2}, $${index + 3})`);
+      values.push(language, fieldName, language, search.$term);
+      index += 4;
+    }
+
     if (fieldValue.$nearSphere) {
       const point = fieldValue.$nearSphere;
       const distance = fieldValue.$maxDistance;
@@ -1084,6 +1134,9 @@ export class PostgresStorageAdapter {
         return key.length > 0;
       });
       columns = keys.map((key, index) => {
+        if (key === '$score') {
+          return `ts_rank_cd(to_tsvector($${2}, $${3}:name), to_tsquery($${4}, $${5}), 32) as score`;
+        }
         return `$${index + values.length + 1}:name`;
       }).join(',');
       values = values.concat(keys);

src/Controllers/DatabaseController.js

@@ -843,7 +843,9 @@ DatabaseController.prototype.find = function(className, query, {
             .then(objects => objects.map(object => {
               object = untransformObjectACL(object);
               return filterSensitiveData(isMaster, aclGroup, className, object)
-            }));
+            })).catch((error) => {
+              throw new Parse.Error(Parse.Error.INTERNAL_SERVER_ERROR, error);
+            });
           }
         }
       });

src/RestQuery.js

@@ -94,7 +94,9 @@ function RestQuery(config, auth, className, restWhere = {}, restOptions = {}, cl
       var fields = restOptions.order.split(',');
       this.findOptions.sort = fields.reduce((sortMap, field) => {
         field = field.trim();
-        if (field[0] == '-') {
+        if (field === '$score') {
+          sortMap.score = {$meta: 'textScore'};
+        } else if (field[0] == '-') {
           sortMap[field.slice(1)] = -1;
         } else {
           sortMap[field] = 1;