feat: Full Text Search Support (#3904)
* Full Text Support * invalid input test * Support for sort * index exist test * clean up * better error messaging * postgres support * error instructions for $diacritic and $case sensitivity * nit * nit * nit * separate test for full text
This commit is contained in:
Diamond Lewis
committed by
Florent Vilmart
Florent Vilmart
parent
5f991e90fb
commit
8b21d5ab80
459
spec/ParseQuery.FullTextSearch.spec.js
Normal file
459
spec/ParseQuery.FullTextSearch.spec.js
Normal file
@@ -0,0 +1,459 @@
|
||||
'use strict';
|
||||
|
||||
const MongoStorageAdapter = require('../src/Adapters/Storage/Mongo/MongoStorageAdapter');
|
||||
const mongoURI = 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase';
|
||||
const PostgresStorageAdapter = require('../src/Adapters/Storage/Postgres/PostgresStorageAdapter');
|
||||
const postgresURI = 'postgres://localhost:5432/parse_server_postgres_adapter_test_database';
|
||||
const Parse = require('parse/node');
|
||||
const rp = require('request-promise');
|
||||
let databaseAdapter;
|
||||
|
||||
const fullTextHelper = () => {
|
||||
if (process.env.PARSE_SERVER_TEST_DB === 'postgres') {
|
||||
if (!databaseAdapter) {
|
||||
databaseAdapter = new PostgresStorageAdapter({ uri: postgresURI });
|
||||
}
|
||||
} else {
|
||||
databaseAdapter = new MongoStorageAdapter({ uri: mongoURI });
|
||||
}
|
||||
const subjects = [
|
||||
'coffee',
|
||||
'Coffee Shopping',
|
||||
'Baking a cake',
|
||||
'baking',
|
||||
'Café Con Leche',
|
||||
'Сырники',
|
||||
'coffee and cream',
|
||||
'Cafe con Leche',
|
||||
];
|
||||
const requests = [];
|
||||
for (const i in subjects) {
|
||||
const request = {
|
||||
method: "POST",
|
||||
body: {
|
||||
subject: subjects[i]
|
||||
},
|
||||
path: "/1/classes/TestObject"
|
||||
};
|
||||
requests.push(request);
|
||||
}
|
||||
return reconfigureServer({
|
||||
appId: 'test',
|
||||
restAPIKey: 'test',
|
||||
publicServerURL: 'http://localhost:8378/1',
|
||||
databaseAdapter
|
||||
}).then(() => {
|
||||
if (process.env.PARSE_SERVER_TEST_DB === 'postgres') {
|
||||
return Parse.Promise.as();
|
||||
}
|
||||
return databaseAdapter.createIndex('TestObject', {subject: 'text'});
|
||||
}).then(() => {
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/batch',
|
||||
body: {
|
||||
requests
|
||||
},
|
||||
json: true,
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
describe('Parse.Query Full Text Search testing', () => {
|
||||
it('fullTextSearch: $search', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'coffee'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(3);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
|
||||
it('fullTextSearch: $search, sort', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'coffee'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
const order = '$score';
|
||||
const keys = '$score';
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, order, keys, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(3);
|
||||
expect(resp.results[0].score);
|
||||
expect(resp.results[1].score);
|
||||
expect(resp.results[2].score);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
|
||||
it('fullTextSearch: $language', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'leche',
|
||||
$language: 'spanish'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(2);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
|
||||
it('fullTextSearch: $diacriticSensitive', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'CAFÉ',
|
||||
$diacriticSensitive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(1);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
|
||||
it('fullTextSearch: $search, invalid input', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: true
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`no request should succeed: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('fullTextSearch: $language, invalid input', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'leche',
|
||||
$language: true
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`no request should succeed: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('fullTextSearch: $caseSensitive, invalid input', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'Coffee',
|
||||
$caseSensitive: 'string'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`no request should succeed: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('fullTextSearch: $diacriticSensitive, invalid input', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'CAFÉ',
|
||||
$diacriticSensitive: 'string'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`no request should succeed: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe_only_db('mongo')('Parse.Query Full Text Search testing', () => {
|
||||
it('fullTextSearch: $search, index not exist', (done) => {
|
||||
return reconfigureServer({
|
||||
appId: 'test',
|
||||
restAPIKey: 'test',
|
||||
publicServerURL: 'http://localhost:8378/1',
|
||||
databaseAdapter: new MongoStorageAdapter({ uri: mongoURI })
|
||||
}).then(() => {
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/batch',
|
||||
body: {
|
||||
requests: [
|
||||
{
|
||||
method: "POST",
|
||||
body: {
|
||||
subject: "coffee is java"
|
||||
},
|
||||
path: "/1/classes/TestObject"
|
||||
},
|
||||
{
|
||||
method: "POST",
|
||||
body: {
|
||||
subject: "java is coffee"
|
||||
},
|
||||
path: "/1/classes/TestObject"
|
||||
}
|
||||
]
|
||||
},
|
||||
json: true,
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'coffee'
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`Text Index should not exist: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INTERNAL_SERVER_ERROR);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('fullTextSearch: $diacriticSensitive - false', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'CAFÉ',
|
||||
$diacriticSensitive: false
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(2);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
|
||||
it('fullTextSearch: $caseSensitive', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'Coffee',
|
||||
$caseSensitive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
expect(resp.results.length).toBe(1);
|
||||
done();
|
||||
}, done.fail);
|
||||
});
|
||||
});
|
||||
|
||||
describe_only_db('postgres')('Parse.Query Full Text Search testing', () => {
|
||||
it('fullTextSearch: $diacriticSensitive - false', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'CAFÉ',
|
||||
$diacriticSensitive: false
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`$diacriticSensitive - false should not supported: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('fullTextSearch: $caseSensitive', (done) => {
|
||||
fullTextHelper().then(() => {
|
||||
const where = {
|
||||
subject: {
|
||||
$text: {
|
||||
$search: {
|
||||
$term: 'Coffee',
|
||||
$caseSensitive: true
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
return rp.post({
|
||||
url: 'http://localhost:8378/1/classes/TestObject',
|
||||
json: { where, '_method': 'GET' },
|
||||
headers: {
|
||||
'X-Parse-Application-Id': 'test',
|
||||
'X-Parse-REST-API-Key': 'test'
|
||||
}
|
||||
});
|
||||
}).then((resp) => {
|
||||
fail(`$caseSensitive should not supported: ${JSON.stringify(resp)}`);
|
||||
done();
|
||||
}).catch((err) => {
|
||||
expect(err.error.code).toEqual(Parse.Error.INVALID_JSON);
|
||||
done();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,11 @@ export default class MongoCollection {
|
||||
// This could be improved a lot but it's not clear if that's a good
|
||||
// idea. Or even if this behavior is a good idea.
|
||||
find(query, { skip, limit, sort, keys, maxTimeMS } = {}) {
|
||||
// Support for Full Text Search - $text
|
||||
if(keys && keys.$score) {
|
||||
delete keys.$score;
|
||||
keys.score = {$meta: 'textScore'};
|
||||
}
|
||||
return this._rawFind(query, { skip, limit, sort, keys, maxTimeMS })
|
||||
.catch(error => {
|
||||
// Check for "no geoindex" error
|
||||
|
||||
@@ -393,6 +393,11 @@ export class MongoStorageAdapter {
|
||||
performInitialization() {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
createIndex(className, index) {
|
||||
return this._adaptiveCollection(className)
|
||||
.then(collection => collection._mongoCollection.createIndex(index));
|
||||
}
|
||||
}
|
||||
|
||||
export default MongoStorageAdapter;
|
||||
|
||||
@@ -228,6 +228,9 @@ function transformQueryKeyValue(className, key, value, schema) {
|
||||
// Handle query constraints
|
||||
const transformedConstraint = transformConstraint(value, expectedTypeIsArray);
|
||||
if (transformedConstraint !== CannotTransform) {
|
||||
if (transformedConstraint.$text) {
|
||||
return {key: '$text', value: transformedConstraint.$text};
|
||||
}
|
||||
return {key, value: transformedConstraint};
|
||||
}
|
||||
|
||||
@@ -576,6 +579,50 @@ function transformConstraint(constraint, inArray) {
|
||||
answer[key] = constraint[key];
|
||||
break;
|
||||
|
||||
case '$text': {
|
||||
const search = constraint[key].$search;
|
||||
if (typeof search !== 'object') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $search, should be object`
|
||||
);
|
||||
}
|
||||
if (!search.$term || typeof search.$term !== 'string') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $term, should be string`
|
||||
);
|
||||
} else {
|
||||
answer[key] = {
|
||||
'$search': search.$term
|
||||
}
|
||||
}
|
||||
if (search.$language && typeof search.$language !== 'string') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $language, should be string`
|
||||
);
|
||||
} else if (search.$language) {
|
||||
answer[key].$language = search.$language;
|
||||
}
|
||||
if (search.$caseSensitive && typeof search.$caseSensitive !== 'boolean') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $caseSensitive, should be boolean`
|
||||
);
|
||||
} else if (search.$caseSensitive) {
|
||||
answer[key].$caseSensitive = search.$caseSensitive;
|
||||
}
|
||||
if (search.$diacriticSensitive && typeof search.$diacriticSensitive !== 'boolean') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $diacriticSensitive, should be boolean`
|
||||
);
|
||||
} else if (search.$diacriticSensitive) {
|
||||
answer[key].$diacriticSensitive = search.$diacriticSensitive;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case '$nearSphere':
|
||||
var point = constraint[key];
|
||||
answer[key] = [point.longitude, point.latitude];
|
||||
|
||||
@@ -324,6 +324,56 @@ const buildWhereClause = ({ schema, query, index }) => {
|
||||
index += 1;
|
||||
}
|
||||
|
||||
if (fieldValue.$text) {
|
||||
const search = fieldValue.$text.$search;
|
||||
let language = 'english';
|
||||
if (typeof search !== 'object') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $search, should be object`
|
||||
);
|
||||
}
|
||||
if (!search.$term || typeof search.$term !== 'string') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $term, should be string`
|
||||
);
|
||||
}
|
||||
if (search.$language && typeof search.$language !== 'string') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $language, should be string`
|
||||
);
|
||||
} else if (search.$language) {
|
||||
language = search.$language;
|
||||
}
|
||||
if (search.$caseSensitive && typeof search.$caseSensitive !== 'boolean') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $caseSensitive, should be boolean`
|
||||
);
|
||||
} else if (search.$caseSensitive) {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $caseSensitive not supported, please use $regex or create a separate lower case column.`
|
||||
);
|
||||
}
|
||||
if (search.$diacriticSensitive && typeof search.$diacriticSensitive !== 'boolean') {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $diacriticSensitive, should be boolean`
|
||||
);
|
||||
} else if (search.$diacriticSensitive === false) {
|
||||
throw new Parse.Error(
|
||||
Parse.Error.INVALID_JSON,
|
||||
`bad $text: $diacriticSensitive - false not supported, install Postgres Unaccent Extension`
|
||||
);
|
||||
}
|
||||
patterns.push(`to_tsvector($${index}, $${index + 1}:name) @@ to_tsquery($${index + 2}, $${index + 3})`);
|
||||
values.push(language, fieldName, language, search.$term);
|
||||
index += 4;
|
||||
}
|
||||
|
||||
if (fieldValue.$nearSphere) {
|
||||
const point = fieldValue.$nearSphere;
|
||||
const distance = fieldValue.$maxDistance;
|
||||
@@ -1084,6 +1134,9 @@ export class PostgresStorageAdapter {
|
||||
return key.length > 0;
|
||||
});
|
||||
columns = keys.map((key, index) => {
|
||||
if (key === '$score') {
|
||||
return `ts_rank_cd(to_tsvector($${2}, $${3}:name), to_tsquery($${4}, $${5}), 32) as score`;
|
||||
}
|
||||
return `$${index + values.length + 1}:name`;
|
||||
}).join(',');
|
||||
values = values.concat(keys);
|
||||
|
||||
@@ -843,7 +843,9 @@ DatabaseController.prototype.find = function(className, query, {
|
||||
.then(objects => objects.map(object => {
|
||||
object = untransformObjectACL(object);
|
||||
return filterSensitiveData(isMaster, aclGroup, className, object)
|
||||
}));
|
||||
})).catch((error) => {
|
||||
throw new Parse.Error(Parse.Error.INTERNAL_SERVER_ERROR, error);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -94,7 +94,9 @@ function RestQuery(config, auth, className, restWhere = {}, restOptions = {}, cl
|
||||
var fields = restOptions.order.split(',');
|
||||
this.findOptions.sort = fields.reduce((sortMap, field) => {
|
||||
field = field.trim();
|
||||
if (field[0] == '-') {
|
||||
if (field === '$score') {
|
||||
sortMap.score = {$meta: 'textScore'};
|
||||
} else if (field[0] == '-') {
|
||||
sortMap[field.slice(1)] = -1;
|
||||
} else {
|
||||
sortMap[field] = 1;
|
||||
|
||||
Reference in New Issue
Block a user