Support for Aggregate Queries (#4207)

* Support for Aggregate Queries

* improve pg and coverage

* Mongo 3.4 aggregates and tests

* replace _id with objectId

* improve tests for objectId

* project with group query

* typo
This commit is contained in:
Diamond Lewis
2017-11-12 13:00:22 -06:00
committed by Florent Vilmart
parent 4e207d32a7
commit 7223add446
8 changed files with 675 additions and 1 deletions

View File

@@ -0,0 +1,412 @@
'use strict';
const Parse = require('parse/node');
const rp = require('request-promise');
const masterKeyHeaders = {
'X-Parse-Application-Id': 'test',
'X-Parse-Rest-API-Key': 'test',
'X-Parse-Master-Key': 'test'
}
const masterKeyOptions = {
headers: masterKeyHeaders,
json: true
}
const loadTestData = () => {
const data1 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['S', 'M']};
const data2 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['M', 'L']};
const data3 = {score: 10, name: 'bar', sender: {group: 'B'}, size: ['S']};
const data4 = {score: 20, name: 'dpl', sender: {group: 'B'}, size: ['S']};
const obj1 = new TestObject(data1);
const obj2 = new TestObject(data2);
const obj3 = new TestObject(data3);
const obj4 = new TestObject(data4);
return Parse.Object.saveAll([obj1, obj2, obj3, obj4]);
}
describe('Parse.Query Aggregate testing', () => {
beforeEach((done) => {
loadTestData().then(done, done);
});
it('should only query aggregate with master key', (done) => {
Parse._request('GET', `aggregate/someClass`, {})
.then(() => {}, (error) => {
expect(error.message).toEqual('unauthorized: master key is required');
done();
});
});
it('invalid query invalid key', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
unknown: {},
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.catch((error) => {
expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY);
done();
});
});
it('invalid query group _id', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { _id: null },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.catch((error) => {
expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY);
done();
});
});
it('invalid query group objectId required', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: {},
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.catch((error) => {
expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY);
done();
});
});
it('group by field', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: '$name' },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(3);
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[1].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[2].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).not.toBe(undefined);
expect(resp.results[1].objectId).not.toBe(undefined);
expect(resp.results[2].objectId).not.toBe(undefined);
done();
}).catch(done.fail);
});
it('group sum query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, total: { $sum: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).toBe(null);
expect(resp.results[0].total).toBe(50);
done();
}).catch(done.fail);
});
it('group count query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, total: { $sum: 1 } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).toBe(null);
expect(resp.results[0].total).toBe(4);
done();
}).catch(done.fail);
});
it('group min query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, minScore: { $min: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).toBe(null);
expect(resp.results[0].minScore).toBe(10);
done();
}).catch(done.fail);
});
it('group max query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, maxScore: { $max: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).toBe(null);
expect(resp.results[0].maxScore).toBe(20);
done();
}).catch(done.fail);
});
it('group avg query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, avgScore: { $avg: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0].hasOwnProperty('objectId')).toBe(true);
expect(resp.results[0].objectId).toBe(null);
expect(resp.results[0].avgScore).toBe(12.5);
done();
}).catch(done.fail);
});
it('limit query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
limit: 2,
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(2);
done();
}).catch(done.fail);
});
it('sort ascending query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
sort: { name: 1 },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(4);
expect(resp.results[0].name).toBe('bar');
expect(resp.results[1].name).toBe('dpl');
expect(resp.results[2].name).toBe('foo');
expect(resp.results[3].name).toBe('foo');
done();
}).catch(done.fail);
});
it('sort decending query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
sort: { name: -1 },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(4);
expect(resp.results[0].name).toBe('foo');
expect(resp.results[1].name).toBe('foo');
expect(resp.results[2].name).toBe('dpl');
expect(resp.results[3].name).toBe('bar');
done();
}).catch(done.fail);
});
it('skip query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
skip: 2,
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(2);
done();
}).catch(done.fail);
});
it('match query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
match: { score: { $gt: 15 }},
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(1);
expect(resp.results[0].score).toBe(20);
done();
}).catch(done.fail);
});
it('project query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
project: { name: 1 },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
resp.results.forEach((result) => {
expect(result.name !== undefined).toBe(true);
expect(result.sender).toBe(undefined);
expect(result.size).toBe(undefined);
expect(result.score).toBe(undefined);
});
done();
}).catch(done.fail);
});
it('project with group query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
project: { score: 1 },
group: { objectId: '$score', score: { $sum: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(2);
resp.results.forEach((result) => {
expect(result.hasOwnProperty('objectId')).toBe(true);
expect(result.name).toBe(undefined);
expect(result.sender).toBe(undefined);
expect(result.size).toBe(undefined);
expect(result.score).not.toBe(undefined);
if (result.objectId === 10) {
expect(result.score).toBe(30);
}
if (result.objectId === 20) {
expect(result.score).toBe(20);
}
});
done();
}).catch(done.fail);
});
it('class does not exist return empty', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, total: { $sum: '$score' } },
}
});
rp.get(Parse.serverURL + '/aggregate/UnknownClass', options)
.then((resp) => {
expect(resp.results.length).toBe(0);
done();
}).catch(done.fail);
});
it('field does not exist return empty', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
group: { objectId: null, total: { $sum: '$unknownfield' } },
}
});
rp.get(Parse.serverURL + '/aggregate/UnknownClass', options)
.then((resp) => {
expect(resp.results.length).toBe(0);
done();
}).catch(done.fail);
});
it('distinct query', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: { distinct: 'score' }
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(2);
expect(resp.results.includes(10)).toBe(true);
expect(resp.results.includes(20)).toBe(true);
done();
}).catch(done.fail);
});
it('distinct query with where', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
distinct: 'score',
where: {
name: 'bar'
}
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0]).toBe(10);
done();
}).catch(done.fail);
});
it('distinct query with where string', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: {
distinct: 'score',
where: JSON.stringify({name:'bar'}),
}
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results[0]).toBe(10);
done();
}).catch(done.fail);
});
it('distinct nested', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: { distinct: 'sender.group' }
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(2);
expect(resp.results.includes('A')).toBe(true);
expect(resp.results.includes('B')).toBe(true);
done();
}).catch(done.fail);
});
it('distinct class does not exist return empty', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: { distinct: 'unknown' }
});
rp.get(Parse.serverURL + '/aggregate/UnknownClass', options)
.then((resp) => {
expect(resp.results.length).toBe(0);
done();
}).catch(done.fail);
});
it('distinct field does not exist return empty', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: { distinct: 'unknown' }
});
const obj = new TestObject();
obj.save().then(() => {
return rp.get(Parse.serverURL + '/aggregate/TestObject', options);
}).then((resp) => {
expect(resp.results.length).toBe(0);
done();
}).catch(done.fail);
});
it('distinct array', (done) => {
const options = Object.assign({}, masterKeyOptions, {
body: { distinct: 'size' }
});
rp.get(Parse.serverURL + '/aggregate/TestObject', options)
.then((resp) => {
expect(resp.results.length).toBe(3);
expect(resp.results.includes('S')).toBe(true);
expect(resp.results.includes('M')).toBe(true);
expect(resp.results.includes('L')).toBe(true);
done();
}).catch(done.fail);
});
});

View File

@@ -60,6 +60,14 @@ export default class MongoCollection {
return countOperation; return countOperation;
} }
distinct(field, query) {
return this._mongoCollection.distinct(field, query);
}
aggregate(pipeline, { maxTimeMS, readPreference } = {}) {
return this._mongoCollection.aggregate(pipeline, { maxTimeMS, readPreference }).toArray();
}
insertOne(object) { insertOne(object) {
return this._mongoCollection.insertOne(object); return this._mongoCollection.insertOne(object);
} }

View File

@@ -405,6 +405,27 @@ export class MongoStorageAdapter {
})); }));
} }
distinct(className, schema, query, fieldName) {
schema = convertParseSchemaToMongoSchema(schema);
return this._adaptiveCollection(className)
.then(collection => collection.distinct(fieldName, transformWhere(className, query, schema)));
}
aggregate(className, pipeline, readPreference) {
readPreference = this._parseReadPreference(readPreference);
return this._adaptiveCollection(className)
.then(collection => collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS }))
.then(results => {
results.forEach(result => {
if (result.hasOwnProperty('_id')) {
result.objectId = result._id;
delete result._id;
}
});
return results;
});
}
_parseReadPreference(readPreference) { _parseReadPreference(readPreference) {
if (readPreference) { if (readPreference) {
switch (readPreference) { switch (readPreference) {

View File

@@ -165,6 +165,10 @@ const transformDotField = (fieldName) => {
return name; return name;
} }
const transformAggregateField = (fieldName) => {
return fieldName.substr(1);
}
const validateKeys = (object) => { const validateKeys = (object) => {
if (typeof object == 'object') { if (typeof object == 'object') {
for (const key in object) { for (const key in object) {
@@ -1366,6 +1370,140 @@ export class PostgresStorageAdapter {
}); });
} }
distinct(className, schema, query, fieldName) {
debug('distinct', className, query);
let field = fieldName;
let column = fieldName;
if (fieldName.indexOf('.') >= 0) {
field = transformDotFieldToComponents(fieldName).join('->');
column = fieldName.split('.')[0];
}
const isArrayField = schema.fields
&& schema.fields[fieldName]
&& schema.fields[fieldName].type === 'Array';
const values = [field, column, className];
const where = buildWhereClause({ schema, query, index: 4 });
values.push(...where.values);
const wherePattern = where.pattern.length > 0 ? `WHERE ${where.pattern}` : '';
let qs = `SELECT DISTINCT ON ($1:raw) $2:raw FROM $3:name ${wherePattern}`;
if (isArrayField) {
qs = `SELECT distinct jsonb_array_elements($1:raw) as $2:raw FROM $3:name ${wherePattern}`;
}
debug(qs, values);
return this._client.any(qs, values)
.catch(() => [])
.then((results) => {
if (fieldName.indexOf('.') === -1) {
return results.map(object => object[field]);
}
const child = fieldName.split('.')[1];
return results.map(object => object[column][child]);
});
}
aggregate(className, pipeline) {
debug('aggregate', className, pipeline);
const values = [className];
let columns = [];
let countField = null;
let wherePattern = '';
let limitPattern = '';
let skipPattern = '';
let sortPattern = '';
let groupPattern = '';
for (let i = 0; i < pipeline.length; i += 1) {
const stage = pipeline[i];
if (stage.$group) {
for (const field in stage.$group) {
const value = stage.$group[field];
if (value === null || value === undefined) {
continue;
}
if (field === '_id') {
columns.push(`${transformAggregateField(value)} AS "objectId"`);
groupPattern = `GROUP BY ${transformAggregateField(value)}`;
continue;
}
if (value.$sum) {
if (typeof value.$sum === 'string') {
columns.push(`SUM(${transformAggregateField(value.$sum)}) AS "${field}"`);
} else {
countField = field;
columns.push(`COUNT(*) AS "${field}"`);
}
}
if (value.$max) {
columns.push(`MAX(${transformAggregateField(value.$max)}) AS "${field}"`);
}
if (value.$min) {
columns.push(`MIN(${transformAggregateField(value.$min)}) AS "${field}"`);
}
if (value.$avg) {
columns.push(`AVG(${transformAggregateField(value.$avg)}) AS "${field}"`);
}
}
columns.join(',');
} else {
columns.push('*');
}
if (stage.$project) {
if (columns.includes('*')) {
columns = [];
}
for (const field in stage.$project) {
const value = stage.$project[field];
if ((value === 1 || value === true)) {
columns.push(field);
}
}
}
if (stage.$match) {
const patterns = [];
for (const field in stage.$match) {
const value = stage.$match[field];
Object.keys(ParseToPosgresComparator).forEach(cmp => {
if (value[cmp]) {
const pgComparator = ParseToPosgresComparator[cmp];
patterns.push(`${field} ${pgComparator} ${value[cmp]}`);
}
});
}
wherePattern = patterns.length > 0 ? `WHERE ${patterns.join(' ')}` : '';
}
if (stage.$limit) {
limitPattern = `LIMIT ${stage.$limit}`;
}
if (stage.$skip) {
skipPattern = `OFFSET ${stage.$skip}`;
}
if (stage.$sort) {
const sort = stage.$sort;
const sorting = Object.keys(sort).map((key) => {
if (sort[key] === 1) {
return `"${key}" ASC`;
}
return `"${key}" DESC`;
}).join(',');
sortPattern = sort !== undefined && Object.keys(sort).length > 0 ? `ORDER BY ${sorting}` : '';
}
}
const qs = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`;
debug(qs, values);
return this._client.any(qs, values).then(results => {
if (countField) {
results[0][countField] = parseInt(results[0][countField], 10);
}
results.forEach(result => {
if (!result.hasOwnProperty('objectId')) {
result.objectId = null;
}
});
return results;
});
}
performInitialization({ VolatileClassesSchemas }) { performInitialization({ VolatileClassesSchemas }) {
debug('performInitialization'); debug('performInitialization');
const promises = VolatileClassesSchemas.map((schema) => { const promises = VolatileClassesSchemas.map((schema) => {

View File

@@ -785,6 +785,8 @@ DatabaseController.prototype.find = function(className, query, {
count, count,
keys, keys,
op, op,
distinct,
pipeline,
readPreference readPreference
} = {}) { } = {}) {
const isMaster = acl === undefined; const isMaster = acl === undefined;
@@ -853,6 +855,18 @@ DatabaseController.prototype.find = function(className, query, {
} else { } else {
return this.adapter.count(className, schema, query, readPreference); return this.adapter.count(className, schema, query, readPreference);
} }
} else if (distinct) {
if (!classExists) {
return [];
} else {
return this.adapter.distinct(className, schema, query, distinct);
}
} else if (pipeline) {
if (!classExists) {
return [];
} else {
return this.adapter.aggregate(className, pipeline, readPreference);
}
} else { } else {
if (!classExists) { if (!classExists) {
return []; return [];

View File

@@ -34,6 +34,7 @@ import { SessionsRouter } from './Routers/SessionsRouter';
import { UsersRouter } from './Routers/UsersRouter'; import { UsersRouter } from './Routers/UsersRouter';
import { PurgeRouter } from './Routers/PurgeRouter'; import { PurgeRouter } from './Routers/PurgeRouter';
import { AudiencesRouter } from './Routers/AudiencesRouter'; import { AudiencesRouter } from './Routers/AudiencesRouter';
import { AggregateRouter } from './Routers/AggregateRouter';
import { ParseServerRESTController } from './ParseServerRESTController'; import { ParseServerRESTController } from './ParseServerRESTController';
import * as controllers from './Controllers'; import * as controllers from './Controllers';
@@ -197,7 +198,8 @@ class ParseServer {
new PurgeRouter(), new PurgeRouter(),
new HooksRouter(), new HooksRouter(),
new CloudCodeRouter(), new CloudCodeRouter(),
new AudiencesRouter() new AudiencesRouter(),
new AggregateRouter()
]; ];
const routes = routers.reduce((memo, router) => { const routes = routers.reduce((memo, router) => {

View File

@@ -86,6 +86,8 @@ function RestQuery(config, auth, className, restWhere = {}, restOptions = {}, cl
case 'count': case 'count':
this.doCount = true; this.doCount = true;
break; break;
case 'distinct':
case 'pipeline':
case 'skip': case 'skip':
case 'limit': case 'limit':
case 'readPreference': case 'readPreference':

View File

@@ -0,0 +1,77 @@
import ClassesRouter from './ClassesRouter';
import rest from '../rest';
import * as middleware from '../middlewares';
import Parse from 'parse/node';
const ALLOWED_KEYS = [
'where',
'distinct',
'project',
'match',
'redact',
'limit',
'skip',
'unwind',
'group',
'sample',
'sort',
'geoNear',
'lookup',
'out',
'indexStats',
'facet',
'bucket',
'bucketAuto',
'sortByCount',
'addFields',
'replaceRoot',
'count',
'graphLookup',
];
export class AggregateRouter extends ClassesRouter {
handleFind(req) {
const body = Object.assign(req.body, ClassesRouter.JSONFromQuery(req.query));
const options = {};
const pipeline = [];
for (const key in body) {
if (ALLOWED_KEYS.indexOf(key) === -1) {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Invalid parameter for query: ${key}`);
}
if (key === 'group') {
if (body[key].hasOwnProperty('_id')) {
throw new Parse.Error(
Parse.Error.INVALID_QUERY,
`Invalid parameter for query: group. Please use objectId instead of _id`
);
}
if (!body[key].hasOwnProperty('objectId')) {
throw new Parse.Error(
Parse.Error.INVALID_QUERY,
`Invalid parameter for query: group. objectId is required`
);
}
body[key]._id = body[key].objectId;
delete body[key].objectId;
}
pipeline.push({ [`$${key}`]: body[key] });
}
if (body.distinct) {
options.distinct = String(body.distinct);
}
options.pipeline = pipeline;
if (typeof body.where === 'string') {
body.where = JSON.parse(body.where);
}
return rest.find(req.config, req.auth, this.className(req), body.where, options, req.info.clientSDK)
.then((response) => { return { response }; });
}
mountRoutes() {
this.route('GET','/aggregate/:className', middleware.promiseEnforceMasterKeyAccess, req => { return this.handleFind(req); });
}
}
export default AggregateRouter;