joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Indexes to Schema API (#4240)

Browse Source 
* Add Indexes to Schema API

* error handling

* ci errors

* postgres support

* full text compound indexes

* pg clean up

* get indexes on startup

* test compound index on startup

* add default _id to index, full Text index on startup

* lint

* fix test
This commit is contained in:
Diamond Lewis
2017-11-25 13:55:34 -06:00
committed by Florent Vilmart
parent 6a1510729a
commit 4bccf96ae7
10 changed files with 965 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/Adapters/Storage/Mongo/MongoSchemaCollection.js
View File

@@ -63,13 +63,20 @@ const defaultCLPS = Object.freeze({
function mongoSchemaToParseSchema(mongoSchema) {
let clps = defaultCLPS;
if (mongoSchema._metadata && mongoSchema._metadata.class_permissions) {
clps = {...emptyCLPS, ...mongoSchema._metadata.class_permissions};
let indexes = {}
if (mongoSchema._metadata) {
if (mongoSchema._metadata.class_permissions) {
clps = {...emptyCLPS, ...mongoSchema._metadata.class_permissions};
}
if (mongoSchema._metadata.indexes) {
indexes = {...mongoSchema._metadata.indexes};
}
}
return {
className: mongoSchema._id,
fields: mongoSchemaFieldsToParseSchemaFields(mongoSchema),
classLevelPermissions: clps,
indexes: indexes,
};
}

130
src/Adapters/Storage/Mongo/MongoStorageAdapter.js
View File

@@ -53,7 +53,7 @@ const convertParseSchemaToMongoSchema = ({...schema}) => {
// Returns { code, error } if invalid, or { result }, an object
// suitable for inserting into _SCHEMA collection, otherwise.
const mongoSchemaFromFieldsAndClassNameAndCLP = (fields, className, classLevelPermissions) => {
const mongoSchemaFromFieldsAndClassNameAndCLP = (fields, className, classLevelPermissions, indexes) => {
const mongoObject = {
_id: className,
objectId: 'string',
@@ -74,6 +74,11 @@ const mongoSchemaFromFieldsAndClassNameAndCLP = (fields, className, classLevelPe
}
}
if (indexes && typeof indexes === 'object' && Object.keys(indexes).length > 0) {
mongoObject._metadata = mongoObject._metadata || {};
mongoObject._metadata.indexes = indexes;
}
return mongoObject;
}
@@ -165,11 +170,81 @@ export class MongoStorageAdapter {
}));
}
setIndexesWithSchemaFormat(className, submittedIndexes, existingIndexes = {}, fields) {
if (submittedIndexes === undefined) {
return Promise.resolve();
}
if (Object.keys(existingIndexes).length === 0) {
existingIndexes = { _id_: { _id: 1} };
}
const deletePromises = [];
const insertedIndexes = [];
Object.keys(submittedIndexes).forEach(name => {
const field = submittedIndexes[name];
if (existingIndexes[name] && field.__op !== 'Delete') {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Index ${name} exists, cannot update.`);
}
if (!existingIndexes[name] && field.__op === 'Delete') {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Index ${name} does not exist, cannot delete.`);
}
if (field.__op === 'Delete') {
const promise = this.dropIndex(className, name);
deletePromises.push(promise);
delete existingIndexes[name];
} else {
Object.keys(field).forEach(key => {
if (!fields.hasOwnProperty(key)) {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Field ${key} does not exist, cannot add index.`);
}
});
existingIndexes[name] = field;
insertedIndexes.push({
key: field,
name,
});
}
});
let insertPromise = Promise.resolve();
if (insertedIndexes.length > 0) {
insertPromise = this.createIndexes(className, insertedIndexes);
}
return Promise.all(deletePromises)
.then(() => insertPromise)
.then(() => this._schemaCollection())
.then(schemaCollection => schemaCollection.updateSchema(className, {
$set: { _metadata: { indexes: existingIndexes } }
}));
}
setIndexesFromMongo(className) {
return this.getIndexes(className).then((indexes) => {
indexes = indexes.reduce((obj, index) => {
if (index.key._fts) {
delete index.key._fts;
delete index.key._ftsx;
for (const field in index.weights) {
index.key[field] = 'text';
}
}
obj[index.name] = index.key;
return obj;
}, {});
return this._schemaCollection()
.then(schemaCollection => schemaCollection.updateSchema(className, {
$set: { _metadata: { indexes: indexes } }
}));
}).catch(() => {
// Ignore if collection not found
return Promise.resolve();
});
}
createClass(className, schema) {
schema = convertParseSchemaToMongoSchema(schema);
const mongoObject = mongoSchemaFromFieldsAndClassNameAndCLP(schema.fields, className, schema.classLevelPermissions);
const mongoObject = mongoSchemaFromFieldsAndClassNameAndCLP(schema.fields, className, schema.classLevelPermissions, schema.indexes);
mongoObject._id = className;
return this._schemaCollection()
return this.setIndexesWithSchemaFormat(className, schema.indexes, {}, schema.fields)
.then(() => this._schemaCollection())
.then(schemaCollection => schemaCollection._collection.insertOne(mongoObject))
.then(result => MongoSchemaCollection._TESTmongoSchemaToParseSchema(result.ops[0]))
.catch(error => {
@@ -353,7 +428,7 @@ export class MongoStorageAdapter {
}, {});
readPreference = this._parseReadPreference(readPreference);
return this.createTextIndexesIfNeeded(className, query)
return this.createTextIndexesIfNeeded(className, query, schema)
.then(() => this._adaptiveCollection(className))
.then(collection => collection.find(mongoWhere, {
skip,
@@ -463,6 +538,11 @@ export class MongoStorageAdapter {
.then(collection => collection._mongoCollection.createIndex(index));
}
createIndexes(className, indexes) {
return this._adaptiveCollection(className)
.then(collection => collection._mongoCollection.createIndexes(indexes));
}
createIndexesIfNeeded(className, fieldName, type) {
if (type && type.type === 'Polygon') {
const index = {
@@ -473,20 +553,26 @@ export class MongoStorageAdapter {
return Promise.resolve();
}
createTextIndexesIfNeeded(className, query) {
createTextIndexesIfNeeded(className, query, schema) {
for(const fieldName in query) {
if (!query[fieldName] || !query[fieldName].$text) {
continue;
}
const index = {
[fieldName]: 'text'
const existingIndexes = schema.indexes;
for (const key in existingIndexes) {
const index = existingIndexes[key];
if (index.hasOwnProperty(fieldName)) {
return Promise.resolve();
}
}
const indexName = `${fieldName}_text`;
const textIndex = {
[indexName]: { [fieldName]: 'text' }
};
return this.createIndex(className, index)
return this.setIndexesWithSchemaFormat(className, textIndex, existingIndexes, schema.fields)
.catch((error) => {
if (error.code === 85) {
throw new Parse.Error(
Parse.Error.INTERNAL_SERVER_ERROR,
'Only one text index is supported, please delete all text indexes to use new field.');
if (error.code === 85) { // Index exist with different options
return this.setIndexesFromMongo(className);
}
throw error;
});
@@ -498,6 +584,26 @@ export class MongoStorageAdapter {
return this._adaptiveCollection(className)
.then(collection => collection._mongoCollection.indexes());
}
dropIndex(className, index) {
return this._adaptiveCollection(className)
.then(collection => collection._mongoCollection.dropIndex(index));
}
dropAllIndexes(className) {
return this._adaptiveCollection(className)
.then(collection => collection._mongoCollection.dropIndexes());
}
updateSchemaWithIndexes() {
return this.getAllClasses()
.then((classes) => {
const promises = classes.map((schema) => {
return this.setIndexesFromMongo(schema.className);
});
return Promise.all(promises);
});
}
}
export default MongoStorageAdapter;

78
src/Adapters/Storage/Postgres/PostgresStorageAdapter.js
View File

@@ -98,10 +98,15 @@ const toParseSchema = (schema) => {
if (schema.classLevelPermissions) {
clps = {...emptyCLPS, ...schema.classLevelPermissions};
}
let indexes = {};
if (schema.indexes) {
indexes = {...schema.indexes};
}
return {
className: schema.className,
fields: schema.fields,
classLevelPermissions: clps,
indexes,
};
}
@@ -608,12 +613,64 @@ export class PostgresStorageAdapter {
});
}
setIndexesWithSchemaFormat(className, submittedIndexes, existingIndexes = {}, fields, conn) {
conn = conn || this._client;
if (submittedIndexes === undefined) {
return Promise.resolve();
}
if (Object.keys(existingIndexes).length === 0) {
existingIndexes = { _id_: { _id: 1} };
}
const deletedIndexes = [];
const insertedIndexes = [];
Object.keys(submittedIndexes).forEach(name => {
const field = submittedIndexes[name];
if (existingIndexes[name] && field.__op !== 'Delete') {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Index ${name} exists, cannot update.`);
}
if (!existingIndexes[name] && field.__op === 'Delete') {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Index ${name} does not exist, cannot delete.`);
}
if (field.__op === 'Delete') {
deletedIndexes.push(name);
delete existingIndexes[name];
} else {
Object.keys(field).forEach(key => {
if (!fields.hasOwnProperty(key)) {
throw new Parse.Error(Parse.Error.INVALID_QUERY, `Field ${key} does not exist, cannot add index.`);
}
});
existingIndexes[name] = field;
insertedIndexes.push({
key: field,
name,
});
}
});
let insertPromise = Promise.resolve();
if (insertedIndexes.length > 0) {
insertPromise = this.createIndexes(className, insertedIndexes, conn);
}
let deletePromise = Promise.resolve();
if (deletedIndexes.length > 0) {
deletePromise = this.dropIndexes(className, deletedIndexes, conn);
}
return deletePromise
.then(() => insertPromise)
.then(() => this._ensureSchemaCollectionExists())
.then(() => {
const values = [className, 'schema', 'indexes', JSON.stringify(existingIndexes)]
return conn.none(`UPDATE "_SCHEMA" SET $2:name = json_object_set_key($2:name, $3::text, $4::jsonb) WHERE "className"=$1 `, values);
});
}
createClass(className, schema) {
return this._client.tx(t => {
const q1 = this.createTable(className, schema, t);
const q2 = t.none('INSERT INTO "_SCHEMA" ("className", "schema", "isParseClass") VALUES ($<className>, $<schema>, true)', { className, schema });
const q3 = this.setIndexesWithSchemaFormat(className, schema.indexes, {}, schema.fields, t);
return t.batch([q1, q2]);
return t.batch([q1, q2, q3]);
})
.then(() => {
return toParseSchema(schema)
@@ -1548,6 +1605,25 @@ export class PostgresStorageAdapter {
console.error(error);
});
}
createIndexes(className, indexes, conn) {
return (conn || this._client).tx(t => t.batch(indexes.map(i => {
return t.none('CREATE INDEX $1:name ON $2:name ($3:name)', [i.name, className, i.key]);
})));
}
dropIndexes(className, indexes, conn) {
return (conn || this._client).tx(t => t.batch(indexes.map(i => t.none('DROP INDEX $1:name', i))));
}
getIndexes(className) {
const qs = 'SELECT * FROM pg_indexes WHERE tablename = ${className}';
return this._client.any(qs, {className});
}
updateSchemaWithIndexes() {
return Promise.resolve();
}
}
function convertPolygonToSQL(polygon) {

4
src/Controllers/DatabaseController.js
View File

@@ -1029,9 +1029,11 @@ DatabaseController.prototype.performInitialization = function() {
throw error;
});
const indexPromise = this.adapter.updateSchemaWithIndexes();
// Create tables for volatile classes
const adapterInit = this.adapter.performInitialization({ VolatileClassesSchemas: SchemaController.VolatileClassesSchemas });
return Promise.all([usernameUniqueness, emailUniqueness, roleUniqueness, adapterInit]);
return Promise.all([usernameUniqueness, emailUniqueness, roleUniqueness, adapterInit, indexPromise]);
}
function joinTableName(className, key) {

67
src/Controllers/SchemaController.js
View File

@@ -287,18 +287,28 @@ const convertAdapterSchemaToParseSchema = ({...schema}) => {
schema.fields.password = { type: 'String' };
}
if (schema.indexes && Object.keys(schema.indexes).length === 0) {
delete schema.indexes;
}
return schema;
}
const injectDefaultSchema = ({className, fields, classLevelPermissions}) => ({
className,
fields: {
...defaultColumns._Default,
...(defaultColumns[className] || {}),
...fields,
},
classLevelPermissions,
});
const injectDefaultSchema = ({className, fields, classLevelPermissions, indexes}) => {
const defaultSchema = {
className,
fields: {
...defaultColumns._Default,
...(defaultColumns[className] || {}),
...fields,
},
classLevelPermissions,
};
if (indexes && Object.keys(indexes).length !== 0) {
defaultSchema.indexes = indexes;
}
return defaultSchema;
};
const _HooksSchema = {className: "_Hooks", fields: defaultColumns._Hooks};
const _GlobalConfigSchema = { className: "_GlobalConfig", fields: defaultColumns._GlobalConfig }
@@ -344,6 +354,7 @@ export default class SchemaController {
_dbAdapter;
data;
perms;
indexes;
constructor(databaseAdapter, schemaCache) {
this._dbAdapter = databaseAdapter;
@@ -352,6 +363,8 @@ export default class SchemaController {
this.data = {};
// this.perms[className][operation] tells you the acl-style permissions
this.perms = {};
// this.indexes[className][operation] tells you the indexes
this.indexes = {};
}
reloadData(options = {clearCache: false}) {
@@ -370,9 +383,11 @@ export default class SchemaController {
.then(allSchemas => {
const data = {};
const perms = {};
const indexes = {};
allSchemas.forEach(schema => {
data[schema.className] = injectDefaultSchema(schema).fields;
perms[schema.className] = schema.classLevelPermissions;
indexes[schema.className] = schema.indexes;
});
// Inject the in-memory classes
@@ -380,13 +395,16 @@ export default class SchemaController {
const schema = injectDefaultSchema({ className });
data[className] = schema.fields;
perms[className] = schema.classLevelPermissions;
indexes[className] = schema.indexes;
});
this.data = data;
this.perms = perms;
this.indexes = indexes;
delete this.reloadDataPromise;
}, (err) => {
this.data = {};
this.perms = {};
this.indexes = {};
delete this.reloadDataPromise;
throw err;
});
@@ -424,7 +442,8 @@ export default class SchemaController {
return Promise.resolve({
className,
fields: this.data[className],
classLevelPermissions: this.perms[className]
classLevelPermissions: this.perms[className],
indexes: this.indexes[className]
});
}
return this._cache.getOneSchema(className).then((cached) => {
@@ -449,13 +468,13 @@ export default class SchemaController {
// on success, and rejects with an error on fail. Ensure you
// have authorization (master key, or client class creation
// enabled) before calling this function.
addClassIfNotExists(className, fields = {}, classLevelPermissions) {
addClassIfNotExists(className, fields = {}, classLevelPermissions, indexes = {}) {
var validationError = this.validateNewClass(className, fields, classLevelPermissions);
if (validationError) {
return Promise.reject(validationError);
}
return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, className }))
return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, indexes, className }))
.then(convertAdapterSchemaToParseSchema)
.then((res) => {
return this._cache.clear().then(() => {
@@ -471,7 +490,7 @@ export default class SchemaController {
});
}
updateClass(className, submittedFields, classLevelPermissions, database) {
updateClass(className, submittedFields, classLevelPermissions, indexes, database) {
return this.getOneSchema(className)
.then(schema => {
const existingFields = schema.fields;
@@ -509,7 +528,6 @@ export default class SchemaController {
if (deletedFields.length > 0) {
deletePromise = this.deleteFields(deletedFields, className, database);
}
return deletePromise // Delete Everything
.then(() => this.reloadData({ clearCache: true })) // Reload our Schema, so we have all the new values
.then(() => {
@@ -520,12 +538,20 @@ export default class SchemaController {
return Promise.all(promises);
})
.then(() => this.setPermissions(className, classLevelPermissions, newSchema))
.then(() => this._dbAdapter.setIndexesWithSchemaFormat(className, indexes, schema.indexes, newSchema))
.then(() => this.reloadData({ clearCache: true }))
//TODO: Move this logic into the database adapter
.then(() => ({
className: className,
fields: this.data[className],
classLevelPermissions: this.perms[className]
}));
.then(() => {
const reloadedSchema = {
className: className,
fields: this.data[className],
classLevelPermissions: this.perms[className],
};
if (this.indexes[className] && Object.keys(this.indexes[className]).length !== 0) {
reloadedSchema.indexes = this.indexes[className];
}
return reloadedSchema;
});
})
.catch(error => {
if (error === undefined) {
@@ -620,8 +646,7 @@ export default class SchemaController {
return Promise.resolve();
}
validateCLP(perms, newSchema);
return this._dbAdapter.setClassLevelPermissions(className, perms)
.then(() => this.reloadData({ clearCache: true }));
return this._dbAdapter.setClassLevelPermissions(className, perms);
}
// Returns a promise that resolves successfully to the new schema

4
src/Routers/SchemasRouter.js
View File

@@ -49,7 +49,7 @@ function createSchema(req) {
}
return req.config.database.loadSchema({ clearCache: true})
.then(schema => schema.addClassIfNotExists(className, req.body.fields, req.body.classLevelPermissions))
.then(schema => schema.addClassIfNotExists(className, req.body.fields, req.body.classLevelPermissions, req.body.indexes))
.then(schema => ({ response: schema }));
}
@@ -65,7 +65,7 @@ function modifySchema(req) {
const className = req.params.className;
return req.config.database.loadSchema({ clearCache: true})
.then(schema => schema.updateClass(className, submittedFields, req.body.classLevelPermissions, req.config.database))
.then(schema => schema.updateClass(className, submittedFields, req.body.classLevelPermissions, req.body.indexes, req.config.database))
.then(result => ({response: result}));
}