joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Postgres adapter (#2012)

Browse Source 
* Remove adaptiveCollection

* Remove an adaptiveCollection use

* Remove an adaptiveCollection

* make adaptiveCollection private

* Remove collection from mongoadapter

* Move schema collection usage into mongo adapter

* stop relying on mongo format for removing join tables

* reduce usage of schemaCollection

* remove uses of _collection

* Move CLP setting into mongo adapter

* remove all uses of schemaCollection

* make schemaCollection private

* remove transform from schemaCollection

* rename some stuff

* Tweak paramaters and stuff

* reorder some params

* reorder find() arguments

* finishsh touching up argument order

* Accept a database adapter as a parameter

* First passing test with postgres!

* Actually use the provided className

* index on unique-indexes: c454180 Revert "Log objects rather than JSON stringified objects (#1922)"

* Start dealing with test shittyness

* Make specific server config for tests async

* Fix email validation

* Fix broken cloud code

* Save callback to variable

* undo

* Fix tests

* Setup travis

* fix travis maybe

* try removing db user

* indentation?

* remove postgres version setting

* sudo maybe?

* use postgres username

* fix check for _PushStatus

* excludes

* remove db=mongo

* allow postgres to fail

* Fix allow failure

* postgres 9.4

* Remove mongo implementations and fix test

* Fix test leaving behind connections
This commit is contained in:
Drew
2016-06-12 16:35:13 -07:00
committed by GitHub
parent d559cb2382
commit 5518edc2a5
20 changed files with 499 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/Adapters/Storage/Mongo/MongoSchemaCollection.js
View File

@@ -1,6 +1,5 @@
import MongoCollection from './MongoCollection';
import * as transform from './MongoTransform';
function mongoFieldToParseSchemaField(type) {
if (type[0] === '*') {
@@ -154,20 +153,12 @@ class MongoSchemaCollection {
}
// Atomically find and delete an object based on query.
// The result is the promise with an object that was in the database before deleting.
// Postgres Note: Translates directly to `DELETE * FROM ... RETURNING *`, which will return data after delete is done.
findAndDeleteSchema(name: string) {
// arguments: query, sort
return this._collection._mongoCollection.findAndRemove(_mongoSchemaQueryFromNameQuery(name), []).then(document => {
// Value is the object where mongo returns multiple fields.
return document.value;
});
return this._collection._mongoCollection.findAndRemove(_mongoSchemaQueryFromNameQuery(name), []);
}
// Add a collection. Currently the input is in mongo format, but that will change to Parse format in a
// later PR. Returns a promise that is expected to resolve with the newly created schema, in Parse format.
// If the class already exists, returns a promise that rejects with undefined as the reason. If the collection
// can't be added for a reason other than it already existing, requirements for rejection reason are TBD.
// Returns a promise that is expected to resolve with the newly created schema, in Parse format.
// If the class already exists, returns a promise that rejects with DUPLICATE_VALUE as the reason.
addSchema(name: string, fields, classLevelPermissions) {
let mongoSchema = mongoSchemaFromFieldsAndClassNameAndCLP(fields, name, classLevelPermissions);
let mongoObject = _mongoSchemaObjectFromNameFields(name, mongoSchema);
@@ -175,9 +166,10 @@ class MongoSchemaCollection {
.then(result => mongoSchemaToParseSchema(result.ops[0]))
.catch(error => {
if (error.code === 11000) { //Mongo's duplicate key error
throw undefined;
throw new Parse.Error(Parse.Error.DUPLICATE_VALUE, 'Class already exists.');
} else {
throw error;
}
throw error;
});
}
@@ -192,8 +184,8 @@ class MongoSchemaCollection {
// Add a field to the schema. If database does not support the field
// type (e.g. mongo doesn't support more than one GeoPoint in a class) reject with an "Incorrect Type"
// Parse error with a desciptive message. If the field already exists, this function must
// not modify the schema, and must reject with an error. Exact error format is TBD. If this function
// is called for a class that doesn't exist, this function must create that class.
// not modify the schema, and must reject with DUPLICATE_VALUE error.
// If this is called for a class that doesn't exist, this function must create that class.
// TODO: throw an error if an unsupported field type is passed. Deciding whether a type is supported
// should be the job of the adapter. Some adapters may not support GeoPoint at all. Others may
@@ -229,10 +221,6 @@ class MongoSchemaCollection {
);
});
}
get transform() {
return transform;
}
}
// Exported for testing reasons and because we haven't moved all mongo schema format

105
src/Adapters/Storage/Mongo/MongoStorageAdapter.js
View File

@@ -69,25 +69,19 @@ export class MongoStorageAdapter {
return this.connectionPromise;
}
collection(name: string) {
return this.connect().then(() => {
return this.database.collection(name);
});
}
adaptiveCollection(name: string) {
_adaptiveCollection(name: string) {
return this.connect()
.then(() => this.database.collection(this._collectionPrefix + name))
.then(rawCollection => new MongoCollection(rawCollection));
}
schemaCollection() {
_schemaCollection() {
return this.connect()
.then(() => this.adaptiveCollection(MongoSchemaCollectionName))
.then(() => this._adaptiveCollection(MongoSchemaCollectionName))
.then(collection => new MongoSchemaCollection(collection));
}
collectionExists(name: string) {
classExists(name) {
return this.connect().then(() => {
return this.database.listCollections({ name: this._collectionPrefix + name }).toArray();
}).then(collections => {
@@ -95,22 +89,42 @@ export class MongoStorageAdapter {
});
}
// Deletes a schema. Resolve if successful. If the schema doesn't
// exist, resolve with undefined. If schema exists, but can't be deleted for some other reason,
// reject with INTERNAL_SERVER_ERROR.
deleteOneSchema(className: string) {
return this.collection(this._collectionPrefix + className).then(collection => collection.drop())
setClassLevelPermissions(className, CLPs) {
return this._schemaCollection()
.then(schemaCollection => schemaCollection.updateSchema(className, {
$set: { _metadata: { class_permissions: CLPs } }
}));
}
createClass(className, schema) {
return this._schemaCollection()
.then(schemaCollection => schemaCollection.addSchema(className, schema.fields, schema.classLevelPermissions));
}
addFieldIfNotExists(className, fieldName, type) {
return this._schemaCollection()
.then(schemaCollection => schemaCollection.addFieldIfNotExists(className, fieldName, type));
}
// Drops a collection. Resolves with true if it was a Parse Schema (eg. _User, Custom, etc.)
// and resolves with false if it wasn't (eg. a join table). Rejects if deletion was impossible.
deleteClass(className) {
return this._adaptiveCollection(className)
.then(collection => collection.drop())
.catch(error => {
// 'ns not found' means collection was already gone. Ignore deletion attempt.
if (error.message == 'ns not found') {
return;
}
throw error;
});
})
// We've dropped the collection, now remove the _SCHEMA document
.then(() => this._schemaCollection())
.then(schemaCollection => schemaCollection.findAndDeleteSchema(className))
}
// Delete all data known to this adatper. Used for testing.
deleteAllSchemas() {
deleteAllClasses() {
return storageAdapterAllCollections(this)
.then(collections => Promise.all(collections.map(collection => collection.drop())));
}
@@ -135,9 +149,14 @@ export class MongoStorageAdapter {
// may do so.
// Returns a Promise.
deleteFields(className: string, fieldNames, pointerFieldNames) {
const nonPointerFieldNames = _.difference(fieldNames, pointerFieldNames);
const mongoFormatNames = nonPointerFieldNames.concat(pointerFieldNames.map(name => `_p_${name}`));
deleteFields(className, schema, fieldNames) {
const mongoFormatNames = fieldNames.map(fieldName => {
if (schema.fields[fieldName].type === 'Pointer') {
return `_p_${fieldName}`
} else {
return fieldName;
}
});
const collectionUpdate = { '$unset' : {} };
mongoFormatNames.forEach(name => {
collectionUpdate['$unset'][name] = null;
@@ -148,33 +167,33 @@ export class MongoStorageAdapter {
schemaUpdate['$unset'][name] = null;
});
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection.updateMany({}, collectionUpdate))
.then(updateResult => this.schemaCollection())
.then(() => this._schemaCollection())
.then(schemaCollection => schemaCollection.updateSchema(className, schemaUpdate));
}
// Return a promise for all schemas known to this adapter, in Parse format. In case the
// schemas cannot be retrieved, returns a promise that rejects. Requirements for the
// rejection reason are TBD.
getAllSchemas() {
return this.schemaCollection().then(schemasCollection => schemasCollection._fetchAllSchemasFrom_SCHEMA());
getAllClasses() {
return this._schemaCollection().then(schemasCollection => schemasCollection._fetchAllSchemasFrom_SCHEMA());
}
// Return a promise for the schema with the given name, in Parse format. If
// this adapter doesn't know about the schema, return a promise that rejects with
// undefined as the reason.
getOneSchema(className) {
return this.schemaCollection()
getClass(className) {
return this._schemaCollection()
.then(schemasCollection => schemasCollection._fechOneSchemaFrom_SCHEMA(className));
}
// TODO: As yet not particularly well specified. Creates an object. Maybe shouldn't even need the schema,
// and should infer from the type. Or maybe does need the schema for validations. Or maybe needs
// the schem only for the legacy mongo format. We'll figure that out later.
createObject(className, object, schema) {
createObject(className, schema, object) {
const mongoObject = parseObjectToMongoObjectForCreate(className, object, schema);
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection.insertOne(mongoObject))
.catch(error => {
if (error.code === 11000) { // Duplicate value
@@ -188,8 +207,8 @@ export class MongoStorageAdapter {
// Remove all objects that match the given Parse Query.
// If no objects match, reject with OBJECT_NOT_FOUND. If objects are found and deleted, resolve with undefined.
// If there is some other error, reject with INTERNAL_SERVER_ERROR.
deleteObjectsByQuery(className, query, schema) {
return this.adaptiveCollection(className)
deleteObjectsByQuery(className, schema, query) {
return this._adaptiveCollection(className)
.then(collection => {
let mongoWhere = transformWhere(className, query, schema);
return collection.deleteMany(mongoWhere)
@@ -205,36 +224,36 @@ export class MongoStorageAdapter {
}
// Apply the update to all objects that match the given Parse Query.
updateObjectsByQuery(className, query, schema, update) {
updateObjectsByQuery(className, schema, query, update) {
const mongoUpdate = transformUpdate(className, update, schema);
const mongoWhere = transformWhere(className, query, schema);
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection.updateMany(mongoWhere, mongoUpdate));
}
// Atomically finds and updates an object based on query.
// Resolve with the updated object.
findOneAndUpdate(className, query, schema, update) {
findOneAndUpdate(className, schema, query, update) {
const mongoUpdate = transformUpdate(className, update, schema);
const mongoWhere = transformWhere(className, query, schema);
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection._mongoCollection.findAndModify(mongoWhere, [], mongoUpdate, { new: true }))
.then(result => result.value);
}
// Hopefully we can get rid of this. It's only used for config and hooks.
upsertOneObject(className, query, schema, update) {
upsertOneObject(className, schema, query, update) {
const mongoUpdate = transformUpdate(className, update, schema);
const mongoWhere = transformWhere(className, query, schema);
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection.upsertOne(mongoWhere, mongoUpdate));
}
// Executes a find. Accepts: className, query in Parse format, and { skip, limit, sort }.
find(className, query, schema, { skip, limit, sort }) {
find(className, schema, query, { skip, limit, sort }) {
let mongoWhere = transformWhere(className, query, schema);
let mongoSort = _.mapKeys(sort, (value, fieldName) => transformKey(className, fieldName, schema));
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection.find(mongoWhere, { skip, limit, sort: mongoSort }))
.then(objects => objects.map(object => mongoObjectToParseObject(className, object, schema)));
}
@@ -244,13 +263,13 @@ export class MongoStorageAdapter {
// As such, we shouldn't expose this function to users of parse until we have an out-of-band
// Way of determining if a field is nullable. Undefined doesn't count against uniqueness,
// which is why we use sparse indexes.
ensureUniqueness(className, fieldNames, schema) {
ensureUniqueness(className, schema, fieldNames) {
let indexCreationRequest = {};
let mongoFieldNames = fieldNames.map(fieldName => transformKey(className, fieldName, schema));
mongoFieldNames.forEach(fieldName => {
indexCreationRequest[fieldName] = 1;
});
return this.adaptiveCollection(className)
return this._adaptiveCollection(className)
.then(collection => collection._ensureSparseUniqueIndexInBackground(indexCreationRequest))
.catch(error => {
if (error.code === 11000) {
@@ -263,12 +282,12 @@ export class MongoStorageAdapter {
// Used in tests
_rawFind(className, query) {
return this.adaptiveCollection(className).then(collection => collection.find(query));
return this._adaptiveCollection(className).then(collection => collection.find(query));
}
// Executs a count.
count(className, query, schema) {
return this.adaptiveCollection(className)
count(className, schema, query) {
return this._adaptiveCollection(className)
.then(collection => collection.count(transformWhere(className, query, schema)));
}
}

187
src/Adapters/Storage/Postgres/PostgresStorageAdapter.js Normal file
View File

@@ -0,0 +1,187 @@
const pgp = require('pg-promise')();
const PostgresRelationDoesNotExistError = '42P01';
const PostgresDuplicateRelationError = '42P07';
export class PostgresStorageAdapter {
// Private
_collectionPrefix: string;
_client;
constructor({
uri,
collectionPrefix = '',
}) {
this._collectionPrefix = collectionPrefix;
this._client = pgp(uri);
}
_ensureSchemaCollectionExists() {
return this._client.query('CREATE TABLE "_SCHEMA" ( "className" varChar(120), "schema" jsonb, "isParseClass" bool, PRIMARY KEY ("className") )')
.catch(error => {
if (error.code === PostgresDuplicateRelationError) {
// Table already exists, must have been created by a different request. Ignore error.
return;
} else {
throw error;
}
});
};
classExists(name) {
return Promise.reject('Not implented yet.')
}
setClassLevelPermissions(className, CLPs) {
return Promise.reject('Not implented yet.')
}
createClass(className, schema) {
return this._client.query('CREATE TABLE $<className:name> ()', { className })
.then(() => this._client.query('INSERT INTO "_SCHEMA" ("className", "schema", "isParseClass") VALUES ($<className>, $<schema>, true)', { className, schema }))
}
addFieldIfNotExists(className, fieldName, type) {
// TODO: Doing this in a transaction is probably a good idea.
return this._client.query('ALTER TABLE "GameScore" ADD COLUMN "score" double precision', { className, fieldName })
.catch(error => {
if (error.code === PostgresRelationDoesNotExistError) {
return this.createClass(className, { fields: { [fieldName]: type } })
} else {
throw error;
}
})
.then(() => this._client.query('SELECT "schema" FROM "_SCHEMA"', { className }))
.then(result => {
if (fieldName in result[0].schema) {
throw "Attempted to add a field that already exists";
} else {
result[0].schema.fields[fieldName] = type;
return this._client.query(
'UPDATE "_SCHEMA" SET "schema"=$<schema> WHERE "className"=$<className>',
{ schema: result[0].schema, className }
);
}
})
}
// Drops a collection. Resolves with true if it was a Parse Schema (eg. _User, Custom, etc.)
// and resolves with false if it wasn't (eg. a join table). Rejects if deletion was impossible.
deleteClass(className) {
return Promise.reject('Not implented yet.')
}
// Delete all data known to this adatper. Used for testing.
deleteAllClasses() {
return this._client.query('SELECT "className" FROM "_SCHEMA"')
.then(results => {
const classes = ['_SCHEMA', ...results.map(result => result.className)];
return Promise.all(classes.map(className => this._client.query('DROP TABLE $<className:name>', { className })));
}, error => {
if (error.code === PostgresRelationDoesNotExistError) {
// No _SCHEMA collection. Don't delete anything.
return;
} else {
throw error;
}
})
}
// Remove the column and all the data. For Relations, the _Join collection is handled
// specially, this function does not delete _Join columns. It should, however, indicate
// that the relation fields does not exist anymore. In mongo, this means removing it from
// the _SCHEMA collection. There should be no actual data in the collection under the same name
// as the relation column, so it's fine to attempt to delete it. If the fields listed to be
// deleted do not exist, this function should return successfully anyways. Checking for
// attempts to delete non-existent fields is the responsibility of Parse Server.
// Pointer field names are passed for legacy reasons: the original mongo
// format stored pointer field names differently in the database, and therefore
// needed to know the type of the field before it could delete it. Future database
// adatpers should ignore the pointerFieldNames argument. All the field names are in
// fieldNames, they show up additionally in the pointerFieldNames database for use
// by the mongo adapter, which deals with the legacy mongo format.
// This function is not obligated to delete fields atomically. It is given the field
// names in a list so that databases that are capable of deleting fields atomically
// may do so.
// Returns a Promise.
deleteFields(className, schema, fieldNames) {
return Promise.reject('Not implented yet.')
}
// Return a promise for all schemas known to this adapter, in Parse format. In case the
// schemas cannot be retrieved, returns a promise that rejects. Requirements for the
// rejection reason are TBD.
getAllClasses() {
return this._ensureSchemaCollectionExists()
.then(() => this._client.query('SELECT * FROM "_SCHEMA"'))
.then(results => results.map(result => ({ className: result.className, ...result.schema })))
}
// Return a promise for the schema with the given name, in Parse format. If
// this adapter doesn't know about the schema, return a promise that rejects with
// undefined as the reason.
getClass(className) {
return this._client.query('SELECT * FROM "_SCHEMA" WHERE "className"=$<className>', { className })
.then(result => {
if (result.length === 1) {
return result;
} else {
throw undefined;
}
});
}
// TODO: remove the mongo format dependency
createObject(className, schema, object) {
return this._client.query('INSERT INTO "GameScore" (score) VALUES ($<score>)', { score: object.score })
.then(() => ({ ops: [object] }));
}
// Remove all objects that match the given Parse Query.
// If no objects match, reject with OBJECT_NOT_FOUND. If objects are found and deleted, resolve with undefined.
// If there is some other error, reject with INTERNAL_SERVER_ERROR.
deleteObjectsByQuery(className, schema, query) {
return Promise.reject('Not implented yet.')
}
// Apply the update to all objects that match the given Parse Query.
updateObjectsByQuery(className, schema, query, update) {
return Promise.reject('Not implented yet.')
}
// Hopefully we can get rid of this in favor of updateObjectsByQuery.
findOneAndUpdate(className, schema, query, update) {
return Promise.reject('Not implented yet.')
}
// Hopefully we can get rid of this. It's only used for config and hooks.
upsertOneObject(className, schema, query, update) {
return Promise.reject('Not implented yet.')
}
// Executes a find. Accepts: className, query in Parse format, and { skip, limit, sort }.
find(className, schema, query, { skip, limit, sort }) {
return this._client.query("SELECT * FROM $<className>", { className })
}
// Create a unique index. Unique indexes on nullable fields are not allowed. Since we don't
// currently know which fields are nullable and which aren't, we ignore that criteria.
// As such, we shouldn't expose this function to users of parse until we have an out-of-band
// Way of determining if a field is nullable. Undefined doesn't count against uniqueness,
// which is why we use sparse indexes.
ensureUniqueness(className, schema, fieldNames) {
return Promise.resolve('ensureUniqueness not implented yet.')
}
// Executs a count.
count(className, schema, query) {
return Promise.reject('Not implented yet.')
}
}
export default PostgresStorageAdapter;
module.exports = PostgresStorageAdapter; // Required for tests