Begin isolating object creation code into an externalizable API. (#1569)

* Tidy up transformKeyValue

* Specialize transformKeyValue for object creation

* remove keys that never appear in creation requests

* rename function

* remove local var

* early exit for simple keys

* Refactor create

* Force class creation when creating an object

* Pass parameters to key value transformer

* No need to check for array in this func

* start using Parse Format schema in MongoTransform

* Remove call to getExpectedType

* add tests to ensure client can't see _PushStatus

spec/MongoTransform.spec.js

@@ -23,11 +23,13 @@ var dummySchema = {
 };
 
 
-describe('parseObjectToMongoObject', () => {
+describe('parseObjectToMongoObjectForCreate', () => {
 
   it('a basic number', (done) => {
     var input = {five: 5};
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input, {
+      fields: {five: {type: 'Number'}}
+    });
     jequal(input, output);
     done();
   });
@@ -37,7 +39,7 @@ describe('parseObjectToMongoObject', () => {
       createdAt: "2015-10-06T21:24:50.332Z",
       updatedAt: "2015-10-06T21:24:50.332Z"
     };
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input);
     expect(output._created_at instanceof Date).toBe(true);
     expect(output._updated_at instanceof Date).toBe(true);
     done();
@@ -49,21 +51,25 @@ describe('parseObjectToMongoObject', () => {
       objectId: 'myId',
       className: 'Blah',
     };
-    var out = transform.parseObjectToMongoObject(dummySchema, null, {pointers: [pointer]});
+    var out = transform.parseObjectToMongoObjectForCreate(dummySchema, null, {pointers: [pointer]},{
+      fields: {pointers: {type: 'Array'}}
+    });
     jequal([pointer], out.pointers);
     done();
   });
 
-  it('a delete op', (done) => {
+  //TODO: object creation requests shouldn't be seeing __op delete, it makes no sense to
+  //have __op delete in a new object. Figure out what this should actually be testing.
+  notWorking('a delete op', (done) => {
     var input = {deleteMe: {__op: 'Delete'}};
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input);
     jequal(output, {});
     done();
   });
 
   it('basic ACL', (done) => {
     var input = {ACL: {'0123': {'read': true, 'write': true}}};
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input);
     // This just checks that it doesn't crash, but it should check format.
     done();
   });
@@ -71,21 +77,27 @@ describe('parseObjectToMongoObject', () => {
   describe('GeoPoints', () => {
     it('plain', (done) => {
       var geoPoint = {__type: 'GeoPoint', longitude: 180, latitude: -180};
-      var out = transform.parseObjectToMongoObject(dummySchema, null, {location: geoPoint});
+      var out = transform.parseObjectToMongoObjectForCreate(dummySchema, null, {location: geoPoint},{
+        fields: {location: {type: 'GeoPoint'}}
+      });
       expect(out.location).toEqual([180, -180]);
       done();
     });
 
     it('in array', (done) => {
       var geoPoint = {__type: 'GeoPoint', longitude: 180, latitude: -180};
-      var out = transform.parseObjectToMongoObject(dummySchema, null, {locations: [geoPoint, geoPoint]});
+      var out = transform.parseObjectToMongoObjectForCreate(dummySchema, null, {locations: [geoPoint, geoPoint]},{
+        fields: {locations: {type: 'Array'}}
+      });
       expect(out.locations).toEqual([geoPoint, geoPoint]);
       done();
     });
 
     it('in sub-object', (done) => {
       var geoPoint = {__type: 'GeoPoint', longitude: 180, latitude: -180};
-      var out = transform.parseObjectToMongoObject(dummySchema, null, { locations: { start: geoPoint }});
+      var out = transform.parseObjectToMongoObjectForCreate(dummySchema, null, { locations: { start: geoPoint }},{
+        fields: {locations: {type: 'Object'}}
+      });
       expect(out).toEqual({ locations: { start: geoPoint } });
       done();
     });
@@ -196,7 +208,9 @@ describe('transform schema key changes', () => {
     var input = {
       somePointer: {__type: 'Pointer', className: 'Micro', objectId: 'oft'}
     };
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input, {
+      fields: {somePointer: {type: 'Pointer'}}
+    });
     expect(typeof output._p_somePointer).toEqual('string');
     expect(output._p_somePointer).toEqual('Micro$oft');
     done();
@@ -206,7 +220,9 @@ describe('transform schema key changes', () => {
     var input = {
       userPointer: {__type: 'Pointer', className: '_User', objectId: 'qwerty'}
     };
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input, {
+      fields: {userPointer: {type: 'Pointer'}}
+    });
     expect(typeof output._p_userPointer).toEqual('string');
     expect(output._p_userPointer).toEqual('_User$qwerty');
     done();
@@ -219,7 +235,7 @@ describe('transform schema key changes', () => {
         "Kevin": { "write": true }
       }
     };
-    var output = transform.parseObjectToMongoObject(dummySchema, null, input);
+    var output = transform.parseObjectToMongoObjectForCreate(dummySchema, null, input);
     expect(typeof output._rperm).toEqual('object');
     expect(typeof output._wperm).toEqual('object');
     expect(output.ACL).toBeUndefined();

spec/Parse.Push.spec.js

@@ -1,5 +1,6 @@
 'use strict';
 
+let request = require('request');
 
 describe('Parse.Push', () => {
 
@@ -89,4 +90,57 @@ describe('Parse.Push', () => {
       done();
     });
   });
+
+  it('should not allow clients to query _PushStatus', done => {
+    setup()
+    .then(() => Parse.Push.send({
+      where: {
+        deviceType: 'ios'
+      },
+      data: {
+        badge: 'increment',
+        alert: 'Hello world!'
+      }
+    }, {useMasterKey: true}))
+    .then(() => {
+      request.get({
+        url: 'http://localhost:8378/1/classes/_PushStatus',
+        json: true,
+        headers: {
+          'X-Parse-Application-Id': 'test',
+        },
+      }, (error, response, body) => {
+        expect(body.results.length).toEqual(0);
+        done();
+      });
+    });
+  });
+
+  it('should allow master key to query _PushStatus', done => {
+    setup()
+    .then(() => Parse.Push.send({
+      where: {
+        deviceType: 'ios'
+      },
+      data: {
+        badge: 'increment',
+        alert: 'Hello world!'
+      }
+    }, {useMasterKey: true}))
+    .then(() => {
+      request.get({
+        url: 'http://localhost:8378/1/classes/_PushStatus',
+        json: true,
+        headers: {
+          'X-Parse-Application-Id': 'test',
+          'X-Parse-Master-Key': 'test',
+        },
+      }, (error, response, body) => {
+        expect(body.results.length).toEqual(1);
+        expect(body.results[0].query).toEqual('{"deviceType":"ios"}');
+        expect(body.results[0].payload).toEqual('{"badge":"increment","alert":"Hello world!"}');
+        done();
+      });
+    });
+  });
 });

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -145,14 +145,16 @@ export class MongoStorageAdapter {
   // this adapter doesn't know about the schema, return a promise that rejects with
   // undefined as the reason.
   getOneSchema(className) {
-    return this.schemaCollection().then(schemasCollection => schemasCollection._fechOneSchemaFrom_SCHEMA(className));
+    return this.schemaCollection()
+    .then(schemasCollection => schemasCollection._fechOneSchemaFrom_SCHEMA(className));
   }
 
-  // TODO: As yet not particularly well specified. Creates an object. Does it really need the schema?
+  // TODO: As yet not particularly well specified. Creates an object. Shouldn't need the
-  // or can it fetch the schema itself? Also the schema is not currently a Parse format schema, and it
+  // schemaController, but MongoTransform still needs it :( maybe shouldn't even need the schema,
-  // should be, if we are passing it at all.
+  // and should infer from the type. Or maybe does need the schema for validations. Or maybe needs
-  createObject(className, object, schema) {
+  // the schem only for the legacy mongo format. We'll figure that out later.
-    const mongoObject = transform.parseObjectToMongoObject(schema, className, object);
+  createObject(className, object, schemaController, parseFormatSchema) {
+    const mongoObject = transform.parseObjectToMongoObjectForCreate(schemaController, className, object, parseFormatSchema);
     return this.adaptiveCollection(className)
     .then(collection => collection.insertOne(mongoObject));
   }

src/Adapters/Storage/Mongo/MongoTransform.js

@@ -21,9 +21,13 @@ var Parse = require('parse/node').Parse;
 // validate: true indicates that key names are to be validated.
 //
 // Returns an object with {key: key, value: value}.
-export function transformKeyValue(schema, className, restKey, restValue, options) {
+export function transformKeyValue(schema, className, restKey, restValue, {
-  options = options || {};
+  inArray,
-
+  inObject,
+  query,
+  update,
+  validate,
+} = {}) {
   // Check if the schema is known since it's a built-in field.
   var key = restKey;
   var timeField = false;
@@ -62,7 +66,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
     return {key: key, value: restValue};
     break;
   case '$or':
-    if (!options.query) {
+    if (!query) {
       throw new Parse.Error(Parse.Error.INVALID_KEY_NAME,
                             'you can only use $or in queries');
     }
@@ -75,7 +79,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
     });
     return {key: '$or', value: mongoSubqueries};
   case '$and':
-    if (!options.query) {
+    if (!query) {
       throw new Parse.Error(Parse.Error.INVALID_KEY_NAME,
                             'you can only use $and in queries');
     }
@@ -91,7 +95,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
     // Other auth data
     var authDataMatch = key.match(/^authData\.([a-zA-Z0-9_]+)\.id$/);
     if (authDataMatch) {
-      if (options.query) {
+      if (query) {
         var provider = authDataMatch[1];
         // Special-case auth data.
         return {key: '_auth_data_'+provider+'.id', value: restValue};
@@ -100,7 +104,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
                             'can only query on ' + key);
       break;
     };
-    if (options.validate && !key.match(/^[a-zA-Z][a-zA-Z0-9_\.]*$/)) {
+    if (validate && !key.match(/^[a-zA-Z][a-zA-Z0-9_\.]*$/)) {
       throw new Parse.Error(Parse.Error.INVALID_KEY_NAME,
                             'invalid key name: ' + key);
     }
@@ -117,24 +121,24 @@ export function transformKeyValue(schema, className, restKey, restValue, options
       (!expected && restValue && restValue.__type == 'Pointer')) {
     key = '_p_' + key;
   }
-  var inArray = (expected && expected.type === 'Array');
+  var expectedTypeIsArray = (expected && expected.type === 'Array');
 
   // Handle query constraints
-  if (options.query) {
+  if (query) {
-    value = transformConstraint(restValue, inArray);
+    value = transformConstraint(restValue, expectedTypeIsArray);
     if (value !== CannotTransform) {
       return {key: key, value: value};
     }
   }
 
-  if (inArray && options.query && !(restValue instanceof Array)) {
+  if (expectedTypeIsArray && query && !(restValue instanceof Array)) {
     return {
       key: key, value: { '$all' : [restValue] }
     };
   }
 
   // Handle atomic values
-  var value = transformAtom(restValue, false, options);
+  var value = transformAtom(restValue, false, { inArray, inObject });
   if (value !== CannotTransform) {
     if (timeField && (typeof value === 'string')) {
       value = new Date(value);
@@ -150,7 +154,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
 
   // Handle arrays
   if (restValue instanceof Array) {
-    if (options.query) {
+    if (query) {
       throw new Parse.Error(Parse.Error.INVALID_JSON,
                             'cannot use array as query param');
     }
@@ -162,7 +166,7 @@ export function transformKeyValue(schema, className, restKey, restValue, options
   }
 
   // Handle update operators
-  value = transformUpdateOperator(restValue, !options.update);
+  value = transformUpdateOperator(restValue, !update);
   if (value !== CannotTransform) {
     return {key: key, value: value};
   }
@@ -198,18 +202,114 @@ function transformWhere(schema, className, restWhere, options = {validate: true}
   return mongoWhere;
 }
 
+const parseObjectKeyValueToMongoObjectKeyValue = (
+  schema,
+  className,
+  restKey,
+  restValue,
+  parseFormatSchema
+) => {
+  // Check if the schema is known since it's a built-in field.
+  let transformedValue;
+  let coercedToDate;
+  switch(restKey) {
+  case 'objectId': return {key: '_id', value: restValue};
+  case '_created_at'://TODO: for some reason, _PushStatus is already transformed when it gets here. For now,
+  // just pass the _created_at through. Later, we should make sure the push status doesn't get transformed inside Parse Server.
+  case 'createdAt':
+    transformedValue = transformAtom(restValue, false);
+    coercedToDate = typeof transformedValue === 'string' ? new Date(transformedValue) : transformedValue
+    return {key: '_created_at', value: coercedToDate};
+  case 'updatedAt':
+    transformedValue = transformAtom(restValue, false);
+    coercedToDate = typeof transformedValue === 'string' ? new Date(transformedValue) : transformedValue
+    return {key: '_updated_at', value: coercedToDate};
+  case 'expiresAt':
+    transformedValue = transformAtom(restValue, false);
+    coercedToDate = typeof transformedValue === 'string' ? new Date(transformedValue) : transformedValue
+    return {key: 'expiresAt', value: coercedToDate};
+  case '_id': //TODO: for some reason, _PushStatus is already transformed when it gets here. For now,
+  // just pass the ID through. Later, we should make sure the push status doesn't get transformed inside Parse Server.
+  case '_rperm':
+  case '_wperm':
+  case '_email_verify_token':
+  case '_hashed_password':
+  case '_perishable_token': return {key: restKey, value: restValue};
+  case 'sessionToken': return {key: '_session_token', value: restValue};
+  default:
+    // Auth data should have been transformed already
+    if (restKey.match(/^authData\.([a-zA-Z0-9_]+)\.id$/)) {
+      throw new Parse.Error(Parse.Error.INVALID_KEY_NAME, 'can only query on ' + restKey);
+    }
+    // Trust that the auth data has been transformed and save it directly
+    if (restKey.match(/^_auth_data_[a-zA-Z0-9_]+$/)) {
+      return {key: restKey, value: restValue};
+    }
+  }
+  //skip straight to transformAtom for Bytes, they don't show up in the schema for some reason
+  if (restValue && restValue.__type !== 'Bytes') {
+    //Note: We may not know the type of a field here, as the user could be saving (null) to a field
+    //That never existed before, meaning we can't infer the type.
+    if (parseFormatSchema.fields[restKey] && parseFormatSchema.fields[restKey].type == 'Pointer' || restValue.__type == 'Pointer') {
+      restKey = '_p_' + restKey;
+    }
+  }
+
+  // Handle atomic values
+  var value = transformAtom(restValue, false, { inArray: false, inObject: false });
+  if (value !== CannotTransform) {
+    return {key: restKey, value: value};
+  }
+
+  // ACLs are handled before this method is called
+  // If an ACL key still exists here, something is wrong.
+  if (restKey === 'ACL') {
+    throw 'There was a problem transforming an ACL.';
+  }
+
+  // Handle arrays
+  if (restValue instanceof Array) {
+    value = restValue.map((restObj) => {
+      var out = transformKeyValue(schema, className, restKey, restObj, { inArray: true });
+      return out.value;
+    });
+    return {key: restKey, value: value};
+  }
+
+  // Handle update operators. TODO: handle within Parse Server. DB adapter shouldn't see update operators in creates.
+  value = transformUpdateOperator(restValue, true);
+  if (value !== CannotTransform) {
+    return {key: restKey, value: value};
+  }
+
+  // Handle normal objects by recursing
+  value = {};
+  for (var subRestKey in restValue) {
+    var subRestValue = restValue[subRestKey];
+    var out = transformKeyValue(schema, className, subRestKey, subRestValue, { inObject: true });
+    // For recursed objects, keep the keys in rest format
+    value[subRestKey] = out.value;
+  }
+  return {key: restKey, value: value};
+}
+
 // Main exposed method to create new objects.
 // restCreate is the "create" clause in REST API form.
-// Returns the mongo form of the object.
+function parseObjectToMongoObjectForCreate(schema, className, restCreate, parseFormatSchema) {
-function parseObjectToMongoObject(schema, className, restCreate) {
   if (className == '_User') {
      restCreate = transformAuthData(restCreate);
   }
   var mongoCreate = transformACL(restCreate);
-  for (var restKey in restCreate) {
+  for (let restKey in restCreate) {
-    var out = transformKeyValue(schema, className, restKey, restCreate[restKey]);
+    let { key, value } = parseObjectKeyValueToMongoObjectKeyValue(
-    if (out.value !== undefined) {
+      schema,
-      mongoCreate[out.key] = out.value;
+      className,
+      restKey,
+      restCreate[restKey],
+      parseFormatSchema
+    );
+    if (value !== undefined) {
+      mongoCreate[key] = value;
     }
   }
   return mongoCreate;
@@ -920,7 +1020,7 @@ var FileCoder = {
 
 module.exports = {
   transformKey,
-  parseObjectToMongoObject,
+  parseObjectToMongoObjectForCreate,
   transformUpdate,
   transformWhere,
   transformSelect,

src/Controllers/DatabaseController.js

@@ -312,24 +312,19 @@ DatabaseController.prototype.create = function(className, object, options = {})
   let originalObject = object;
   object = deepcopy(object);
 
-  var schema;
   var isMaster = !('acl' in options);
   var aclGroup = options.acl || [];
 
   return this.validateClassName(className)
-    .then(() => this.loadSchema())
+  .then(() => this.loadSchema())
-    .then(s => {
+  .then(schemaController => {
-      schema = s;
+    return (isMaster ? Promise.resolve() : schemaController.validatePermission(className, aclGroup, 'create'))
-      if (!isMaster) {
-        return schema.validatePermission(className, aclGroup, 'create');
-      }
-      return Promise.resolve();
-    })
     .then(() => this.handleRelationUpdates(className, null, object))
-    .then(() => this.adapter.createObject(className, object, schema))
+    .then(() => schemaController.enforceClassExists(className))
-    .then(result => {
+    .then(() => schemaController.getOneSchema(className))
-      return sanitizeDatabaseResult(originalObject, result.ops[0]);
+    .then(schema => this.adapter.createObject(className, object, schemaController, schema))
-    });
+    .then(result => sanitizeDatabaseResult(originalObject, result.ops[0]));
+  })
 };
 
 DatabaseController.prototype.canAddField = function(schema, className, object, aclGroup) {

src/Controllers/SchemaController.js

@@ -91,7 +91,7 @@ const requiredColumns = Object.freeze({
   _Role: ["name", "ACL"]
 });
 
-const systemClasses = Object.freeze(['_User', '_Installation', '_Role', '_Session', '_Product']);
+const systemClasses = Object.freeze(['_User', '_Installation', '_Role', '_Session', '_Product', '_PushStatus']);
 
 // 10 alpha numberic chars + uppercase
 const userIdRegex = /^[a-zA-Z0-9]{10}$/;
@@ -341,12 +341,8 @@ class SchemaController {
 
   // Returns a promise that resolves successfully to the new schema
   // object or fails with a reason.
-  // If 'freeze' is true, refuse to update the schema.
+  // If 'freeze' is true, refuse to modify the schema.
-  // WARNING: this function has side-effects, and doesn't actually
+  enforceClassExists(className, freeze) {
-  // do any validation of the format of the className. You probably
-  // should use classNameIsValid or addClassIfNotExists or something
-  // like that instead. TODO: rename or remove this function.
-  validateClassName(className, freeze) {
     if (this.data[className]) {
       return Promise.resolve(this);
     }
@@ -366,7 +362,7 @@ class SchemaController {
       return this.reloadData();
     }).then(() => {
       // Ensure that the schema now validates
-      return this.validateClassName(className, true);
+      return this.enforceClassExists(className, true);
     }, () => {
       // The schema still doesn't validate. Give up
       throw new Parse.Error(Parse.Error.INVALID_JSON, 'schema class name does not revalidate');
@@ -547,7 +543,7 @@ class SchemaController {
   // valid.
   validateObject(className, object, query) {
     let geocount = 0;
-    let promise = this.validateClassName(className);
+    let promise = this.enforceClassExists(className);
     for (let fieldName in object) {
       if (object[fieldName] === undefined) {
         continue;
@@ -642,15 +638,6 @@ class SchemaController {
     return this.reloadData().then(() => !!(this.data[className]));
   }
 
-  // Helper function to check if a field is a pointer, returns true or false.
-  isPointer(className, key) {
-    let expected = this.getExpectedType(className, key);
-    if (expected && expected.charAt(0) == '*') {
-      return true;
-    }
-    return false;
-  };
-
   getRelationFields(className) {
     if (this.data && this.data[className]) {
       let classData = this.data[className];