Merge branch 'master' of https://github.com/ParsePlatform/parse-server into mcdonald-gcs-adapter

Get GCSAdapter up to snuff with FilesController + FilesControllerTestFactory

* 'master' of https://github.com/ParsePlatform/parse-server: (102 commits)
  Remove duplicated instructions
  Release and Changelog for 2.1.4
  fixes missing coverage with sh script
  Fix update system schema
  Adds optional COVERAGE
  Allows to pass no where in $select clause
  Sanitize objectId in
  Fix delete schema when actual collection does not exist
  Fix replace query overwrite the existing query object.
  Fix create system class with relation/pointer
  Use throws syntax for errors in SchemasRouter.
  Completely migrate SchemasRouter to new MongoCollection API.
  Add tests that verify installationId in Cloud Code triggers.
  Propagate installationId in all Cloud Code triggers.
  Add test
  expiresAt should be a Date, not a string. Fixes #776
  Fix missing 'let/var' in OneSignalPushAdapter.spec.
  Don't run any afterSave hooks if none are registered.
  Fix : remove query count limit
  Flatten custom operations in request.object in afterSave hooks.
  ...

src/Adapters/AdapterLoader.js

@@ -1,35 +1,43 @@
+export function loadAdapter(adapter, defaultAdapter, options) {
 
-export function loadAdapter(options, defaultAdapter) {
-  let adapter;
-
-  // We have options and options have adapter key
-  if (options) {
-    // Pass an adapter as a module name, a function or an instance
-    if (typeof options == "string" || typeof options == "function" || options.constructor != Object) {
-      adapter = options;
+  if (!adapter)  
+  {
+    if (!defaultAdapter) {
+      return options;
     }
-    if (options.adapter) {
-      adapter = options.adapter;
+    // Load from the default adapter when no adapter is set
+    return loadAdapter(defaultAdapter, undefined, options);
+  } else if (typeof adapter === "function") {
+    try {
+      return adapter(options);
+    } catch(e) {
+      var Adapter = adapter;
+      return new Adapter(options);
     }
-  }
-  
-  if (!adapter) {
-    adapter = defaultAdapter;
-  }
-
-  // This is a string, require the module
-  if (typeof adapter === "string") {
+  } else if (typeof adapter === "string") {
     adapter = require(adapter);
     // If it's define as a module, get the default
     if (adapter.default) {
       adapter = adapter.default;
     }
+    
+    return loadAdapter(adapter, undefined, options);
+  } else if (adapter.module) {
+    return loadAdapter(adapter.module, undefined, adapter.options);
+  } else if (adapter.class) {
+    return loadAdapter(adapter.class, undefined, adapter.options);
+  } else if (adapter.adapter) {
+    return loadAdapter(adapter.adapter, undefined, adapter.options);
+  } else {
+    // Try to load the defaultAdapter with the options
+    // The default adapter should throw if the options are
+    // incompatible
+    try {
+      return loadAdapter(defaultAdapter, undefined, adapter);
+    } catch (e) {};
   }
-  // From there it's either a function or an object
-  // if it's an function, instanciate and pass the options 
-  if (typeof adapter === "function") {
-    var Adapter = adapter;
-    adapter = new Adapter(options);
-  }
-  return adapter;
+  // return the adapter as is as it's unusable otherwise
+  return adapter;     
 }
+
+export default loadAdapter;

src/Adapters/Email/MailAdapter.js

@@ -0,0 +1,23 @@
+
+/*
+  Mail Adapter prototype
+  A MailAdapter should implement at least sendMail()
+ */
+export class MailAdapter {
+  /*
+   * A method for sending mail
+   * @param options would have the parameters
+   * - to: the recipient
+   * - text: the raw text of the message
+   * - subject: the subject of the email
+   */
+  sendMail(options) {}
+  
+  /* You can implement those methods if you want
+   * to provide HTML templates etc...
+   */
+  // sendVerificationEmail({ link, appName, user }) {}
+  // sendPasswordResetEmail({ link, appName, user }) {}
+}
+
+export default MailAdapter;

src/Adapters/Email/SimpleMailgunAdapter.js

@@ -0,0 +1,32 @@
+import Mailgun from 'mailgun-js';
+
+let SimpleMailgunAdapter = mailgunOptions => {
+	if (!mailgunOptions || !mailgunOptions.apiKey || !mailgunOptions.domain) {
+		throw 'SimpleMailgunAdapter requires an API Key and domain.';
+	}
+	let mailgun = Mailgun(mailgunOptions);
+
+	let sendMail = ({to, subject, text}) => {
+		let data = {
+			from: mailgunOptions.fromAddress,
+			to: to,
+			subject: subject,
+			text: text,
+		}
+
+		return new Promise((resolve, reject) => {
+			mailgun.messages().send(data, (err, body) => {
+				if (typeof err !== 'undefined') {
+					reject(err);
+				}
+				resolve(body);
+			});
+		});
+	}
+
+	return Object.freeze({
+    sendMail: sendMail
+	});
+}
+
+module.exports = SimpleMailgunAdapter

src/Adapters/Files/FilesAdapter.js

@@ -8,11 +8,23 @@
 // * getFileLocation(config, request, filename)
 //
 // Default is GridStoreAdapter, which requires mongo
-// and for the API server to be using the ExportAdapter
+// and for the API server to be using the DatabaseController with Mongo
 // database adapter.
 
 export class FilesAdapter {
-  createFile(config, filename, data) { }
+  /* this method is responsible to store the file in order to be retrived later by it's file name
+   * 
+   * 
+   * @param config the current config
+   * @param filename the filename to save
+   * @param data the buffer of data from the file
+   * @param contentType the supposed contentType
+   * @discussion the contentType can be undefined if the controller was not able to determine it 
+   * 
+   * @return a promise that should fail if the storage didn't succeed
+   * 
+   */
+  createFile(config, filename: string, data, contentType: string) { }
 
   deleteFile(config, filename) { }
 

src/Adapters/Files/GCSAdapter.js

@@ -1,16 +1,18 @@
 // GCSAdapter
 // Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage
-import * as gcloud from 'gcloud';
+import { storage } from 'gcloud';
 import { FilesAdapter } from './FilesAdapter';
+import requiredParameter from '../../requiredParameter';
 
 export class GCSAdapter extends FilesAdapter {
   // GCS Project ID and the name of a corresponding Keyfile are required.
+  // Unlike the S3 adapter, you must create a new Cloud Storage bucket, as this is not created automatically.
   // See https://googlecloudplatform.github.io/gcloud-node/#/docs/master/guides/authentication
   // for more details.
   constructor(
-    projectId,
-    keyFilename,
-    bucket,
+    projectId = requiredParameter('GCSAdapter requires a GCP Project ID'),
+    keyFilename = requiredParameter('GCSAdapter requires a GCP keyfile'),
+    bucket = requiredParameter('GCSAdapter requires a GCS bucket name'),
     { bucketPrefix = '',
       directAccess = false } = {}
   ) {
@@ -20,21 +22,25 @@ export class GCSAdapter extends FilesAdapter {
     this._bucketPrefix = bucketPrefix;
     this._directAccess = directAccess;
 
-    let gcsOptions = {
+    let options = {
       projectId: projectId,
       keyFilename: keyFilename
     };
 
-    this._gcsClient = new gcloud.storage(gcsOptions);
+    this._gcsClient = new storage(options);
   }
 
   // For a given config object, filename, and data, store a file in GCS.
   // Resolves the promise or fails with an error.
-  createFile(config, filename, data) {
+  createFile(config, filename, data, contentType) {
+    let params = {
+      contentType: contentType || 'application/octet-stream'
+    };
+
     return new Promise((resolve, reject) => {
       let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
       // gcloud supports upload(file) not upload(bytes), so we need to stream.
-      var uploadStream = file.createWriteStream();
+      var uploadStream = file.createWriteStream(params);
       uploadStream.on('error', (err) => {
         return reject(err);
       }).on('finish', () => {
@@ -61,6 +67,7 @@ export class GCSAdapter extends FilesAdapter {
     return new Promise((resolve, reject) => {
       let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
       file.delete((err, res) => {
+        console.log("delete: ", filename, err, res);
         if(err !== null) {
           return reject(err);
         }
@@ -74,11 +81,19 @@ export class GCSAdapter extends FilesAdapter {
   getFileData(config, filename) {
     return new Promise((resolve, reject) => {
       let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
-      file.download((err, data) => {
-        if (err !== null) {
-          return reject(err);
+      // Check for existence, since gcloud-node seemed to be caching the result
+      file.exists((err, exists) => {
+        if (exists) {
+          file.download((err, data) => {
+            console.log("get: ", filename, err, data);
+            if (err !== null) {
+              return reject(err);
+            }
+            return resolve(data);
+          });
+        } else {
+          reject(err);
         }
-        resolve(data);
       });
     });
   }

src/Adapters/Files/GridStoreAdapter.js

@@ -1,28 +1,47 @@
-// GridStoreAdapter
-//
-// Stores files in Mongo using GridStore
-// Requires the database adapter to be based on mongoclient
+/**
+ GridStoreAdapter
+ Stores files in Mongo using GridStore
+ Requires the database adapter to be based on mongoclient
 
-import { GridStore } from 'mongodb';
+ @flow weak
+ */
+
+import { MongoClient, GridStore, Db} from 'mongodb';
 import { FilesAdapter } from './FilesAdapter';
 
 export class GridStoreAdapter extends FilesAdapter {
+  _databaseURI: string;
+  _connectionPromise: Promise<Db>;
+
+  constructor(mongoDatabaseURI: string) {
+    super();
+    this._databaseURI = mongoDatabaseURI;
+    this._connect();
+  }
+
+  _connect() {
+    if (!this._connectionPromise) {
+      this._connectionPromise = MongoClient.connect(this._databaseURI);
+    }
+    return this._connectionPromise;
+  }
+
   // For a given config object, filename, and data, store a file
   // Returns a promise
-  createFile(config, filename, data) {
-    return config.database.connect().then(() => {
-      let gridStore = new GridStore(config.database.db, filename, 'w');
+  createFile(config, filename: string, data, contentType) {
+    return this._connect().then(database => {
+      let gridStore = new GridStore(database, filename, 'w');
       return gridStore.open();
-    }).then((gridStore) => {
+    }).then(gridStore => {
       return gridStore.write(data);
-    }).then((gridStore) => {
+    }).then(gridStore => {
       return gridStore.close();
     });
   }
 
-  deleteFile(config, filename) {
-    return config.database.connect().then(() => {
-      let gridStore = new GridStore(config.database.db, filename, 'w');
+  deleteFile(config, filename: string) {
+    return this._connect().then(database => {
+      let gridStore = new GridStore(database, filename, 'w');
       return gridStore.open();
     }).then((gridStore) => {
       return gridStore.unlink();
@@ -31,13 +50,14 @@ export class GridStoreAdapter extends FilesAdapter {
     });
   }
 
-  getFileData(config, filename) {
-    return config.database.connect().then(() => {
-      return GridStore.exist(config.database.db, filename);
-    }).then(() => {
-      let gridStore = new GridStore(config.database.db, filename, 'r');
-      return gridStore.open();
-    }).then((gridStore) => {
+  getFileData(config, filename: string) {
+    return this._connect().then(database => {
+      return GridStore.exist(database, filename)
+        .then(() => {
+          let gridStore = new GridStore(database, filename, 'r');
+          return gridStore.open();
+        });
+    }).then(gridStore => {
       return gridStore.read();
     });
   }

src/Adapters/Files/S3Adapter.js

@@ -4,23 +4,38 @@
 
 import * as AWS from 'aws-sdk';
 import { FilesAdapter } from './FilesAdapter';
+import requiredParameter from '../../requiredParameter';
 
 const DEFAULT_S3_REGION = "us-east-1";
 
+function parseS3AdapterOptions(...options) {
+  if (options.length === 1 && typeof options[0] == "object") {
+    return options;
+  }
+  
+  const additionalOptions = options[3] || {};
+  
+  return {
+    accessKey: options[0],
+    secretKey: options[1],
+    bucket: options[2],
+    region: additionalOptions.region
+  }
+}
+
 export class S3Adapter extends FilesAdapter {
   // Creates an S3 session.
   // Providing AWS access and secret keys is mandatory
   // Region and bucket will use sane defaults if omitted
   constructor(
-    accessKey,
-    secretKey,
-    bucket,
-    { region = DEFAULT_S3_REGION,
-      bucketPrefix = '',
-      directAccess = false } = {}
-  ) {
+     accessKey = requiredParameter('S3Adapter requires an accessKey'),
+     secretKey = requiredParameter('S3Adapter requires a secretKey'),
+     bucket,
+     { region = DEFAULT_S3_REGION,
+       bucketPrefix = '',
+       directAccess = false } = {}) {
     super();
-
+    
     this._region = region;
     this._bucket = bucket;
     this._bucketPrefix = bucketPrefix;
@@ -33,11 +48,27 @@ export class S3Adapter extends FilesAdapter {
     };
     AWS.config._region = this._region;
     this._s3Client = new AWS.S3(s3Options);
+    this._hasBucket = false;
+  }
+  
+  createBucket() {
+    var promise;
+    if (this._hasBucket) {
+      promise = Promise.resolve();
+    } else {
+      promise = new Promise((resolve, reject) => {
+        this._s3Client.createBucket(() => {
+          this._hasBucket = true;
+          resolve();
+        });
+      }); 
+    }
+    return promise;
   }
 
   // For a given config object, filename, and data, store a file in S3
   // Returns a promise containing the S3 object creation response
-  createFile(config, filename, data) {
+  createFile(config, filename, data, contentType) {
     let params = {
       Key: this._bucketPrefix + filename,
       Body: data
@@ -45,26 +76,33 @@ export class S3Adapter extends FilesAdapter {
     if (this._directAccess) {
       params.ACL = "public-read"
     }
-    return new Promise((resolve, reject) => {
-      this._s3Client.upload(params, (err, data) => {
-        if (err !== null) {
-          return reject(err);
-        }
-        resolve(data);
+    if (contentType) {
+      params.ContentType = contentType;
+    }
+    return this.createBucket().then(() => {
+      return new Promise((resolve, reject) => {
+        this._s3Client.upload(params, (err, data) => {
+          if (err !== null) {
+            return reject(err);
+          }
+          resolve(data);
+        });
       });
     });
   }
 
   deleteFile(config, filename) {
-    return new Promise((resolve, reject) => {
-      let params = {
-        Key: this._bucketPrefix + filename
-      };
-      this._s3Client.deleteObject(params, (err, data) =>{
-        if(err !== null) {
-          return reject(err);
-        }
-        resolve(data);
+    return this.createBucket().then(() => {
+      return new Promise((resolve, reject) => {
+        let params = {
+          Key: this._bucketPrefix + filename
+        };
+        this._s3Client.deleteObject(params, (err, data) =>{
+          if(err !== null) {
+            return reject(err);
+          }
+          resolve(data);
+        });
       });
     });
   }
@@ -73,12 +111,18 @@ export class S3Adapter extends FilesAdapter {
   // Returns a promise that succeeds with the buffer result from S3
   getFileData(config, filename) {
     let params = {Key: this._bucketPrefix + filename};
-    return new Promise((resolve, reject) => {
-      this._s3Client.getObject(params, (err, data) => {
-        if (err !== null) {
-          return reject(err);
-        }
-        resolve(data.Body);
+    return this.createBucket().then(() => {
+      return new Promise((resolve, reject) => {
+        this._s3Client.getObject(params, (err, data) => {
+          if (err !== null) {
+            return reject(err);
+          }
+          // Something happend here...
+          if (data && !data.Body) {
+            return reject(data);
+          }
+          resolve(data.Body);
+        });
       });
     });
   }

src/Adapters/Logger/FileLoggerAdapter.js

@@ -101,7 +101,6 @@ let _verifyTransports = ({infoLogger, errorLogger, logsFolder}) => {
 export class FileLoggerAdapter extends LoggerAdapter {
   constructor(options = {}) {
     super();
-
     this._logsFolder = options.logsFolder || LOGS_FOLDER;
 
     // check logs folder exists

src/Adapters/Push/OneSignalPushAdapter.js

@@ -18,6 +18,10 @@ export class OneSignalPushAdapter extends PushAdapter {
     this.validPushTypes = ['ios', 'android'];
     this.senderMap = {};
     this.OneSignalConfig = {};
+    const { oneSignalAppId, oneSignalApiKey } = pushConfig;
+    if (!oneSignalAppId || !oneSignalApiKey) {
+      throw "Trying to initialize OneSignalPushAdapter without oneSignalAppId or oneSignalApiKey";
+    }
     this.OneSignalConfig['appId'] = pushConfig['oneSignalAppId'];
     this.OneSignalConfig['apiKey'] = pushConfig['oneSignalApiKey'];
     

src/Adapters/Push/ParsePushAdapter.js

@@ -14,6 +14,10 @@ export class ParsePushAdapter extends PushAdapter {
     super(pushConfig);
     this.validPushTypes = ['ios', 'android'];
     this.senderMap = {};
+    // used in PushController for Dashboard Features
+    this.feature = {
+      immediatePush: true
+    };
     let pushTypes = Object.keys(pushConfig);
     
     for (let pushType of pushTypes) {

src/Adapters/Storage/Mongo/MongoCollection.js

@@ -0,0 +1,76 @@
+
+let mongodb = require('mongodb');
+let Collection = mongodb.Collection;
+
+export default class MongoCollection {
+  _mongoCollection:Collection;
+
+  constructor(mongoCollection:Collection) {
+    this._mongoCollection = mongoCollection;
+  }
+
+  // Does a find with "smart indexing".
+  // Currently this just means, if it needs a geoindex and there is
+  // none, then build the geoindex.
+  // This could be improved a lot but it's not clear if that's a good
+  // idea. Or even if this behavior is a good idea.
+  find(query, { skip, limit, sort } = {}) {
+    return this._rawFind(query, { skip, limit, sort })
+      .catch(error => {
+        // Check for "no geoindex" error
+        if (error.code != 17007 ||
+          !error.message.match(/unable to find index for .geoNear/)) {
+          throw error;
+        }
+        // Figure out what key needs an index
+        let key = error.message.match(/field=([A-Za-z_0-9]+) /)[1];
+        if (!key) {
+          throw error;
+        }
+
+        var index = {};
+        index[key] = '2d';
+        //TODO: condiser moving index creation logic into Schema.js
+        return this._mongoCollection.createIndex(index)
+          // Retry, but just once.
+          .then(() => this._rawFind(query, { skip, limit, sort }));
+      });
+  }
+
+  _rawFind(query, { skip, limit, sort } = {}) {
+    return this._mongoCollection
+      .find(query, { skip, limit, sort })
+      .toArray();
+  }
+
+  count(query, { skip, limit, sort } = {}) {
+    return this._mongoCollection.count(query, { skip, limit, sort });
+  }
+
+  // Atomically finds and updates an object based on query.
+  // The result is the promise with an object that was in the database !AFTER! changes.
+  // Postgres Note: Translates directly to `UPDATE * SET * ... RETURNING *`, which will return data after the change is done.
+  findOneAndUpdate(query, update) {
+    // arguments: query, sort, update, options(optional)
+    // Setting `new` option to true makes it return the after document, not the before one.
+    return this._mongoCollection.findAndModify(query, [], update, { new: true }).then(document => {
+      // Value is the object where mongo returns multiple fields.
+      return document.value;
+    })
+  }
+
+  // Atomically find and delete an object based on query.
+  // The result is the promise with an object that was in the database before deleting.
+  // Postgres Note: Translates directly to `DELETE * FROM ... RETURNING *`, which will return data after delete is done.
+  findOneAndDelete(query) {
+    // arguments: query, sort
+    return this._mongoCollection.findAndRemove(query, []).then(document => {
+      // Value is the object where mongo returns multiple fields.
+      return document.value;
+    });
+  }
+
+  drop() {
+    return this._mongoCollection.drop();
+  }
+}

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -0,0 +1,68 @@
+
+import MongoCollection from './MongoCollection';
+
+let mongodb = require('mongodb');
+let MongoClient = mongodb.MongoClient;
+
+export class MongoStorageAdapter {
+  // Private
+  _uri: string;
+  // Public
+  connectionPromise;
+  database;
+
+  constructor(uri: string) {
+    this._uri = uri;
+  }
+
+  connect() {
+    if (this.connectionPromise) {
+      return this.connectionPromise;
+    }
+
+    this.connectionPromise = MongoClient.connect(this._uri).then(database => {
+      this.database = database;
+    });
+    return this.connectionPromise;
+  }
+
+  collection(name: string) {
+    return this.connect().then(() => {
+      return this.database.collection(name);
+    });
+  }
+
+  adaptiveCollection(name: string) {
+    return this.connect()
+      .then(() => this.database.collection(name))
+      .then(rawCollection => new MongoCollection(rawCollection));
+  }
+
+  collectionExists(name: string) {
+    return this.connect().then(() => {
+      return this.database.listCollections({ name: name }).toArray();
+    }).then(collections => {
+      return collections.length > 0;
+    });
+  }
+
+  dropCollection(name: string) {
+    return this.collection(name).then(collection => collection.drop());
+  }
+  // Used for testing only right now.
+  collectionsContaining(match: string) {
+    return this.connect().then(() => {
+      return this.database.collections();
+    }).then(collections => {
+      return collections.filter(collection => {
+        if (collection.namespace.match(/\.system\./)) {
+          return false;
+        }
+        return (collection.collectionName.indexOf(match) == 0);
+      });
+    });
+  }
+}
+
+export default MongoStorageAdapter;
+module.exports = MongoStorageAdapter; // Required for tests