Merge pull request #708 from mcdonamp/mcdonald-gcs-adapter

Yet Another FileAdapter: Google Cloud Storage

.gitignore

@@ -40,3 +40,6 @@ lib/
 
 # cache folder
 .cache
+
+# Mac DS_Store files
+.DS_Store

README.md

@@ -135,9 +135,14 @@ PARSE_SERVER_MAX_UPLOAD_SIZE
 
 ```
 
-##### Configuring S3 Adapter
+##### Configuring File Adapters
+Parse Server allows developers to choose from several options when hosting files: the `GridStoreAdapter`, which backed by MongoDB; the `S3Adapter`, which is backed by [Amazon S3](https://aws.amazon.com/s3/); or the `GCSAdapter`, which is backed by [Google Cloud Storage](https://cloud.google.com/storage/).
 
-You can use the following environment variable setup the S3 adapter
+`GridStoreAdapter` is used by default and requires no setup, but if you're interested in using S3 or GCS, additional configuration information is available below.
+
+###### Configuring `S3Adapter`
+
+You can use the following environment variable setup to enable the S3 adapter:
 
 ```js
 S3_ACCESS_KEY
@@ -149,6 +154,19 @@ S3_DIRECT_ACCESS
 
 ```
 
+###### Configuring `GCSAdapter`
+
+You can use the following environment variable setup to enable the GCS adapter:
+
+```js
+GCP_PROJECT_ID
+GCP_KEYFILE_PATH
+GCS_BUCKET
+GCS_BUCKET_PREFIX
+GCS_DIRECT_ACCESS
+
+```
+
 ## Contributing
 
 We really want Parse to be yours, to see it grow and thrive in the open source community. Please see the [Contributing to Parse Server guide](CONTRIBUTING.md).

package.json

@@ -28,6 +28,7 @@
     "commander": "^2.9.0",
     "deepcopy": "^0.6.1",
     "express": "^4.13.4",
+    "gcloud": "^0.28.0",
     "mailgun-js": "^0.7.7",
     "mime": "^1.3.4",
     "mongodb": "~2.1.0",

spec/AdapterLoader.spec.js

@@ -3,6 +3,7 @@ var loadAdapter = require("../src/Adapters/AdapterLoader").loadAdapter;
 var FilesAdapter = require("../src/Adapters/Files/FilesAdapter").default;
 var ParsePushAdapter = require("../src/Adapters/Push/ParsePushAdapter");
 var S3Adapter = require("../src/Adapters/Files/S3Adapter").default;
+var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").default;
 
 describe("AdapterLoader", ()=>{
 
@@ -109,4 +110,13 @@ describe("AdapterLoader", ()=>{
     }).not.toThrow();
     done();
   })
+
+  it("should load GCSAdapter from direct passing", (done) => {
+    var gcsAdapter = new GCSAdapter("projectId", "path/to/keyfile", "bucket")
+    expect(() => {
+      var adapter = loadAdapter(gcsAdapter, FilesAdapter);
+      expect(adapter).toBe(gcsAdapter);
+    }).not.toThrow();
+    done();
+  })
 });

spec/FilesController.spec.js

@@ -1,6 +1,7 @@
 var FilesController = require('../src/Controllers/FilesController').FilesController;
 var GridStoreAdapter = require("../src/Adapters/Files/GridStoreAdapter").GridStoreAdapter;
 var S3Adapter = require("../src/Adapters/Files/S3Adapter").S3Adapter;
+var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").GCSAdapter;
 var Config = require("../src/Config");
 
 var FCTestFactory = require("./FilesControllerTestFactory");
@@ -30,4 +31,22 @@ describe("FilesController",()=>{
   } else if (!process.env.TRAVIS) {
     console.log("set S3_ACCESS_KEY and S3_SECRET_KEY to test S3Adapter")
   }
+
+  if (process.env.GCP_PROJECT_ID && process.env.GCP_KEYFILE_PATH && process.env.GCS_BUCKET) {
+
+    // Test the GCS Adapter
+    var gcsAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET);
+
+    FCTestFactory.testAdapter("GCSAdapter", gcsAdapter);
+
+    // Test GCS with direct access
+    var gcsDirectAccessAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET, {
+      directAccess: true
+    });
+
+    FCTestFactory.testAdapter("GCSAdapterDirect", gcsDirectAccessAdapter);
+
+  } else if (!process.env.TRAVIS) {
+    console.log("set GCP_PROJECT_ID, GCP_KEYFILE_PATH, and GCS_BUCKET to test GCSAdapter")
+  }
 });

spec/FilesControllerTestFactory.js

@@ -1,4 +1,3 @@
-
 var FilesController = require('../src/Controllers/FilesController').FilesController;
 var Config = require("../src/Config");
 

src/Adapters/AdapterLoader.js

@@ -1,5 +1,4 @@
 export function loadAdapter(adapter, defaultAdapter, options) {
-
   if (!adapter)
   {
     if (!defaultAdapter) {

src/Adapters/Files/GCSAdapter.js

@@ -0,0 +1,125 @@
+// GCSAdapter
+// Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage
+import { storage } from 'gcloud';
+import { FilesAdapter } from './FilesAdapter';
+import requiredParameter from '../../requiredParameter';
+
+function requiredOrFromEnvironment(env, name) {
+  let environmentVariable = process.env[env];
+  if (!environmentVariable) {
+    requiredParameter(`GCSAdapter requires an ${name}`);
+  }
+  return environmentVariable;
+}
+
+function fromEnvironmentOrDefault(env, defaultValue) {
+  let environmentVariable = process.env[env];
+  if (environmentVariable) {
+    return environmentVariable;
+  }
+  return defaultValue;
+}
+
+export class GCSAdapter extends FilesAdapter {
+  // GCS Project ID and the name of a corresponding Keyfile are required.
+  // Unlike the S3 adapter, you must create a new Cloud Storage bucket, as this is not created automatically.
+  // See https://googlecloudplatform.github.io/gcloud-node/#/docs/master/guides/authentication
+  // for more details.
+  constructor(
+    projectId = requiredOrFromEnvironment('GCP_PROJECT_ID', 'projectId'),
+    keyFilename = requiredOrFromEnvironment('GCP_KEYFILE_PATH', 'keyfile path'),
+    bucket = requiredOrFromEnvironment('GCS_BUCKET', 'bucket name'),
+    { bucketPrefix = fromEnvironmentOrDefault('GCS_BUCKET_PREFIX', ''),
+      directAccess = fromEnvironmentOrDefault('GCS_DIRECT_ACCESS', false) } = {}) {
+    super();
+
+    this._bucket = bucket;
+    this._bucketPrefix = bucketPrefix;
+    this._directAccess = directAccess;
+
+    let options = {
+      projectId: projectId,
+      keyFilename: keyFilename
+    };
+
+    this._gcsClient = new storage(options);
+  }
+
+  // For a given config object, filename, and data, store a file in GCS.
+  // Resolves the promise or fails with an error.
+  createFile(config, filename, data, contentType) {
+    let params = {
+      contentType: contentType || 'application/octet-stream'
+    };
+
+    return new Promise((resolve, reject) => {
+      let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
+      // gcloud supports upload(file) not upload(bytes), so we need to stream.
+      var uploadStream = file.createWriteStream(params);
+      uploadStream.on('error', (err) => {
+        return reject(err);
+      }).on('finish', () => {
+        // Second call to set public read ACL after object is uploaded.
+        if (this._directAccess) {
+          file.makePublic((err, res) => {
+            if (err !== null) {
+              return reject(err);
+            }
+            resolve();
+          });
+        } else {
+          resolve();
+        }
+      });
+      uploadStream.write(data);
+      uploadStream.end();
+    });
+  }
+
+  // Deletes a file with the given file name.
+  // Returns a promise that succeeds with the delete response, or fails with an error.
+  deleteFile(config, filename) {
+    return new Promise((resolve, reject) => {
+      let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
+      file.delete((err, res) => {
+        if(err !== null) {
+          return reject(err);
+        }
+        resolve(res);
+      });
+    });
+  }
+
+  // Search for and return a file if found by filename.
+  // Returns a promise that succeeds with the buffer result from GCS, or fails with an error.
+  getFileData(config, filename) {
+    return new Promise((resolve, reject) => {
+      let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
+      // Check for existence, since gcloud-node seemed to be caching the result
+      file.exists((err, exists) => {
+        if (exists) {
+          file.download((err, data) => {
+            if (err !== null) {
+              return reject(err);
+            }
+            return resolve(data);
+          });
+        } else {
+          reject(err);
+        }
+      });
+    });
+  }
+
+  // Generates and returns the location of a file stored in GCS for the given request and filename.
+  // The location is the direct GCS link if the option is set,
+  // otherwise we serve the file through parse-server.
+  getFileLocation(config, filename) {
+    if (this._directAccess) {
+      return `https://${this._bucket}.storage.googleapis.com/${this._bucketPrefix + filename}`;
+    }
+    return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename));
+  }
+}
+
+export default GCSAdapter;

src/index.js

@@ -24,6 +24,7 @@ import { FileLoggerAdapter }   from './Adapters/Logger/FileLoggerAdapter';
 import { FilesController }     from './Controllers/FilesController';
 import { FilesRouter }         from './Routers/FilesRouter';
 import { FunctionsRouter }     from './Routers/FunctionsRouter';
+import { GCSAdapter }          from './Adapters/Files/GCSAdapter';
 import { GlobalConfigRouter }  from './Routers/GlobalConfigRouter';
 import { GridStoreAdapter }    from './Adapters/Files/GridStoreAdapter';
 import { HooksController }     from './Controllers/HooksController';
@@ -259,4 +260,5 @@ function addParseCloud() {
 module.exports = {
   ParseServer: ParseServer,
   S3Adapter: S3Adapter,
+  GCSAdapter: GCSAdapter
 };