joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1172 from ParsePlatform/flovilmart.externalAdapters

Browse Source 
Moves Files adapters to external packages
This commit is contained in:
Florent Vilmart
2016-03-24 14:46:15 -04:00
parent 82ebba4db8 83f5207d44
commit f4ad4c1af5
12 changed files with 46 additions and 538 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
package.json
View File

@@ -19,7 +19,6 @@
"license": "BSD-3-Clause",
"dependencies": {
"apn": "^1.7.5",
"aws-sdk": "~2.2.33",
"babel-polyfill": "^6.5.0",
"babel-runtime": "^6.5.0",
"bcrypt-nodejs": "0.0.3",
@@ -28,7 +27,6 @@
"commander": "^2.9.0",
"deepcopy": "^0.6.1",
"express": "^4.13.4",
"gcloud": "^0.28.0",
"lru-cache": "^4.0.0",
"mailgun-js": "^0.7.7",
"mime": "^1.3.4",
@@ -36,6 +34,9 @@
"multer": "^1.1.0",
"node-gcm": "^0.14.0",
"parse": "^1.8.0",
"parse-server-fs-adapter": "^1.0.0",
"parse-server-gcs-adapter": "^1.0.0",
"parse-server-s3-adapter": "^1.0.0",
"redis": "^2.5.0-1",
"request": "^2.65.0",
"tv4": "^1.2.7",

12
spec/AdapterLoader.spec.js
View File

@@ -1,9 +1,9 @@
var loadAdapter = require("../src/Adapters/AdapterLoader").loadAdapter;
var FilesAdapter = require("../src/Adapters/Files/FilesAdapter").default;
var FilesAdapter = require("parse-server-fs-adapter").default;
var S3Adapter = require("parse-server-s3-adapter").default;
var GCSAdapter = require("parse-server-gcs-adapter").default;
var ParsePushAdapter = require("../src/Adapters/Push/ParsePushAdapter");
var S3Adapter = require("../src/Adapters/Files/S3Adapter").default;
var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").default;
describe("AdapterLoader", ()=>{
@@ -38,7 +38,11 @@ describe("AdapterLoader", ()=>{
adapter: adapterPath
});
expect(adapter instanceof FilesAdapter).toBe(true);
expect(typeof adapter).toBe('object');
expect(typeof adapter.createFile).toBe('function');
expect(typeof adapter.deleteFile).toBe('function');
expect(typeof adapter.getFileData).toBe('function');
expect(typeof adapter.getFileLocation).toBe('function');
done();
});

72
spec/FilesController.spec.js
View File

@@ -1,64 +1,30 @@
var FilesController = require('../src/Controllers/FilesController').FilesController;
var GridStoreAdapter = require("../src/Adapters/Files/GridStoreAdapter").GridStoreAdapter;
var S3Adapter = require("../src/Adapters/Files/S3Adapter").S3Adapter;
var GCSAdapter = require("../src/Adapters/Files/GCSAdapter").GCSAdapter;
var FileSystemAdapter = require("../src/Adapters/Files/FileSystemAdapter").FileSystemAdapter;
var Config = require("../src/Config");
var FCTestFactory = require("./FilesControllerTestFactory");
var FilesController = require('../src/Controllers/FilesController').default;
// Small additional tests to improve overall coverage
describe("FilesController",()=>{
describe("FilesController",() =>{
it("should properly expand objects", (done) => {
// Test the grid store adapter
var gridStoreAdapter = new GridStoreAdapter('mongodb://localhost:27017/parse');
FCTestFactory.testAdapter("GridStoreAdapter", gridStoreAdapter);
var config = new Config(Parse.applicationId);
var gridStoreAdapter = new GridStoreAdapter('mongodb://localhost:27017/parse');
var filesController = new FilesController(gridStoreAdapter)
var result = filesController.expandFilesInObject(config, function(){});
if (process.env.S3_ACCESS_KEY && process.env.S3_SECRET_KEY) {
expect(result).toBeUndefined();
// Test the S3 Adapter
var s3Adapter = new S3Adapter(process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY, 'parse.server.tests');
var fullFile = {
type: '__type',
url: "http://an.url"
}
FCTestFactory.testAdapter("S3Adapter",s3Adapter);
var anObject = {
aFile: fullFile
}
filesController.expandFilesInObject(config, anObject);
expect(anObject.aFile.url).toEqual("http://an.url");
// Test S3 with direct access
var s3DirectAccessAdapter = new S3Adapter(process.env.S3_ACCESS_KEY, process.env.S3_SECRET_KEY, 'parse.server.tests', {
directAccess: true
});
FCTestFactory.testAdapter("S3AdapterDirect", s3DirectAccessAdapter);
} else if (!process.env.TRAVIS) {
console.log("set S3_ACCESS_KEY and S3_SECRET_KEY to test S3Adapter")
}
if (process.env.GCP_PROJECT_ID && process.env.GCP_KEYFILE_PATH && process.env.GCS_BUCKET) {
// Test the GCS Adapter
var gcsAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET);
FCTestFactory.testAdapter("GCSAdapter", gcsAdapter);
// Test GCS with direct access
var gcsDirectAccessAdapter = new GCSAdapter(process.env.GCP_PROJECT_ID, process.env.GCP_KEYFILE_PATH, process.env.GCS_BUCKET, {
directAccess: true
});
FCTestFactory.testAdapter("GCSAdapterDirect", gcsDirectAccessAdapter);
} else if (!process.env.TRAVIS) {
console.log("set GCP_PROJECT_ID, GCP_KEYFILE_PATH, and GCS_BUCKET to test GCSAdapter")
}
try {
// Test the file system adapter
var fsAdapter = new FileSystemAdapter({
filesSubDirectory: 'sub1/sub2'
});
FCTestFactory.testAdapter("FileSystemAdapter", fsAdapter);
} catch (e) {
console.log("Give write access to the file system to test the FileSystemAdapter. Error: " + e);
}
done();
})
});

72
spec/FilesControllerTestFactory.js
View File

@@ -1,72 +0,0 @@
var FilesController = require('../src/Controllers/FilesController').FilesController;
var Config = require("../src/Config");
var testAdapter = function(name, adapter) {
// Small additional tests to improve overall coverage
var config = new Config(Parse.applicationId);
var filesController = new FilesController(adapter);
describe("FilesController with "+name,()=>{
it("should properly expand objects", (done) => {
var result = filesController.expandFilesInObject(config, function(){});
expect(result).toBeUndefined();
var fullFile = {
type: '__type',
url: "http://an.url"
}
var anObject = {
aFile: fullFile
}
filesController.expandFilesInObject(config, anObject);
expect(anObject.aFile.url).toEqual("http://an.url");
done();
})
it("should properly create, read, delete files", (done) => {
var filename;
filesController.createFile(config, "file.txt", "hello world").then( (result) => {
ok(result.url);
ok(result.name);
filename = result.name;
expect(result.name.match(/file.txt/)).not.toBe(null);
return filesController.getFileData(config, filename);
}, (err) => {
fail("The adapter should create the file");
console.error(err);
done();
}).then((result) => {
expect(result instanceof Buffer).toBe(true);
expect(result.toString('utf-8')).toEqual("hello world");
return filesController.deleteFile(config, filename);
}, (err) => {
fail("The adapter should get the file");
console.error(err);
done();
}).then((result) => {
filesController.getFileData(config, filename).then((res) => {
fail("the file should be deleted");
done();
}, (err) => {
done();
});
}, (err) => {
fail("The adapter should delete the file");
console.error(err);
done();
});
}, 5000); // longer tests
});
}
module.exports = {
testAdapter: testAdapter
}

120
src/Adapters/Files/FileSystemAdapter.js
View File

@@ -1,120 +0,0 @@
// FileSystemAdapter
//
// Stores files in local file system
// Requires write access to the server's file system.
import { FilesAdapter } from './FilesAdapter';
import colors from 'colors';
var fs = require('fs');
var path = require('path');
var pathSep = require('path').sep;
export class FileSystemAdapter extends FilesAdapter {
constructor({filesSubDirectory = ''} = {}) {
super();
this._filesDir = filesSubDirectory;
this._mkdir(this._getApplicationDir());
if (!this._applicationDirExist()) {
throw "Files directory doesn't exist.";
}
}
// For a given config object, filename, and data, store a file
// Returns a promise
createFile(config, filename, data) {
return new Promise((resolve, reject) => {
let filepath = this._getLocalFilePath(filename);
fs.writeFile(filepath, data, (err) => {
if(err !== null) {
return reject(err);
}
resolve(data);
});
});
}
deleteFile(config, filename) {
return new Promise((resolve, reject) => {
let filepath = this._getLocalFilePath(filename);
fs.readFile( filepath , function (err, data) {
if(err !== null) {
return reject(err);
}
fs.unlink(filepath, (unlinkErr) => {
if(err !== null) {
return reject(unlinkErr);
}
resolve(data);
});
});
});
}
getFileData(config, filename) {
return new Promise((resolve, reject) => {
let filepath = this._getLocalFilePath(filename);
fs.readFile( filepath , function (err, data) {
if(err !== null) {
return reject(err);
}
resolve(data);
});
});
}
getFileLocation(config, filename) {
return (config.mount + '/' + this._getLocalFilePath(filename));
}
/*
Helpers
--------------- */
_getApplicationDir() {
if (this._filesDir) {
return path.join('files', this._filesDir);
} else {
return 'files';
}
}
_applicationDirExist() {
return fs.existsSync(this._getApplicationDir());
}
_getLocalFilePath(filename) {
let applicationDir = this._getApplicationDir();
if (!fs.existsSync(applicationDir)) {
this._mkdir(applicationDir);
}
return path.join(applicationDir, encodeURIComponent(filename));
}
_mkdir(dirPath) {
// snippet found on -> https://gist.github.com/danherbert-epam/3960169
let dirs = dirPath.split(pathSep);
var root = "";
while (dirs.length > 0) {
var dir = dirs.shift();
if (dir === "") { // If directory starts with a /, the first path will be an empty string.
root = pathSep;
}
if (!fs.existsSync(path.join(root, dir))) {
try {
fs.mkdirSync(path.join(root, dir));
}
catch (e) {
if ( e.code == 'EACCES' ) {
throw new Error("PERMISSION ERROR: In order to use the FileSystemAdapter, write access to the server's file system is required.");
}
}
}
root = path.join(root, dir, pathSep);
}
}
}
export default FileSystemAdapter;

16
src/Adapters/Files/FilesAdapter.js
View File

@@ -13,22 +13,20 @@
export class FilesAdapter {
/* this method is responsible to store the file in order to be retrived later by it's file name
*
*
* @param config the current config
*
* @param filename the filename to save
* @param data the buffer of data from the file
* @param contentType the supposed contentType
* @discussion the contentType can be undefined if the controller was not able to determine it
*
* @discussion the contentType can be undefined if the controller was not able to determine it
*
* @return a promise that should fail if the storage didn't succeed
*
*
*/
createFile(config, filename: string, data, contentType: string) { }
createFile(filename: string, data, contentType: string) { }
deleteFile(config, filename) { }
deleteFile(filename) { }
getFileData(config, filename) { }
getFileData(filename) { }
getFileLocation(config, filename) { }
}

125
src/Adapters/Files/GCSAdapter.js
View File

@@ -1,125 +0,0 @@
// GCSAdapter
// Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage
import { storage } from 'gcloud';
import { FilesAdapter } from './FilesAdapter';
import requiredParameter from '../../requiredParameter';
function requiredOrFromEnvironment(env, name) {
let environmentVariable = process.env[env];
if (!environmentVariable) {
requiredParameter(`GCSAdapter requires an ${name}`);
}
return environmentVariable;
}
function fromEnvironmentOrDefault(env, defaultValue) {
let environmentVariable = process.env[env];
if (environmentVariable) {
return environmentVariable;
}
return defaultValue;
}
export class GCSAdapter extends FilesAdapter {
// GCS Project ID and the name of a corresponding Keyfile are required.
// Unlike the S3 adapter, you must create a new Cloud Storage bucket, as this is not created automatically.
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/master/guides/authentication
// for more details.
constructor(
projectId = requiredOrFromEnvironment('GCP_PROJECT_ID', 'projectId'),
keyFilename = requiredOrFromEnvironment('GCP_KEYFILE_PATH', 'keyfile path'),
bucket = requiredOrFromEnvironment('GCS_BUCKET', 'bucket name'),
{ bucketPrefix = fromEnvironmentOrDefault('GCS_BUCKET_PREFIX', ''),
directAccess = fromEnvironmentOrDefault('GCS_DIRECT_ACCESS', false) } = {}) {
super();
this._bucket = bucket;
this._bucketPrefix = bucketPrefix;
this._directAccess = directAccess;
let options = {
projectId: projectId,
keyFilename: keyFilename
};
this._gcsClient = new storage(options);
}
// For a given config object, filename, and data, store a file in GCS.
// Resolves the promise or fails with an error.
createFile(config, filename, data, contentType) {
let params = {
contentType: contentType || 'application/octet-stream'
};
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
// gcloud supports upload(file) not upload(bytes), so we need to stream.
var uploadStream = file.createWriteStream(params);
uploadStream.on('error', (err) => {
return reject(err);
}).on('finish', () => {
// Second call to set public read ACL after object is uploaded.
if (this._directAccess) {
file.makePublic((err, res) => {
if (err !== null) {
return reject(err);
}
resolve();
});
} else {
resolve();
}
});
uploadStream.write(data);
uploadStream.end();
});
}
// Deletes a file with the given file name.
// Returns a promise that succeeds with the delete response, or fails with an error.
deleteFile(config, filename) {
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
file.delete((err, res) => {
if(err !== null) {
return reject(err);
}
resolve(res);
});
});
}
// Search for and return a file if found by filename.
// Returns a promise that succeeds with the buffer result from GCS, or fails with an error.
getFileData(config, filename) {
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
// Check for existence, since gcloud-node seemed to be caching the result
file.exists((err, exists) => {
if (exists) {
file.download((err, data) => {
if (err !== null) {
return reject(err);
}
return resolve(data);
});
} else {
reject(err);
}
});
});
}
// Generates and returns the location of a file stored in GCS for the given request and filename.
// The location is the direct GCS link if the option is set,
// otherwise we serve the file through parse-server.
getFileLocation(config, filename) {
if (this._directAccess) {
return `https://${this._bucket}.storage.googleapis.com/${this._bucketPrefix + filename}`;
}
return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename));
}
}
export default GCSAdapter;

6
src/Adapters/Files/GridStoreAdapter.js
View File

@@ -28,7 +28,7 @@ export class GridStoreAdapter extends FilesAdapter {
// For a given config object, filename, and data, store a file
// Returns a promise
createFile(config, filename: string, data, contentType) {
createFile(filename: string, data, contentType) {
return this._connect().then(database => {
let gridStore = new GridStore(database, filename, 'w');
return gridStore.open();
@@ -39,7 +39,7 @@ export class GridStoreAdapter extends FilesAdapter {
});
}
deleteFile(config, filename: string) {
deleteFile(filename: string) {
return this._connect().then(database => {
let gridStore = new GridStore(database, filename, 'w');
return gridStore.open();
@@ -50,7 +50,7 @@ export class GridStoreAdapter extends FilesAdapter {
});
}
getFileData(config, filename: string) {
getFileData(filename: string) {
return this._connect().then(database => {
return GridStore.exist(database, filename)
.then(() => {

141
src/Adapters/Files/S3Adapter.js
View File

@@ -1,141 +0,0 @@
// S3Adapter
//
// Stores Parse files in AWS S3.
import * as AWS from 'aws-sdk';
import { FilesAdapter } from './FilesAdapter';
import requiredParameter from '../../requiredParameter';
const DEFAULT_S3_REGION = "us-east-1";
function requiredOrFromEnvironment(env, name) {
let environmentVariable = process.env[env];
if (!environmentVariable) {
requiredParameter(`S3Adapter requires an ${name}`);
}
return environmentVariable;
}
function fromEnvironmentOrDefault(env, defaultValue) {
let environmentVariable = process.env[env];
if (environmentVariable) {
return environmentVariable;
}
return defaultValue;
}
export class S3Adapter extends FilesAdapter {
// Creates an S3 session.
// Providing AWS access and secret keys is mandatory
// Region and bucket will use sane defaults if omitted
constructor(
accessKey = requiredOrFromEnvironment('S3_ACCESS_KEY', 'accessKey'),
secretKey = requiredOrFromEnvironment('S3_SECRET_KEY', 'secretKey'),
bucket = fromEnvironmentOrDefault('S3_BUCKET', undefined),
{ region = fromEnvironmentOrDefault('S3_REGION', DEFAULT_S3_REGION),
bucketPrefix = fromEnvironmentOrDefault('S3_BUCKET_PREFIX', ''),
directAccess = fromEnvironmentOrDefault('S3_DIRECT_ACCESS', false) } = {}) {
super();
this._region = region;
this._bucket = bucket;
this._bucketPrefix = bucketPrefix;
this._directAccess = directAccess;
let s3Options = {
accessKeyId: accessKey,
secretAccessKey: secretKey,
params: { Bucket: this._bucket }
};
AWS.config._region = this._region;
this._s3Client = new AWS.S3(s3Options);
this._hasBucket = false;
}
createBucket() {
var promise;
if (this._hasBucket) {
promise = Promise.resolve();
} else {
promise = new Promise((resolve, reject) => {
this._s3Client.createBucket(() => {
this._hasBucket = true;
resolve();
});
});
}
return promise;
}
// For a given config object, filename, and data, store a file in S3
// Returns a promise containing the S3 object creation response
createFile(config, filename, data, contentType) {
let params = {
Key: this._bucketPrefix + filename,
Body: data
};
if (this._directAccess) {
params.ACL = "public-read"
}
if (contentType) {
params.ContentType = contentType;
}
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
this._s3Client.upload(params, (err, data) => {
if (err !== null) {
return reject(err);
}
resolve(data);
});
});
});
}
deleteFile(config, filename) {
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
let params = {
Key: this._bucketPrefix + filename
};
this._s3Client.deleteObject(params, (err, data) =>{
if(err !== null) {
return reject(err);
}
resolve(data);
});
});
});
}
// Search for and return a file if found by filename
// Returns a promise that succeeds with the buffer result from S3
getFileData(config, filename) {
let params = {Key: this._bucketPrefix + filename};
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
this._s3Client.getObject(params, (err, data) => {
if (err !== null) {
return reject(err);
}
// Something happend here...
if (data && !data.Body) {
return reject(data);
}
resolve(data.Body);
});
});
});
}
// Generates and returns the location of a file stored in S3 for the given request and filename
// The location is the direct S3 link if the option is set, otherwise we serve the file through parse-server
getFileLocation(config, filename) {
if (this._directAccess) {
return `https://${this._bucket}.s3.amazonaws.com/${this._bucketPrefix + filename}`;
}
return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename));
}
}
export default S3Adapter;

6
src/Controllers/FilesController.js
View File

@@ -9,7 +9,7 @@ import mime from 'mime';
export class FilesController extends AdaptableController {
getFileData(config, filename) {
return this.adapter.getFileData(config, filename);
return this.adapter.getFileData(filename);
}
createFile(config, filename, data, contentType) {
@@ -27,7 +27,7 @@ export class FilesController extends AdaptableController {
filename = randomHexString(32) + '_' + filename;
var location = this.adapter.getFileLocation(config, filename);
return this.adapter.createFile(config, filename, data, contentType).then(() => {
return this.adapter.createFile(filename, data, contentType).then(() => {
return Promise.resolve({
url: location,
name: filename
@@ -36,7 +36,7 @@ export class FilesController extends AdaptableController {
}
deleteFile(config, filename) {
return this.adapter.deleteFile(config, filename);
return this.adapter.deleteFile(filename);
}
/**

3
src/ParseServer.js
View File

@@ -25,7 +25,6 @@ import { FileLoggerAdapter } from './Adapters/Logger/FileLoggerAdapter';
import { FilesController } from './Controllers/FilesController';
import { FilesRouter } from './Routers/FilesRouter';
import { FunctionsRouter } from './Routers/FunctionsRouter';
import { GCSAdapter } from './Adapters/Files/GCSAdapter';
import { GlobalConfigRouter } from './Routers/GlobalConfigRouter';
import { GridStoreAdapter } from './Adapters/Files/GridStoreAdapter';
import { HooksController } from './Controllers/HooksController';
@@ -42,13 +41,11 @@ import { PushController } from './Controllers/PushController';
import { PushRouter } from './Routers/PushRouter';
import { randomString } from './cryptoUtils';
import { RolesRouter } from './Routers/RolesRouter';
import { S3Adapter } from './Adapters/Files/S3Adapter';
import { SchemasRouter } from './Routers/SchemasRouter';
import { SessionsRouter } from './Routers/SessionsRouter';
import { setFeature } from './features';
import { UserController } from './Controllers/UserController';
import { UsersRouter } from './Routers/UsersRouter';
import { FileSystemAdapter } from './Adapters/Files/FileSystemAdapter';
// Mutate the Parse object to add the Cloud Code handlers
addParseCloud();

6
src/index.js
View File

@@ -1,7 +1,7 @@
import ParseServer from './ParseServer'
import { GCSAdapter } from './Adapters/Files/GCSAdapter';
import { S3Adapter } from './Adapters/Files/S3Adapter';
import { FileSystemAdapter } from './Adapters/Files/FileSystemAdapter';
import { GCSAdapter } from 'parse-server-gcs-adapter';
import { S3Adapter } from 'parse-server-s3-adapter';
import { FileSystemAdapter } from 'parse-server-fs-adapter';
// Factory function
let _ParseServer = function(options) {