joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/ParsePlatform/parse-server into mcdonald-gcs-adapter

Browse Source 
Get GCSAdapter up to snuff with FilesController + FilesControllerTestFactory

* 'master' of https://github.com/ParsePlatform/parse-server: (102 commits)
  Remove duplicated instructions
  Release and Changelog for 2.1.4
  fixes missing coverage with sh script
  Fix update system schema
  Adds optional COVERAGE
  Allows to pass no where in $select clause
  Sanitize objectId in
  Fix delete schema when actual collection does not exist
  Fix replace query overwrite the existing query object.
  Fix create system class with relation/pointer
  Use throws syntax for errors in SchemasRouter.
  Completely migrate SchemasRouter to new MongoCollection API.
  Add tests that verify installationId in Cloud Code triggers.
  Propagate installationId in all Cloud Code triggers.
  Add test
  expiresAt should be a Date, not a string. Fixes #776
  Fix missing 'let/var' in OneSignalPushAdapter.spec.
  Don't run any afterSave hooks if none are registered.
  Fix : remove query count limit
  Flatten custom operations in request.object in afterSave hooks.
  ...
This commit is contained in:
Mike McDonald
2016-03-03 22:36:25 -08:00
parent 3e41a21e27 e3d26e2b4e
commit 1dc346715d
83 changed files with 4170 additions and 1458 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/Adapters/Files/FilesAdapter.js
View File

@@ -8,11 +8,23 @@
// * getFileLocation(config, request, filename)
//
// Default is GridStoreAdapter, which requires mongo
// and for the API server to be using the ExportAdapter
// and for the API server to be using the DatabaseController with Mongo
// database adapter.
export class FilesAdapter {
createFile(config, filename, data) { }
/* this method is responsible to store the file in order to be retrived later by it's file name
*
*
* @param config the current config
* @param filename the filename to save
* @param data the buffer of data from the file
* @param contentType the supposed contentType
* @discussion the contentType can be undefined if the controller was not able to determine it
*
* @return a promise that should fail if the storage didn't succeed
*
*/
createFile(config, filename: string, data, contentType: string) { }
deleteFile(config, filename) { }

39
src/Adapters/Files/GCSAdapter.js
View File

@@ -1,16 +1,18 @@
// GCSAdapter
// Store Parse Files in Google Cloud Storage: https://cloud.google.com/storage
import * as gcloud from 'gcloud';
import { storage } from 'gcloud';
import { FilesAdapter } from './FilesAdapter';
import requiredParameter from '../../requiredParameter';
export class GCSAdapter extends FilesAdapter {
// GCS Project ID and the name of a corresponding Keyfile are required.
// Unlike the S3 adapter, you must create a new Cloud Storage bucket, as this is not created automatically.
// See https://googlecloudplatform.github.io/gcloud-node/#/docs/master/guides/authentication
// for more details.
constructor(
projectId,
keyFilename,
bucket,
projectId = requiredParameter('GCSAdapter requires a GCP Project ID'),
keyFilename = requiredParameter('GCSAdapter requires a GCP keyfile'),
bucket = requiredParameter('GCSAdapter requires a GCS bucket name'),
{ bucketPrefix = '',
directAccess = false } = {}
) {
@@ -20,21 +22,25 @@ export class GCSAdapter extends FilesAdapter {
this._bucketPrefix = bucketPrefix;
this._directAccess = directAccess;
let gcsOptions = {
let options = {
projectId: projectId,
keyFilename: keyFilename
};
this._gcsClient = new gcloud.storage(gcsOptions);
this._gcsClient = new storage(options);
}
// For a given config object, filename, and data, store a file in GCS.
// Resolves the promise or fails with an error.
createFile(config, filename, data) {
createFile(config, filename, data, contentType) {
let params = {
contentType: contentType || 'application/octet-stream'
};
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
// gcloud supports upload(file) not upload(bytes), so we need to stream.
var uploadStream = file.createWriteStream();
var uploadStream = file.createWriteStream(params);
uploadStream.on('error', (err) => {
return reject(err);
}).on('finish', () => {
@@ -61,6 +67,7 @@ export class GCSAdapter extends FilesAdapter {
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
file.delete((err, res) => {
console.log("delete: ", filename, err, res);
if(err !== null) {
return reject(err);
}
@@ -74,11 +81,19 @@ export class GCSAdapter extends FilesAdapter {
getFileData(config, filename) {
return new Promise((resolve, reject) => {
let file = this._gcsClient.bucket(this._bucket).file(this._bucketPrefix + filename);
file.download((err, data) => {
if (err !== null) {
return reject(err);
// Check for existence, since gcloud-node seemed to be caching the result
file.exists((err, exists) => {
if (exists) {
file.download((err, data) => {
console.log("get: ", filename, err, data);
if (err !== null) {
return reject(err);
}
return resolve(data);
});
} else {
reject(err);
}
resolve(data);
});
});
}

60
src/Adapters/Files/GridStoreAdapter.js
View File

@@ -1,28 +1,47 @@
// GridStoreAdapter
//
// Stores files in Mongo using GridStore
// Requires the database adapter to be based on mongoclient
/**
GridStoreAdapter
Stores files in Mongo using GridStore
Requires the database adapter to be based on mongoclient
import { GridStore } from 'mongodb';
@flow weak
*/
import { MongoClient, GridStore, Db} from 'mongodb';
import { FilesAdapter } from './FilesAdapter';
export class GridStoreAdapter extends FilesAdapter {
_databaseURI: string;
_connectionPromise: Promise<Db>;
constructor(mongoDatabaseURI: string) {
super();
this._databaseURI = mongoDatabaseURI;
this._connect();
}
_connect() {
if (!this._connectionPromise) {
this._connectionPromise = MongoClient.connect(this._databaseURI);
}
return this._connectionPromise;
}
// For a given config object, filename, and data, store a file
// Returns a promise
createFile(config, filename, data) {
return config.database.connect().then(() => {
let gridStore = new GridStore(config.database.db, filename, 'w');
createFile(config, filename: string, data, contentType) {
return this._connect().then(database => {
let gridStore = new GridStore(database, filename, 'w');
return gridStore.open();
}).then((gridStore) => {
}).then(gridStore => {
return gridStore.write(data);
}).then((gridStore) => {
}).then(gridStore => {
return gridStore.close();
});
}
deleteFile(config, filename) {
return config.database.connect().then(() => {
let gridStore = new GridStore(config.database.db, filename, 'w');
deleteFile(config, filename: string) {
return this._connect().then(database => {
let gridStore = new GridStore(database, filename, 'w');
return gridStore.open();
}).then((gridStore) => {
return gridStore.unlink();
@@ -31,13 +50,14 @@ export class GridStoreAdapter extends FilesAdapter {
});
}
getFileData(config, filename) {
return config.database.connect().then(() => {
return GridStore.exist(config.database.db, filename);
}).then(() => {
let gridStore = new GridStore(config.database.db, filename, 'r');
return gridStore.open();
}).then((gridStore) => {
getFileData(config, filename: string) {
return this._connect().then(database => {
return GridStore.exist(database, filename)
.then(() => {
let gridStore = new GridStore(database, filename, 'r');
return gridStore.open();
});
}).then(gridStore => {
return gridStore.read();
});
}

104
src/Adapters/Files/S3Adapter.js
View File

@@ -4,23 +4,38 @@
import * as AWS from 'aws-sdk';
import { FilesAdapter } from './FilesAdapter';
import requiredParameter from '../../requiredParameter';
const DEFAULT_S3_REGION = "us-east-1";
function parseS3AdapterOptions(...options) {
if (options.length === 1 && typeof options[0] == "object") {
return options;
}
const additionalOptions = options[3] || {};
return {
accessKey: options[0],
secretKey: options[1],
bucket: options[2],
region: additionalOptions.region
}
}
export class S3Adapter extends FilesAdapter {
// Creates an S3 session.
// Providing AWS access and secret keys is mandatory
// Region and bucket will use sane defaults if omitted
constructor(
accessKey,
secretKey,
bucket,
{ region = DEFAULT_S3_REGION,
bucketPrefix = '',
directAccess = false } = {}
) {
accessKey = requiredParameter('S3Adapter requires an accessKey'),
secretKey = requiredParameter('S3Adapter requires a secretKey'),
bucket,
{ region = DEFAULT_S3_REGION,
bucketPrefix = '',
directAccess = false } = {}) {
super();
this._region = region;
this._bucket = bucket;
this._bucketPrefix = bucketPrefix;
@@ -33,11 +48,27 @@ export class S3Adapter extends FilesAdapter {
};
AWS.config._region = this._region;
this._s3Client = new AWS.S3(s3Options);
this._hasBucket = false;
}
createBucket() {
var promise;
if (this._hasBucket) {
promise = Promise.resolve();
} else {
promise = new Promise((resolve, reject) => {
this._s3Client.createBucket(() => {
this._hasBucket = true;
resolve();
});
});
}
return promise;
}
// For a given config object, filename, and data, store a file in S3
// Returns a promise containing the S3 object creation response
createFile(config, filename, data) {
createFile(config, filename, data, contentType) {
let params = {
Key: this._bucketPrefix + filename,
Body: data
@@ -45,26 +76,33 @@ export class S3Adapter extends FilesAdapter {
if (this._directAccess) {
params.ACL = "public-read"
}
return new Promise((resolve, reject) => {
this._s3Client.upload(params, (err, data) => {
if (err !== null) {
return reject(err);
}
resolve(data);
if (contentType) {
params.ContentType = contentType;
}
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
this._s3Client.upload(params, (err, data) => {
if (err !== null) {
return reject(err);
}
resolve(data);
});
});
});
}
deleteFile(config, filename) {
return new Promise((resolve, reject) => {
let params = {
Key: this._bucketPrefix + filename
};
this._s3Client.deleteObject(params, (err, data) =>{
if(err !== null) {
return reject(err);
}
resolve(data);
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
let params = {
Key: this._bucketPrefix + filename
};
this._s3Client.deleteObject(params, (err, data) =>{
if(err !== null) {
return reject(err);
}
resolve(data);
});
});
});
}
@@ -73,12 +111,18 @@ export class S3Adapter extends FilesAdapter {
// Returns a promise that succeeds with the buffer result from S3
getFileData(config, filename) {
let params = {Key: this._bucketPrefix + filename};
return new Promise((resolve, reject) => {
this._s3Client.getObject(params, (err, data) => {
if (err !== null) {
return reject(err);
}
resolve(data.Body);
return this.createBucket().then(() => {
return new Promise((resolve, reject) => {
this._s3Client.getObject(params, (err, data) => {
if (err !== null) {
return reject(err);
}
// Something happend here...
if (data && !data.Body) {
return reject(data);
}
resolve(data.Body);
});
});
});
}