joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support Metadata in GridFSAdapter (#6660)

Browse Source 
* Support Metadata in GridFSAdapter

* Useful for testing in the JS SDK
* Adds new endpoint to be used with `Parse.File.getData`
* Allows file adapters to return tags as well as future data.

* fix tests

* Make getMetadata optional

* Revert "fix tests"

This reverts commit 7706da13c688027483974e854b5b24321fb070cd.

* improve coverage
This commit is contained in:
Diamond Lewis
2020-05-08 15:32:20 -05:00
committed by GitHub
parent c32ff20f4f
commit 370215a39b
6 changed files with 162 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
spec/FilesController.spec.js
View File

@@ -8,6 +8,7 @@ const GridStoreAdapter = require('../lib/Adapters/Files/GridStoreAdapter')
.GridStoreAdapter;
const Config = require('../lib/Config');
const FilesController = require('../lib/Controllers/FilesController').default;
const databaseURI = 'mongodb://localhost:27017/parse';
const mockAdapter = {
createFile: () => {
@@ -23,13 +24,13 @@ const mockAdapter = {
// Small additional tests to improve overall coverage
describe('FilesController', () => {
it('should properly expand objects', done => {
it('should properly expand objects', (done) => {
const config = Config.get(Parse.applicationId);
const gridStoreAdapter = new GridFSBucketAdapter(
'mongodb://localhost:27017/parse'
);
const filesController = new FilesController(gridStoreAdapter);
const result = filesController.expandFilesInObject(config, function() {});
const result = filesController.expandFilesInObject(config, function () {});
expect(result).toBeUndefined();
@@ -47,7 +48,7 @@ describe('FilesController', () => {
done();
});
it('should create a server log on failure', done => {
it('should create a server log on failure', (done) => {
const logController = new LoggerController(new WinstonLoggerAdapter());
reconfigureServer({ filesAdapter: mockAdapter })
@@ -56,22 +57,20 @@ describe('FilesController', () => {
() => done.fail('should not succeed'),
() => setImmediate(() => Promise.resolve('done'))
)
.then(() => new Promise(resolve => setTimeout(resolve, 200)))
.then(() => new Promise((resolve) => setTimeout(resolve, 200)))
.then(() =>
logController.getLogs({ from: Date.now() - 1000, size: 1000 })
)
.then(logs => {
.then((logs) => {
// we get two logs here: 1. the source of the failure to save the file
// and 2 the message that will be sent back to the client.
const log1 = logs.find(
x => x.message === 'Error creating a file: it failed with xyz'
(x) => x.message === 'Error creating a file: it failed with xyz'
);
expect(log1.level).toBe('error');
const log2 = logs.find(
x => x.message === 'it failed with xyz'
);
const log2 = logs.find((x) => x.message === 'it failed with xyz');
expect(log2.level).toBe('error');
expect(log2.code).toBe(130);
@@ -79,7 +78,7 @@ describe('FilesController', () => {
});
});
it('should create a parse error when a string is returned', done => {
it('should create a parse error when a string is returned', (done) => {
const mock2 = mockAdapter;
mock2.validateFilename = () => {
return 'Bad file! No biscuit!';
@@ -92,7 +91,7 @@ describe('FilesController', () => {
done();
});
it('should add a unique hash to the file name when the preserveFileName option is false', done => {
it('should add a unique hash to the file name when the preserveFileName option is false', (done) => {
const config = Config.get(Parse.applicationId);
const gridStoreAdapter = new GridFSBucketAdapter(
'mongodb://localhost:27017/parse'
@@ -115,7 +114,7 @@ describe('FilesController', () => {
done();
});
it('should not add a unique hash to the file name when the preserveFileName option is true', done => {
it('should not add a unique hash to the file name when the preserveFileName option is true', (done) => {
const config = Config.get(Parse.applicationId);
const gridStoreAdapter = new GridFSBucketAdapter(
'mongodb://localhost:27017/parse'
@@ -137,7 +136,16 @@ describe('FilesController', () => {
done();
});
it('should reject slashes in file names', done => {
it('should handle adapter without getMetadata', async () => {
const gridStoreAdapter = new GridFSBucketAdapter(databaseURI);
gridStoreAdapter.getMetadata = null;
const filesController = new FilesController(gridStoreAdapter);
const result = await filesController.getMetadata();
expect(result).toEqual({});
});
it('should reject slashes in file names', (done) => {
const gridStoreAdapter = new GridFSBucketAdapter(
'mongodb://localhost:27017/parse'
);
@@ -146,7 +154,7 @@ describe('FilesController', () => {
done();
});
it('should also reject slashes in file names', done => {
it('should also reject slashes in file names', (done) => {
const gridStoreAdapter = new GridStoreAdapter(
'mongodb://localhost:27017/parse'
);

69
spec/GridFSBucketStorageAdapter.spec.js
View File

@@ -4,6 +4,8 @@ const GridFSBucketAdapter = require('../lib/Adapters/Files/GridFSBucketAdapter')
.GridFSBucketAdapter;
const { randomString } = require('../lib/cryptoUtils');
const databaseURI = 'mongodb://localhost:27017/parse';
const request = require('../lib/request');
const Config = require('../lib/Config');
async function expectMissingFile(gfsAdapter, name) {
try {
@@ -33,6 +35,73 @@ describe_only_db('mongo')('GridFSBucket and GridStore interop', () => {
expect(gfsResult.toString('utf8')).toBe(originalString);
});
it('should save metadata', async () => {
const gfsAdapter = new GridFSBucketAdapter(databaseURI);
const originalString = 'abcdefghi';
const metadata = { hello: 'world' };
await gfsAdapter.createFile('myFileName', originalString, null, {
metadata,
});
const gfsResult = await gfsAdapter.getFileData('myFileName');
expect(gfsResult.toString('utf8')).toBe(originalString);
let gfsMetadata = await gfsAdapter.getMetadata('myFileName');
expect(gfsMetadata.metadata).toEqual(metadata);
// Empty json for file not found
gfsMetadata = await gfsAdapter.getMetadata('myUnknownFile');
expect(gfsMetadata).toEqual({});
});
it('should save metadata with file', async () => {
const gfsAdapter = new GridFSBucketAdapter(databaseURI);
await reconfigureServer({ filesAdapter: gfsAdapter });
const str = 'Hello World!';
const data = [];
for (let i = 0; i < str.length; i++) {
data.push(str.charCodeAt(i));
}
const metadata = { foo: 'bar' };
const file = new Parse.File('hello.txt', data, 'text/plain');
file.addMetadata('foo', 'bar');
await file.save();
let fileData = await gfsAdapter.getMetadata(file.name());
expect(fileData.metadata).toEqual(metadata);
// Can only add metadata on create
file.addMetadata('hello', 'world');
await file.save();
fileData = await gfsAdapter.getMetadata(file.name());
expect(fileData.metadata).toEqual(metadata);
const headers = {
'X-Parse-Application-Id': 'test',
'X-Parse-REST-API-Key': 'rest',
};
const response = await request({
method: 'GET',
headers,
url: `http://localhost:8378/1/files/test/metadata/${file.name()}`,
});
fileData = response.data;
expect(fileData.metadata).toEqual(metadata);
});
it('should handle getMetadata error', async () => {
const config = Config.get('test');
config.filesController.getMetadata = () => Promise.reject();
const headers = {
'X-Parse-Application-Id': 'test',
'X-Parse-REST-API-Key': 'rest',
};
const response = await request({
method: 'GET',
headers,
url: `http://localhost:8378/1/files/test/metadata/filename.txt`,
});
expect(response.data).toEqual({});
});
it('properly fetches a large file from GridFS', async () => {
const gfsAdapter = new GridFSBucketAdapter(databaseURI);
const twoMegabytesFile = randomString(2048 * 1024);

8
src/Adapters/Files/FilesAdapter.js
View File

@@ -88,6 +88,14 @@ export class FilesAdapter {
* @returns {Promise} Data for byte range
*/
// handleFileStream(filename: string, res: any, req: any, contentType: string): Promise
/** Responsible for retrieving metadata and tags
*
* @param {string} filename - the filename to retrieve metadata
*
* @return {Promise} a promise that should pass with metadata
*/
// getMetadata(filename: string): Promise<any> {}
}
/**

28
src/Adapters/Files/GridFSBucketAdapter.js
View File

@@ -32,7 +32,7 @@ export class GridFSBucketAdapter extends FilesAdapter {
this._connectionPromise = MongoClient.connect(
this._databaseURI,
this._mongoOptions
).then(client => {
).then((client) => {
this._client = client;
return client.db(client.s.options.dbName);
});
@@ -41,14 +41,16 @@ export class GridFSBucketAdapter extends FilesAdapter {
}
_getBucket() {
return this._connect().then(database => new GridFSBucket(database));
return this._connect().then((database) => new GridFSBucket(database));
}
// For a given config object, filename, and data, store a file
// Returns a promise
async createFile(filename: string, data) {
async createFile(filename: string, data, contentType, options = {}) {
const bucket = await this._getBucket();
const stream = await bucket.openUploadStream(filename);
const stream = await bucket.openUploadStream(filename, {
metadata: options.metadata,
});
await stream.write(data);
stream.end();
return new Promise((resolve, reject) => {
@@ -64,7 +66,7 @@ export class GridFSBucketAdapter extends FilesAdapter {
throw new Error('FileNotFound');
}
return Promise.all(
documents.map(doc => {
documents.map((doc) => {
return bucket.delete(doc._id);
})
);
@@ -76,13 +78,13 @@ export class GridFSBucketAdapter extends FilesAdapter {
stream.read();
return new Promise((resolve, reject) => {
const chunks = [];
stream.on('data', data => {
stream.on('data', (data) => {
chunks.push(data);
});
stream.on('end', () => {
resolve(Buffer.concat(chunks));
});
stream.on('error', err => {
stream.on('error', (err) => {
reject(err);
});
});
@@ -98,6 +100,16 @@ export class GridFSBucketAdapter extends FilesAdapter {
);
}
async getMetadata(filename) {
const bucket = await this._getBucket();
const files = await bucket.find({ filename }).toArray();
if (files.length === 0) {
return {};
}
const { metadata } = files[0];
return { metadata };
}
async handleFileStream(filename: string, req, res, contentType) {
const bucket = await this._getBucket();
const files = await bucket.find({ filename }).toArray();
@@ -122,7 +134,7 @@ export class GridFSBucketAdapter extends FilesAdapter {
});
const stream = bucket.openDownloadStreamByName(filename);
stream.start(start);
stream.on('data', chunk => {
stream.on('data', (chunk) => {
res.write(chunk);
});
stream.on('error', () => {

9
src/Controllers/FilesController.js
View File

@@ -45,6 +45,13 @@ export class FilesController extends AdaptableController {
return this.adapter.deleteFile(filename);
}
getMetadata(filename) {
if (typeof this.adapter.getMetadata === 'function') {
return this.adapter.getMetadata(filename);
}
return Promise.resolve({});
}
/**
* Find file references in REST-format object and adds the url key
* with the current mount point and app id.
@@ -52,7 +59,7 @@ export class FilesController extends AdaptableController {
*/
expandFilesInObject(config, object) {
if (object instanceof Array) {
object.map(obj => this.expandFilesInObject(config, obj));
object.map((obj) => this.expandFilesInObject(config, obj));
return;
}
if (typeof object !== 'object') {

62
src/Routers/FilesRouter.js
View File

@@ -10,14 +10,16 @@ const http = require('http');
const downloadFileFromURI = (uri) => {
return new Promise((res, rej) => {
http.get(uri, (response) => {
response.setDefaultEncoding('base64');
let body = `data:${response.headers['content-type']};base64,`;
response.on('data', data => body += data);
response.on('end', () => res(body));
}).on('error', (e) => {
rej(`Error downloading file from ${uri}: ${e.message}`);
});
http
.get(uri, (response) => {
response.setDefaultEncoding('base64');
let body = `data:${response.headers['content-type']};base64,`;
response.on('data', (data) => (body += data));
response.on('end', () => res(body));
})
.on('error', (e) => {
rej(`Error downloading file from ${uri}: ${e.message}`);
});
});
};
@@ -38,14 +40,15 @@ const errorMessageFromError = (e) => {
return e.message;
}
return undefined;
}
};
export class FilesRouter {
expressRouter({ maxUploadSize = '20Mb' } = {}) {
var router = express.Router();
router.get('/files/:appId/:filename', this.getHandler);
router.get('/files/:appId/metadata/:filename', this.metadataHandler);
router.post('/files', function(req, res, next) {
router.post('/files', function (req, res, next) {
next(
new Parse.Error(Parse.Error.INVALID_FILE_NAME, 'Filename not provided.')
);
@@ -88,7 +91,7 @@ export class FilesRouter {
} else {
filesController
.getFileData(config, filename)
.then(data => {
.then((data) => {
res.status(200);
res.set('Content-Type', contentType);
res.set('Content-Length', data.length);
@@ -135,7 +138,7 @@ export class FilesRouter {
fileObject,
config,
req.auth
)
);
let saveResult;
// if a new ParseFile is returned check if it's an already saved file
if (triggerResult instanceof Parse.File) {
@@ -187,16 +190,12 @@ export class FilesRouter {
res.status(201);
res.set('Location', saveResult.url);
res.json(saveResult);
} catch (e) {
logger.error('Error creating a file: ', e);
const errorMessage = errorMessageFromError(e) || `Could not store file: ${fileObject.file._name}.`;
next(
new Parse.Error(
Parse.Error.FILE_SAVE_ERROR,
errorMessage
)
);
const errorMessage =
errorMessageFromError(e) ||
`Could not store file: ${fileObject.file._name}.`;
next(new Parse.Error(Parse.Error.FILE_SAVE_ERROR, errorMessage));
}
}
@@ -207,7 +206,7 @@ export class FilesRouter {
// run beforeDeleteFile trigger
const file = new Parse.File(filename);
file._url = filesController.adapter.getFileLocation(req.config, filename);
const fileObject = { file, fileSize: null }
const fileObject = { file, fileSize: null };
await triggers.maybeRunFileTrigger(
triggers.Types.beforeDeleteFile,
fileObject,
@@ -229,12 +228,21 @@ export class FilesRouter {
} catch (e) {
logger.error('Error deleting a file: ', e);
const errorMessage = errorMessageFromError(e) || `Could not delete file.`;
next(
new Parse.Error(
Parse.Error.FILE_DELETE_ERROR,
errorMessage
)
);
next(new Parse.Error(Parse.Error.FILE_DELETE_ERROR, errorMessage));
}
}
async metadataHandler(req, res) {
const config = Config.get(req.params.appId);
const { filesController } = config;
const { filename } = req.params;
try {
const data = await filesController.getMetadata(filename);
res.status(200);
res.json(data);
} catch (e) {
res.status(200);
res.json({});
}
}
}