Add fileKey rotation to GridFSBucketAdapter (#6768)

* add fileKey encryption to GridFSBucketStorageAdapter

* remove fileAdapter options from test spec

* ensure promise doesn't fall through in getFileData

* switch secretKey to fileKey

* add fileKey rotation for GridFSBucketAdapter

* improve catching decryption errors in testcases

* add testcase for rotating key from oldKey to noKey leaving all files decrypted

* removed fileKey from legacy test links. From the looks of the tests and the fileKey was appended to links. This key is now an encryption key

* clean up code

* make more consistant with FSAdapter

* use encryptionKey instead of fileKey

* Update ParseFile.spec.js

revert

spec/GridFSBucketStorageAdapter.spec.js

@@ -34,20 +34,365 @@ describe_only_db('mongo')('GridFSBucket and GridStore interop', () => {
     expect(gfsResult.toString('utf8')).toBe(originalString);
   });
 
-  it('an encypted file created in GridStore should be available in GridFS', async () => {
+  it('should save an encrypted file that can only be decrypted by a GridFS adapter with the encryptionKey', async () => {
-    const gsAdapter = new GridStoreAdapter(databaseURI);
+    const unencryptedAdapter = new GridFSBucketAdapter(databaseURI);
-    const gfsAdapter = new GridFSBucketAdapter(
+    const encryptedAdapter = new GridFSBucketAdapter(
       databaseURI,
       {},
       '89E4AFF1-DFE4-4603-9574-BFA16BB446FD'
     );
-    await expectMissingFile(gfsAdapter, 'myFileName');
+    await expectMissingFile(encryptedAdapter, 'myFileName');
     const originalString = 'abcdefghi';
-    await gfsAdapter.createFile('myFileName', originalString);
+    await encryptedAdapter.createFile('myFileName', originalString);
-    const gsResult = await gsAdapter.getFileData('myFileName');
+    const unencryptedResult = await unencryptedAdapter.getFileData(
-    expect(gsResult.toString('utf8')).not.toBe(originalString);
+      'myFileName'
-    const gfsResult = await gfsAdapter.getFileData('myFileName');
+    );
-    expect(gfsResult.toString('utf8')).toBe(originalString);
+    expect(unencryptedResult.toString('utf8')).not.toBe(originalString);
+    const encryptedResult = await encryptedAdapter.getFileData('myFileName');
+    expect(encryptedResult.toString('utf8')).toBe(originalString);
+  });
+
+  it('should rotate key of all unencrypted GridFS files to encrypted files', async () => {
+    const unencryptedAdapter = new GridFSBucketAdapter(databaseURI);
+    const encryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      '89E4AFF1-DFE4-4603-9574-BFA16BB446FD'
+    );
+    const fileName1 = 'file1.txt';
+    const data1 = 'hello world';
+    const fileName2 = 'file2.txt';
+    const data2 = 'hello new world';
+    //Store unecrypted files
+    await unencryptedAdapter.createFile(fileName1, data1);
+    const unencryptedResult1 = await unencryptedAdapter.getFileData(fileName1);
+    expect(unencryptedResult1.toString('utf8')).toBe(data1);
+    await unencryptedAdapter.createFile(fileName2, data2);
+    const unencryptedResult2 = await unencryptedAdapter.getFileData(fileName2);
+    expect(unencryptedResult2.toString('utf8')).toBe(data2);
+    //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter
+    const {
+      rotated,
+      notRotated,
+    } = await encryptedAdapter.rotateEncryptionKey();
+    expect(rotated.length).toEqual(2);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName1;
+      }).length
+    ).toEqual(1);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName2;
+      }).length
+    ).toEqual(1);
+    expect(notRotated.length).toEqual(0);
+    let result = await encryptedAdapter.getFileData(fileName1);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data1);
+    const encryptedData1 = await unencryptedAdapter.getFileData(fileName1);
+    expect(encryptedData1.toString('utf-8')).not.toEqual(unencryptedResult1);
+    result = await encryptedAdapter.getFileData(fileName2);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data2);
+    const encryptedData2 = await unencryptedAdapter.getFileData(fileName2);
+    expect(encryptedData2.toString('utf-8')).not.toEqual(unencryptedResult2);
+  });
+
+  it('should rotate key of all old encrypted GridFS files to encrypted files', async () => {
+    const oldEncryptionKey = 'oldKeyThatILoved';
+    const oldEncryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      oldEncryptionKey
+    );
+    const encryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      'newKeyThatILove'
+    );
+    const fileName1 = 'file1.txt';
+    const data1 = 'hello world';
+    const fileName2 = 'file2.txt';
+    const data2 = 'hello new world';
+    //Store unecrypted files
+    await oldEncryptedAdapter.createFile(fileName1, data1);
+    const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(
+      fileName1
+    );
+    expect(oldEncryptedResult1.toString('utf8')).toBe(data1);
+    await oldEncryptedAdapter.createFile(fileName2, data2);
+    const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(
+      fileName2
+    );
+    expect(oldEncryptedResult2.toString('utf8')).toBe(data2);
+    //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter
+    const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({
+      oldKey: oldEncryptionKey,
+    });
+    expect(rotated.length).toEqual(2);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName1;
+      }).length
+    ).toEqual(1);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName2;
+      }).length
+    ).toEqual(1);
+    expect(notRotated.length).toEqual(0);
+    let result = await encryptedAdapter.getFileData(fileName1);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data1);
+    let decryptionError1;
+    let encryptedData1;
+    try {
+      encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1);
+    } catch (err) {
+      decryptionError1 = err;
+    }
+    expect(decryptionError1).toMatch('Error');
+    expect(encryptedData1).toBeUndefined();
+    result = await encryptedAdapter.getFileData(fileName2);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data2);
+    let decryptionError2;
+    let encryptedData2;
+    try {
+      encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2);
+    } catch (err) {
+      decryptionError2 = err;
+    }
+    expect(decryptionError2).toMatch('Error');
+    expect(encryptedData2).toBeUndefined();
+  });
+
+  it('should rotate key of all old encrypted GridFS files to unencrypted files', async () => {
+    const oldEncryptionKey = 'oldKeyThatILoved';
+    const oldEncryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      oldEncryptionKey
+    );
+    const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI);
+    const fileName1 = 'file1.txt';
+    const data1 = 'hello world';
+    const fileName2 = 'file2.txt';
+    const data2 = 'hello new world';
+    //Store unecrypted files
+    await oldEncryptedAdapter.createFile(fileName1, data1);
+    const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(
+      fileName1
+    );
+    expect(oldEncryptedResult1.toString('utf8')).toBe(data1);
+    await oldEncryptedAdapter.createFile(fileName2, data2);
+    const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(
+      fileName2
+    );
+    expect(oldEncryptedResult2.toString('utf8')).toBe(data2);
+    //Check if unEncrypted adapter can read data and make sure it's not the same as oldEncrypted adapter
+    const {
+      rotated,
+      notRotated,
+    } = await unEncryptedAdapter.rotateEncryptionKey({
+      oldKey: oldEncryptionKey,
+    });
+    expect(rotated.length).toEqual(2);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName1;
+      }).length
+    ).toEqual(1);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName2;
+      }).length
+    ).toEqual(1);
+    expect(notRotated.length).toEqual(0);
+    let result = await unEncryptedAdapter.getFileData(fileName1);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data1);
+    let decryptionError1;
+    let encryptedData1;
+    try {
+      encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1);
+    } catch (err) {
+      decryptionError1 = err;
+    }
+    expect(decryptionError1).toMatch('Error');
+    expect(encryptedData1).toBeUndefined();
+    result = await unEncryptedAdapter.getFileData(fileName2);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data2);
+    let decryptionError2;
+    let encryptedData2;
+    try {
+      encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2);
+    } catch (err) {
+      decryptionError2 = err;
+    }
+    expect(decryptionError2).toMatch('Error');
+    expect(encryptedData2).toBeUndefined();
+  });
+
+  it('should only encrypt specified fileNames', async () => {
+    const oldEncryptionKey = 'oldKeyThatILoved';
+    const oldEncryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      oldEncryptionKey
+    );
+    const encryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      'newKeyThatILove'
+    );
+    const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI);
+    const fileName1 = 'file1.txt';
+    const data1 = 'hello world';
+    const fileName2 = 'file2.txt';
+    const data2 = 'hello new world';
+    //Store unecrypted files
+    await oldEncryptedAdapter.createFile(fileName1, data1);
+    const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(
+      fileName1
+    );
+    expect(oldEncryptedResult1.toString('utf8')).toBe(data1);
+    await oldEncryptedAdapter.createFile(fileName2, data2);
+    const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(
+      fileName2
+    );
+    expect(oldEncryptedResult2.toString('utf8')).toBe(data2);
+    //Inject unecrypted file to see if causes an issue
+    const fileName3 = 'file3.txt';
+    const data3 = 'hello past world';
+    await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8');
+    //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter
+    const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({
+      oldKey: oldEncryptionKey,
+      fileNames: [fileName1, fileName2],
+    });
+    expect(rotated.length).toEqual(2);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName1;
+      }).length
+    ).toEqual(1);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName2;
+      }).length
+    ).toEqual(1);
+    expect(notRotated.length).toEqual(0);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName3;
+      }).length
+    ).toEqual(0);
+    let result = await encryptedAdapter.getFileData(fileName1);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data1);
+    let decryptionError1;
+    let encryptedData1;
+    try {
+      encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1);
+    } catch (err) {
+      decryptionError1 = err;
+    }
+    expect(decryptionError1).toMatch('Error');
+    expect(encryptedData1).toBeUndefined();
+    result = await encryptedAdapter.getFileData(fileName2);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data2);
+    let decryptionError2;
+    let encryptedData2;
+    try {
+      encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2);
+    } catch (err) {
+      decryptionError2 = err;
+    }
+    expect(decryptionError2).toMatch('Error');
+    expect(encryptedData2).toBeUndefined();
+  });
+
+  it("should return fileNames of those it can't encrypt with the new key", async () => {
+    const oldEncryptionKey = 'oldKeyThatILoved';
+    const oldEncryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      oldEncryptionKey
+    );
+    const encryptedAdapter = new GridFSBucketAdapter(
+      databaseURI,
+      {},
+      'newKeyThatILove'
+    );
+    const unEncryptedAdapter = new GridFSBucketAdapter(databaseURI);
+    const fileName1 = 'file1.txt';
+    const data1 = 'hello world';
+    const fileName2 = 'file2.txt';
+    const data2 = 'hello new world';
+    //Store unecrypted files
+    await oldEncryptedAdapter.createFile(fileName1, data1);
+    const oldEncryptedResult1 = await oldEncryptedAdapter.getFileData(
+      fileName1
+    );
+    expect(oldEncryptedResult1.toString('utf8')).toBe(data1);
+    await oldEncryptedAdapter.createFile(fileName2, data2);
+    const oldEncryptedResult2 = await oldEncryptedAdapter.getFileData(
+      fileName2
+    );
+    expect(oldEncryptedResult2.toString('utf8')).toBe(data2);
+    //Inject unecrypted file to see if causes an issue
+    const fileName3 = 'file3.txt';
+    const data3 = 'hello past world';
+    await unEncryptedAdapter.createFile(fileName3, data3, 'text/utf8');
+    //Check if encrypted adapter can read data and make sure it's not the same as unEncrypted adapter
+    const { rotated, notRotated } = await encryptedAdapter.rotateEncryptionKey({
+      oldKey: oldEncryptionKey,
+    });
+    expect(rotated.length).toEqual(2);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName1;
+      }).length
+    ).toEqual(1);
+    expect(
+      rotated.filter(function (value) {
+        return value === fileName2;
+      }).length
+    ).toEqual(1);
+    expect(notRotated.length).toEqual(1);
+    expect(
+      notRotated.filter(function (value) {
+        return value === fileName3;
+      }).length
+    ).toEqual(1);
+    let result = await encryptedAdapter.getFileData(fileName1);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data1);
+    let decryptionError1;
+    let encryptedData1;
+    try {
+      encryptedData1 = await oldEncryptedAdapter.getFileData(fileName1);
+    } catch (err) {
+      decryptionError1 = err;
+    }
+    expect(decryptionError1).toMatch('Error');
+    expect(encryptedData1).toBeUndefined();
+    result = await encryptedAdapter.getFileData(fileName2);
+    expect(result instanceof Buffer).toBe(true);
+    expect(result.toString('utf-8')).toEqual(data2);
+    let decryptionError2;
+    let encryptedData2;
+    try {
+      encryptedData2 = await oldEncryptedAdapter.getFileData(fileName2);
+    } catch (err) {
+      decryptionError2 = err;
+    }
+    expect(decryptionError2).toMatch('Error');
+    expect(encryptedData2).toBeUndefined();
   });
 
   it('should save metadata', async () => {

src/Adapters/Files/GridFSBucketAdapter.js

@@ -21,16 +21,16 @@ export class GridFSBucketAdapter extends FilesAdapter {
   constructor(
     mongoDatabaseURI = defaults.DefaultMongoURI,
     mongoOptions = {},
-    fileKey = undefined
+    encryptionKey = undefined
   ) {
     super();
     this._databaseURI = mongoDatabaseURI;
     this._algorithm = 'aes-256-gcm';
-    this._fileKey =
+    this._encryptionKey =
-      fileKey !== undefined
+      encryptionKey !== undefined
         ? crypto
           .createHash('sha256')
-          .update(String(fileKey))
+          .update(String(encryptionKey))
           .digest('base64')
           .substr(0, 32)
         : null;
@@ -65,16 +65,26 @@ export class GridFSBucketAdapter extends FilesAdapter {
     const stream = await bucket.openUploadStream(filename, {
       metadata: options.metadata,
     });
-    if (this._fileKey !== null) {
+    if (this._encryptionKey !== null) {
-      const iv = crypto.randomBytes(16);
+      try {
-      const cipher = crypto.createCipheriv(this._algorithm, this._fileKey, iv);
+        const iv = crypto.randomBytes(16);
-      const encryptedResult = Buffer.concat([
+        const cipher = crypto.createCipheriv(
-        cipher.update(data),
+          this._algorithm,
-        cipher.final(),
+          this._encryptionKey,
-        iv,
+          iv
-        cipher.getAuthTag(),
+        );
-      ]);
+        const encryptedResult = Buffer.concat([
-      await stream.write(encryptedResult);
+          cipher.update(data),
+          cipher.final(),
+          iv,
+          cipher.getAuthTag(),
+        ]);
+        await stream.write(encryptedResult);
+      } catch (err) {
+        return new Promise((resolve, reject) => {
+          return reject(err);
+        });
+      }
     } else {
       await stream.write(data);
     }
@@ -109,21 +119,27 @@ export class GridFSBucketAdapter extends FilesAdapter {
       });
       stream.on('end', () => {
         const data = Buffer.concat(chunks);
-        if (this._fileKey !== null) {
+        if (this._encryptionKey !== null) {
-          const authTagLocation = data.length - 16;
+          try {
-          const ivLocation = data.length - 32;
+            const authTagLocation = data.length - 16;
-          const authTag = data.slice(authTagLocation);
+            const ivLocation = data.length - 32;
-          const iv = data.slice(ivLocation, authTagLocation);
+            const authTag = data.slice(authTagLocation);
-          const encrypted = data.slice(0, ivLocation);
+            const iv = data.slice(ivLocation, authTagLocation);
-          const decipher = crypto.createDecipheriv(
+            const encrypted = data.slice(0, ivLocation);
-            this._algorithm,
+            const decipher = crypto.createDecipheriv(
-            this._fileKey,
+              this._algorithm,
-            iv
+              this._encryptionKey,
-          );
+              iv
-          decipher.setAuthTag(authTag);
+            );
-          return resolve(
+            decipher.setAuthTag(authTag);
-            Buffer.concat([decipher.update(encrypted), decipher.final()])
+            const decrypted = Buffer.concat([
-          );
+              decipher.update(encrypted),
+              decipher.final(),
+            ]);
+            return resolve(decrypted);
+          } catch (err) {
+            return reject(err);
+          }
         }
         resolve(data);
       });
@@ -133,6 +149,79 @@ export class GridFSBucketAdapter extends FilesAdapter {
     });
   }
 
+  async rotateEncryptionKey(options = {}) {
+    var fileNames = [];
+    var oldKeyFileAdapter = {};
+    const bucket = await this._getBucket();
+    if (options.oldKey !== undefined) {
+      oldKeyFileAdapter = new GridFSBucketAdapter(
+        this._databaseURI,
+        this._mongoOptions,
+        options.oldKey
+      );
+    } else {
+      oldKeyFileAdapter = new GridFSBucketAdapter(
+        this._databaseURI,
+        this._mongoOptions
+      );
+    }
+    if (options.fileNames !== undefined) {
+      fileNames = options.fileNames;
+    } else {
+      const fileNamesIterator = await bucket.find().toArray();
+      fileNamesIterator.forEach(file => {
+        fileNames.push(file.filename);
+      });
+    }
+    return new Promise(resolve => {
+      var fileNamesNotRotated = fileNames;
+      var fileNamesRotated = [];
+      var fileNameTotal = fileNames.length;
+      var fileNameIndex = 0;
+      fileNames.forEach(fileName => {
+        oldKeyFileAdapter
+          .getFileData(fileName)
+          .then(plainTextData => {
+            //Overwrite file with data encrypted with new key
+            this.createFile(fileName, plainTextData)
+              .then(() => {
+                fileNamesRotated.push(fileName);
+                fileNamesNotRotated = fileNamesNotRotated.filter(function (
+                  value
+                ) {
+                  return value !== fileName;
+                });
+                fileNameIndex += 1;
+                if (fileNameIndex == fileNameTotal) {
+                  resolve({
+                    rotated: fileNamesRotated,
+                    notRotated: fileNamesNotRotated,
+                  });
+                }
+              })
+              .catch(() => {
+                fileNameIndex += 1;
+                if (fileNameIndex == fileNameTotal) {
+                  resolve({
+                    rotated: fileNamesRotated,
+                    notRotated: fileNamesNotRotated,
+                  });
+                }
+              });
+          })
+          .catch(() => {
+            fileNameIndex += 1;
+            if (fileNameIndex == fileNameTotal) {
+              resolve({
+                rotated: fileNamesRotated,
+                notRotated: fileNamesNotRotated,
+              });
+            }
+          });
+      });
+    });
+  }
+
   getFileLocation(config, filename) {
     return (
       config.mount +

src/Options/Definitions.js

@@ -149,6 +149,10 @@ module.exports.ParseServerOptions = {
     action: parsers.booleanParser,
     default: false,
   },
+  encryptionKey: {
+    env: 'PARSE_SERVER_ENCRYPTION_KEY',
+    help: 'Key for encrypting your files',
+  },
   expireInactiveSessions: {
     env: 'PARSE_SERVER_EXPIRE_INACTIVE_SESSIONS',
     help: 'Sets wether we should expire the inactive sessions, defaults to true',