From ddb0fb8a273717bfb2b15b60c5d3f77207558607 Mon Sep 17 00:00:00 2001 From: Florent Vilmart Date: Sat, 17 Sep 2016 16:52:02 -0400 Subject: [PATCH] Adds redis cache for distributed environments (#2691) * Makes schemaCache clearning promise-based * Adds redis cache adapter for distributed systems * Adds redis service to travis * allow pg to fail --- .travis.yml | 4 ++ spec/helper.js | 5 ++ src/Adapters/Cache/RedisCacheAdapter.js | 69 +++++++++++++++++++++++++ src/Controllers/SchemaController.js | 51 +++++++++++------- src/index.js | 3 +- 5 files changed, 112 insertions(+), 20 deletions(-) create mode 100644 src/Adapters/Cache/RedisCacheAdapter.js diff --git a/.travis.yml b/.travis.yml index d035051b..5ba46a5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ node_js: - '6.1' services: - postgresql + - redis-server addons: postgresql: '9.4' before_script: @@ -18,8 +19,11 @@ env: - MONGODB_VERSION=3.0.8 - MONGODB_VERSION=3.2.6 - PARSE_SERVER_TEST_DB=postgres + - PARSE_SERVER_TEST_CACHE=redis matrix: fast_finish: true + allow_failures: + - env: PARSE_SERVER_TEST_DB=postgres branches: only: - master diff --git a/spec/helper.js b/spec/helper.js index 9c3c7d7c..28fe2a2b 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -24,6 +24,7 @@ var MongoStorageAdapter = require('../src/Adapters/Storage/Mongo/MongoStorageAda const GridStoreAdapter = require('../src/Adapters/Files/GridStoreAdapter').GridStoreAdapter; const FSAdapter = require('parse-server-fs-adapter'); const PostgresStorageAdapter = require('../src/Adapters/Storage/Postgres/PostgresStorageAdapter'); +const RedisCacheAdapter = require('../src/Adapters/Cache/RedisCacheAdapter').default; const mongoURI = 'mongodb://localhost:27017/parseServerMongoAdapterTestDatabase'; const postgresURI = 'postgres://localhost:5432/parse_server_postgres_adapter_test_database'; @@ -101,6 +102,10 @@ var defaultConfiguration = { } }; +if (process.env.PARSE_SERVER_TEST_CACHE === 'redis') { + defaultConfiguration.cacheAdapter = new RedisCacheAdapter(); +} + let openConnections = {}; // Set up a default API server for testing with default configuration. diff --git a/src/Adapters/Cache/RedisCacheAdapter.js b/src/Adapters/Cache/RedisCacheAdapter.js new file mode 100644 index 00000000..65d2045f --- /dev/null +++ b/src/Adapters/Cache/RedisCacheAdapter.js @@ -0,0 +1,69 @@ +import redis from 'redis'; +import logger from '../../logger'; + +function debug() { + logger.debug.apply(logger, ['RedisCacheAdapter', ...arguments]); +} + +export class RedisCacheAdapter { + + constructor(ctx) { + this.client = redis.createClient(ctx); + this.p = Promise.resolve(); + } + + get(key) { + debug('get', key); + this.p = this.p.then(() => { + return new Promise((resolve, _) => { + this.client.get(key, function(err, res) { + debug('-> get', key, res); + if(!res) { + return resolve(null); + } + resolve(JSON.parse(res)); + }); + }); + }); + return this.p; + } + + put(key, value, ttl) { + value = JSON.stringify(value); + debug('put', key, value, ttl); + this.p = this.p.then(() => { + return new Promise((resolve, _) => { + this.client.set(key, value, function(err, res) { + resolve(); + }); + }); + }); + return this.p; + } + + del(key) { + debug('del', key); + this.p = this.p.then(() => { + return new Promise((resolve, _) => { + this.client.del(key, function(err, res) { + resolve(); + }); + }); + }); + return this.p; + } + + clear() { + debug('clear'); + this.p = this.p.then(() => { + return new Promise((resolve, _) => { + this.client.flushall(function(err, res) { + resolve(); + }); + }); + }); + return this.p; + } +} + +export default RedisCacheAdapter; diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index fd51ee9e..d84f45ed 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -323,15 +323,20 @@ export default class SchemaController { } reloadData(options = {clearCache: false}) { + let promise = Promise.resolve(); if (options.clearCache) { - this._cache.clear(); + promise = promise.then(() => { + return this._cache.clear(); + }); } if (this.reloadDataPromise && !options.clearCache) { return this.reloadDataPromise; } this.data = {}; this.perms = {}; - this.reloadDataPromise = this.getAllClasses(options) + this.reloadDataPromise = promise.then(() => { + return this.getAllClasses(options); + }) .then(allSchemas => { allSchemas.forEach(schema => { this.data[schema.className] = injectDefaultSchema(schema).fields; @@ -355,10 +360,13 @@ export default class SchemaController { } getAllClasses(options = {clearCache: false}) { + let promise = Promise.resolve(); if (options.clearCache) { - this._cache.clear(); + promise = this._cache.clear(); } - return this._cache.getAllClasses().then((allClasses) => { + return promise.then(() => { + return this._cache.getAllClasses() + }).then((allClasses) => { if (allClasses && allClasses.length && !options.clearCache) { return Promise.resolve(allClasses); } @@ -373,22 +381,25 @@ export default class SchemaController { } getOneSchema(className, allowVolatileClasses = false, options = {clearCache: false}) { + let promise = Promise.resolve(); if (options.clearCache) { - this._cache.clear(); + promise = this._cache.clear(); } - if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { - return Promise.resolve(this.data[className]); - } - return this._cache.getOneSchema(className).then((cached) => { - if (cached && !options.clearCache) { - return Promise.resolve(cached); + return promise.then(() => { + if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { + return Promise.resolve(this.data[className]); } - return this._dbAdapter.getClass(className) - .then(injectDefaultSchema) - .then((result) => { - return this._cache.setOneSchema(className, result).then(() => { - return result; - }) + return this._cache.getOneSchema(className).then((cached) => { + if (cached && !options.clearCache) { + return Promise.resolve(cached); + } + return this._dbAdapter.getClass(className) + .then(injectDefaultSchema) + .then((result) => { + return this._cache.setOneSchema(className, result).then(() => { + return result; + }) + }); }); }); } @@ -409,8 +420,9 @@ export default class SchemaController { return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, className })) .then(convertAdapterSchemaToParseSchema) .then((res) => { - this._cache.clear(); - return res; + return this._cache.clear().then(() => { + return Promise.resolve(res); + }); }) .catch(error => { if (error && error.code === Parse.Error.DUPLICATE_VALUE) { @@ -508,6 +520,7 @@ export default class SchemaController { } }) .catch(error => { + console.error(error); // The schema still doesn't validate. Give up throw new Parse.Error(Parse.Error.INVALID_JSON, 'schema class name does not revalidate'); }); diff --git a/src/index.js b/src/index.js index 39d2e95f..ec722db0 100644 --- a/src/index.js +++ b/src/index.js @@ -3,6 +3,7 @@ import S3Adapter from 'parse-server-s3-adapter' import FileSystemAdapter from 'parse-server-fs-adapter' import InMemoryCacheAdapter from './Adapters/Cache/InMemoryCacheAdapter' import NullCacheAdapter from './Adapters/Cache/NullCacheAdapter' +import RedisCacheAdapter from './Adapters/Cache/RedisCacheAdapter' import TestUtils from './TestUtils'; import { useExternal } from './deprecated'; import { getLogger } from './logger'; @@ -22,4 +23,4 @@ Object.defineProperty(module.exports, 'logger', { }); export default ParseServer; -export { S3Adapter, GCSAdapter, FileSystemAdapter, InMemoryCacheAdapter, NullCacheAdapter, TestUtils, _ParseServer as ParseServer }; +export { S3Adapter, GCSAdapter, FileSystemAdapter, InMemoryCacheAdapter, NullCacheAdapter, RedisCacheAdapter, TestUtils, _ParseServer as ParseServer };