joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve single schema cache (#7214)

Browse Source 
* Initial Commit

* fix flaky test

* temporary set ci timeout

* turn off ci check

* fix postgres tests

* fix tests

* node flaky test

* remove improvements

* Update SchemaPerformance.spec.js

* fix tests

* revert ci

* Create Singleton Object

* properly clear cache testing

* Cleanup

* remove fit

* try PushController.spec

* try push test rewrite

* try push enqueue time

* Increase test timeout

* remove pg server creation test

* xit push tests

* more xit

* remove skipped tests

* Fix conflicts

* reduce ci timeout

* fix push tests

* Revert "fix push tests"

This reverts commit 05aba62f1cbbca7d5d3e80b9444529f59407cb56.

* improve initialization

* fix flaky tests

* xit flaky test

* Update CHANGELOG.md

* enable debug logs

* Update LogsRouter.spec.js

* create initial indexes in series

* lint

* horizontal scaling documentation

* Update Changelog

* change horizontalScaling db option

* Add enableSchemaHooks option

* move enableSchemaHooks to databaseOptions
This commit is contained in:
Diamond Lewis
2021-03-16 16:05:36 -05:00
committed by GitHub
parent 32fc45d2d2
commit a02014f557
38 changed files with 673 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
src/Controllers/DatabaseController.js
View File

@@ -13,6 +13,9 @@ import deepcopy from 'deepcopy';
import logger from '../logger';
import * as SchemaController from './SchemaController';
import { StorageAdapter } from '../Adapters/Storage/StorageAdapter';
import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
import SchemaCache from '../Adapters/Cache/SchemaCache';
import type { LoadSchemaOptions } from './types';
import type { QueryOptions, FullQueryOptions } from '../Adapters/Storage/StorageAdapter';
function addWriteACL(query, acl) {
@@ -230,9 +233,6 @@ const filterSensitiveData = (
return object;
};
import type { LoadSchemaOptions } from './types';
import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
// Runs an update on the database.
// Returns a promise for an object with the new values for field
// modifications that don't know their results ahead of time, like
@@ -398,9 +398,8 @@ class DatabaseController {
schemaPromise: ?Promise<SchemaController.SchemaController>;
_transactionalSession: ?any;
constructor(adapter: StorageAdapter, schemaCache: any) {
constructor(adapter: StorageAdapter) {
this.adapter = adapter;
this.schemaCache = schemaCache;
// We don't want a mutable this.schema, because then you could have
// one request that uses different schemas for different parts of
// it. Instead, use loadSchema to get a schema.
@@ -434,7 +433,7 @@ class DatabaseController {
if (this.schemaPromise != null) {
return this.schemaPromise;
}
this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options);
this.schemaPromise = SchemaController.load(this.adapter, options);
this.schemaPromise.then(
() => delete this.schemaPromise,
() => delete this.schemaPromise
@@ -916,7 +915,8 @@ class DatabaseController {
*/
deleteEverything(fast: boolean = false): Promise<any> {
this.schemaPromise = null;
return Promise.all([this.adapter.deleteAllClasses(fast), this.schemaCache.clear()]);
SchemaCache.clear();
return this.adapter.deleteAllClasses(fast);
}
// Returns a promise for a list of related ids given an owning id.
@@ -1325,8 +1325,12 @@ class DatabaseController {
}
deleteSchema(className: string): Promise<void> {
let schemaController;
return this.loadSchema({ clearCache: true })
.then(schemaController => schemaController.getOneSchema(className, true))
.then(s => {
schemaController = s;
return schemaController.getOneSchema(className, true);
})
.catch(error => {
if (error === undefined) {
return { fields: {} };
@@ -1356,7 +1360,8 @@ class DatabaseController {
this.adapter.deleteClass(joinTableName(className, name))
)
).then(() => {
return;
SchemaCache.del(className);
return schemaController.reloadData();
});
} else {
return Promise.resolve();
@@ -1688,108 +1693,64 @@ class DatabaseController {
...SchemaController.defaultColumns._Idempotency,
},
};
await this.loadSchema().then(schema => schema.enforceClassExists('_User'));
await this.loadSchema().then(schema => schema.enforceClassExists('_Role'));
if (this.adapter instanceof MongoStorageAdapter) {
await this.loadSchema().then(schema => schema.enforceClassExists('_Idempotency'));
}
const userClassPromise = this.loadSchema().then(schema => schema.enforceClassExists('_User'));
const roleClassPromise = this.loadSchema().then(schema => schema.enforceClassExists('_Role'));
const idempotencyClassPromise =
this.adapter instanceof MongoStorageAdapter
? this.loadSchema().then(schema => schema.enforceClassExists('_Idempotency'))
: Promise.resolve();
await this.adapter.ensureUniqueness('_User', requiredUserFields, ['username']).catch(error => {
logger.warn('Unable to ensure uniqueness for usernames: ', error);
throw error;
});
const usernameUniqueness = userClassPromise
.then(() => this.adapter.ensureUniqueness('_User', requiredUserFields, ['username']))
await this.adapter
.ensureIndex('_User', requiredUserFields, ['username'], 'case_insensitive_username', true)
.catch(error => {
logger.warn('Unable to ensure uniqueness for usernames: ', error);
logger.warn('Unable to create case insensitive username index: ', error);
throw error;
});
const usernameCaseInsensitiveIndex = userClassPromise
.then(() =>
this.adapter.ensureIndex(
'_User',
requiredUserFields,
['username'],
'case_insensitive_username',
true
)
)
await this.adapter
.ensureIndex('_User', requiredUserFields, ['username'], 'case_insensitive_username', true)
.catch(error => {
logger.warn('Unable to create case insensitive username index: ', error);
throw error;
});
const emailUniqueness = userClassPromise
.then(() => this.adapter.ensureUniqueness('_User', requiredUserFields, ['email']))
.catch(error => {
logger.warn('Unable to ensure uniqueness for user email addresses: ', error);
throw error;
});
await this.adapter.ensureUniqueness('_User', requiredUserFields, ['email']).catch(error => {
logger.warn('Unable to ensure uniqueness for user email addresses: ', error);
throw error;
});
const emailCaseInsensitiveIndex = userClassPromise
.then(() =>
this.adapter.ensureIndex(
'_User',
requiredUserFields,
['email'],
'case_insensitive_email',
true
)
)
await this.adapter
.ensureIndex('_User', requiredUserFields, ['email'], 'case_insensitive_email', true)
.catch(error => {
logger.warn('Unable to create case insensitive email index: ', error);
throw error;
});
const roleUniqueness = roleClassPromise
.then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']))
.catch(error => {
logger.warn('Unable to ensure uniqueness for role name: ', error);
throw error;
});
await this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']).catch(error => {
logger.warn('Unable to ensure uniqueness for role name: ', error);
throw error;
});
if (this.adapter instanceof MongoStorageAdapter) {
await this.adapter
.ensureUniqueness('_Idempotency', requiredIdempotencyFields, ['reqId'])
.catch(error => {
logger.warn('Unable to ensure uniqueness for idempotency request ID: ', error);
throw error;
});
const idempotencyRequestIdIndex =
this.adapter instanceof MongoStorageAdapter
? idempotencyClassPromise
.then(() =>
this.adapter.ensureUniqueness('_Idempotency', requiredIdempotencyFields, ['reqId'])
)
.catch(error => {
logger.warn('Unable to ensure uniqueness for idempotency request ID: ', error);
throw error;
})
: Promise.resolve();
const idempotencyExpireIndex =
this.adapter instanceof MongoStorageAdapter
? idempotencyClassPromise
.then(() =>
this.adapter.ensureIndex(
'_Idempotency',
requiredIdempotencyFields,
['expire'],
'ttl',
false,
{ ttl: 0 }
)
)
.catch(error => {
logger.warn('Unable to create TTL index for idempotency expire date: ', error);
throw error;
})
: Promise.resolve();
const indexPromise = this.adapter.updateSchemaWithIndexes();
return Promise.all([
usernameUniqueness,
usernameCaseInsensitiveIndex,
emailUniqueness,
emailCaseInsensitiveIndex,
roleUniqueness,
idempotencyRequestIdIndex,
idempotencyExpireIndex,
indexPromise,
]);
await this.adapter
.ensureIndex('_Idempotency', requiredIdempotencyFields, ['expire'], 'ttl', false, {
ttl: 0,
})
.catch(error => {
logger.warn('Unable to create TTL index for idempotency expire date: ', error);
throw error;
});
}
await this.adapter.updateSchemaWithIndexes();
}
static _validateQuery: any => void;

55
src/Controllers/SchemaCache.js
View File

@@ -1,55 +0,0 @@
const MAIN_SCHEMA = '__MAIN_SCHEMA';
const SCHEMA_CACHE_PREFIX = '__SCHEMA';
import { randomString } from '../cryptoUtils';
import defaults from '../defaults';
export default class SchemaCache {
cache: Object;
constructor(cacheController, ttl = defaults.schemaCacheTTL, singleCache = false) {
this.ttl = ttl;
if (typeof ttl == 'string') {
this.ttl = parseInt(ttl);
}
this.cache = cacheController;
this.prefix = SCHEMA_CACHE_PREFIX;
if (!singleCache) {
this.prefix += randomString(20);
}
}
getAllClasses() {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.get(this.prefix + MAIN_SCHEMA);
}
setAllClasses(schema) {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.put(this.prefix + MAIN_SCHEMA, schema, this.ttl);
}
getOneSchema(className) {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.get(this.prefix + MAIN_SCHEMA).then(cachedSchemas => {
cachedSchemas = cachedSchemas || [];
const schema = cachedSchemas.find(cachedSchema => {
return cachedSchema.className === className;
});
if (schema) {
return Promise.resolve(schema);
}
return Promise.resolve(null);
});
}
clear() {
return this.cache.del(this.prefix + MAIN_SCHEMA);
}
}

125
src/Controllers/SchemaController.js
View File

@@ -17,6 +17,7 @@
// @flow-disable-next
const Parse = require('parse/node').Parse;
import { StorageAdapter } from '../Adapters/Storage/StorageAdapter';
import SchemaCache from '../Adapters/Cache/SchemaCache';
import DatabaseController from './DatabaseController';
import Config from '../Config';
// @flow-disable-next
@@ -682,15 +683,13 @@ const typeToString = (type: SchemaField | string): string => {
export default class SchemaController {
_dbAdapter: StorageAdapter;
schemaData: { [string]: Schema };
_cache: any;
reloadDataPromise: ?Promise<any>;
protectedFields: any;
userIdRegEx: RegExp;
constructor(databaseAdapter: StorageAdapter, schemaCache: any) {
constructor(databaseAdapter: StorageAdapter) {
this._dbAdapter = databaseAdapter;
this._cache = schemaCache;
this.schemaData = new SchemaData();
this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields);
this.protectedFields = Config.get(Parse.applicationId).protectedFields;
const customIds = Config.get(Parse.applicationId).allowCustomObjectId;
@@ -699,6 +698,10 @@ export default class SchemaController {
const autoIdRegEx = /^[a-zA-Z0-9]{1,}$/;
this.userIdRegEx = customIds ? customIdRegEx : autoIdRegEx;
this._dbAdapter.watch(() => {
this.reloadData({ clearCache: true });
});
}
reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise<any> {
@@ -725,12 +728,11 @@ export default class SchemaController {
if (options.clearCache) {
return this.setAllClasses();
}
return this._cache.getAllClasses().then(allClasses => {
if (allClasses && allClasses.length) {
return Promise.resolve(allClasses);
}
return this.setAllClasses();
});
const cached = SchemaCache.all();
if (cached && cached.length) {
return Promise.resolve(cached);
}
return this.setAllClasses();
}
setAllClasses(): Promise<Array<Schema>> {
@@ -738,11 +740,7 @@ export default class SchemaController {
.getAllClasses()
.then(allSchemas => allSchemas.map(injectDefaultSchema))
.then(allSchemas => {
/* eslint-disable no-console */
this._cache
.setAllClasses(allSchemas)
.catch(error => console.error('Error saving schema to cache:', error));
/* eslint-enable no-console */
SchemaCache.put(allSchemas);
return allSchemas;
});
}
@@ -752,32 +750,28 @@ export default class SchemaController {
allowVolatileClasses: boolean = false,
options: LoadSchemaOptions = { clearCache: false }
): Promise<Schema> {
let promise = Promise.resolve();
if (options.clearCache) {
promise = this._cache.clear();
SchemaCache.clear();
}
return promise.then(() => {
if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) {
const data = this.schemaData[className];
return Promise.resolve({
className,
fields: data.fields,
classLevelPermissions: data.classLevelPermissions,
indexes: data.indexes,
});
}
return this._cache.getOneSchema(className).then(cached => {
if (cached && !options.clearCache) {
return Promise.resolve(cached);
}
return this.setAllClasses().then(allSchemas => {
const oneSchema = allSchemas.find(schema => schema.className === className);
if (!oneSchema) {
return Promise.reject(undefined);
}
return oneSchema;
});
if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) {
const data = this.schemaData[className];
return Promise.resolve({
className,
fields: data.fields,
classLevelPermissions: data.classLevelPermissions,
indexes: data.indexes,
});
}
const cached = SchemaCache.get(className);
if (cached && !options.clearCache) {
return Promise.resolve(cached);
}
return this.setAllClasses().then(allSchemas => {
const oneSchema = allSchemas.find(schema => schema.className === className);
if (!oneSchema) {
return Promise.reject(undefined);
}
return oneSchema;
});
}
@@ -788,7 +782,7 @@ export default class SchemaController {
// on success, and rejects with an error on fail. Ensure you
// have authorization (master key, or client class creation
// enabled) before calling this function.
addClassIfNotExists(
async addClassIfNotExists(
className: string,
fields: SchemaFields = {},
classLevelPermissions: any,
@@ -803,9 +797,8 @@ export default class SchemaController {
}
return Promise.reject(validationError);
}
return this._dbAdapter
.createClass(
try {
const adapterSchema = await this._dbAdapter.createClass(
className,
convertSchemaToAdapterSchema({
fields,
@@ -813,18 +806,18 @@ export default class SchemaController {
indexes,
className,
})
)
.then(convertAdapterSchemaToParseSchema)
.catch(error => {
if (error && error.code === Parse.Error.DUPLICATE_VALUE) {
throw new Parse.Error(
Parse.Error.INVALID_CLASS_NAME,
`Class ${className} already exists.`
);
} else {
throw error;
}
});
);
// TODO: Remove by updating schema cache directly
await this.reloadData({ clearCache: true });
const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema);
return parseSchema;
} catch (error) {
if (error && error.code === Parse.Error.DUPLICATE_VALUE) {
throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`);
} else {
throw error;
}
}
}
updateClass(
@@ -938,9 +931,8 @@ export default class SchemaController {
}
// We don't have this class. Update the schema
return (
// The schema update succeeded. Reload the schema
this.addClassIfNotExists(className)
// The schema update succeeded. Reload the schema
.then(() => this.reloadData({ clearCache: true }))
.catch(() => {
// The schema update failed. This can be okay - it might
// have failed because there's a race condition and a different
@@ -1050,12 +1042,16 @@ export default class SchemaController {
}
// Sets the Class-level permissions for a given className, which must exist.
setPermissions(className: string, perms: any, newSchema: SchemaFields) {
async setPermissions(className: string, perms: any, newSchema: SchemaFields) {
if (typeof perms === 'undefined') {
return Promise.resolve();
}
validateCLP(perms, newSchema, this.userIdRegEx);
return this._dbAdapter.setClassLevelPermissions(className, perms);
await this._dbAdapter.setClassLevelPermissions(className, perms);
const cached = SchemaCache.get(className);
if (cached) {
cached.classLevelPermissions = perms;
}
}
// Returns a promise that resolves successfully to the new schema
@@ -1203,7 +1199,9 @@ export default class SchemaController {
);
});
})
.then(() => this._cache.clear());
.then(() => {
SchemaCache.clear();
});
}
// Validates an object provided in REST format.
@@ -1245,6 +1243,7 @@ export default class SchemaController {
const enforceFields = results.filter(result => !!result);
if (enforceFields.length !== 0) {
// TODO: Remove by updating schema cache directly
await this.reloadData({ clearCache: true });
}
this.ensureFields(enforceFields);
@@ -1413,12 +1412,8 @@ export default class SchemaController {
}
// Returns a promise for a new Schema.
const load = (
dbAdapter: StorageAdapter,
schemaCache: any,
options: any
): Promise<SchemaController> => {
const schema = new SchemaController(dbAdapter, schemaCache);
const load = (dbAdapter: StorageAdapter, options: any): Promise<SchemaController> => {
const schema = new SchemaController(dbAdapter);
return schema.reloadData(options).then(() => schema);
};

23
src/Controllers/index.js
View File

@@ -15,7 +15,6 @@ import { PushController } from './PushController';
import { PushQueue } from '../Push/PushQueue';
import { PushWorker } from '../Push/PushWorker';
import DatabaseController from './DatabaseController';
import SchemaCache from './SchemaCache';
// Adapters
import { GridFSBucketAdapter } from '../Adapters/Files/GridFSBucketAdapter';
@@ -26,6 +25,7 @@ import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
import PostgresStorageAdapter from '../Adapters/Storage/Postgres/PostgresStorageAdapter';
import ParsePushAdapter from '@parse/push-adapter';
import ParseGraphQLController from './ParseGraphQLController';
import SchemaCache from '../Adapters/Cache/SchemaCache';
export function getControllers(options: ParseServerOptions) {
const loggerController = getLoggerController(options);
@@ -41,7 +41,7 @@ export function getControllers(options: ParseServerOptions) {
const cacheController = getCacheController(options);
const analyticsController = getAnalyticsController(options);
const liveQueryController = getLiveQueryController(options);
const databaseController = getDatabaseController(options, cacheController);
const databaseController = getDatabaseController(options);
const hooksController = getHooksController(options, databaseController);
const authDataManager = getAuthDataManager(options);
const parseGraphQLController = getParseGraphQLController(options, {
@@ -64,6 +64,7 @@ export function getControllers(options: ParseServerOptions) {
databaseController,
hooksController,
authDataManager,
schemaCache: SchemaCache,
};
}
@@ -141,17 +142,8 @@ export function getLiveQueryController(options: ParseServerOptions): LiveQueryCo
return new LiveQueryController(options.liveQuery);
}
export function getDatabaseController(
options: ParseServerOptions,
cacheController: CacheController
): DatabaseController {
const {
databaseURI,
databaseOptions,
collectionPrefix,
schemaCacheTTL,
enableSingleSchemaCache,
} = options;
export function getDatabaseController(options: ParseServerOptions): DatabaseController {
const { databaseURI, collectionPrefix, databaseOptions } = options;
let { databaseAdapter } = options;
if (
(databaseOptions ||
@@ -165,10 +157,7 @@ export function getDatabaseController(
} else {
databaseAdapter = loadAdapter(databaseAdapter);
}
return new DatabaseController(
databaseAdapter,
new SchemaCache(cacheController, schemaCacheTTL, enableSingleSchemaCache)
);
return new DatabaseController(databaseAdapter);
}
export function getHooksController(