joe/kami-parse-server
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve single schema cache (#7214)

Browse Source 
* Initial Commit

* fix flaky test

* temporary set ci timeout

* turn off ci check

* fix postgres tests

* fix tests

* node flaky test

* remove improvements

* Update SchemaPerformance.spec.js

* fix tests

* revert ci

* Create Singleton Object

* properly clear cache testing

* Cleanup

* remove fit

* try PushController.spec

* try push test rewrite

* try push enqueue time

* Increase test timeout

* remove pg server creation test

* xit push tests

* more xit

* remove skipped tests

* Fix conflicts

* reduce ci timeout

* fix push tests

* Revert "fix push tests"

This reverts commit 05aba62f1cbbca7d5d3e80b9444529f59407cb56.

* improve initialization

* fix flaky tests

* xit flaky test

* Update CHANGELOG.md

* enable debug logs

* Update LogsRouter.spec.js

* create initial indexes in series

* lint

* horizontal scaling documentation

* Update Changelog

* change horizontalScaling db option

* Add enableSchemaHooks option

* move enableSchemaHooks to databaseOptions
This commit is contained in:
Diamond Lewis
2021-03-16 16:05:36 -05:00
committed by GitHub
parent 32fc45d2d2
commit a02014f557
38 changed files with 673 additions and 937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/Adapters/Auth/instagram.js
View File

@@ -8,7 +8,7 @@ function validateAuthData(authData) {
const apiURL = authData.apiURL || defaultURL;
const path = `${apiURL}me?fields=id&access_token=${authData.access_token}`;
return httpsRequest.get(path).then(response => {
const user = response.data ? response.data : response
const user = response.data ? response.data : response;
if (user && user.id == authData.id) {
return;
}

23
src/Adapters/Cache/SchemaCache.js Normal file
View File

@@ -0,0 +1,23 @@
const SchemaCache = {};
export default {
all() {
return [...(SchemaCache.allClasses || [])];
},
get(className) {
return this.all().find(cached => cached.className === className);
},
put(allSchema) {
SchemaCache.allClasses = allSchema;
},
del(className) {
this.put(this.all().filter(cached => cached.className !== className));
},
clear() {
delete SchemaCache.allClasses;
},
};

18
src/Adapters/Storage/Mongo/MongoStorageAdapter.js
View File

@@ -113,12 +113,15 @@ export class MongoStorageAdapter implements StorageAdapter {
_uri: string;
_collectionPrefix: string;
_mongoOptions: Object;
_onchange: any;
_stream: any;
// Public
connectionPromise: ?Promise<any>;
database: any;
client: MongoClient;
_maxTimeMS: ?number;
canSortOnJoinTables: boolean;
enableSchemaHooks: boolean;
constructor({ uri = defaults.DefaultMongoURI, collectionPrefix = '', mongoOptions = {} }: any) {
this._uri = uri;
@@ -126,13 +129,20 @@ export class MongoStorageAdapter implements StorageAdapter {
this._mongoOptions = mongoOptions;
this._mongoOptions.useNewUrlParser = true;
this._mongoOptions.useUnifiedTopology = true;
this._onchange = () => {};
// MaxTimeMS is not a global MongoDB client option, it is applied per operation.
this._maxTimeMS = mongoOptions.maxTimeMS;
this.canSortOnJoinTables = true;
this.enableSchemaHooks = !!mongoOptions.enableSchemaHooks;
delete mongoOptions.enableSchemaHooks;
delete mongoOptions.maxTimeMS;
}
watch(callback: () => void): void {
this._onchange = callback;
}
connect() {
if (this.connectionPromise) {
return this.connectionPromise;
@@ -198,7 +208,13 @@ export class MongoStorageAdapter implements StorageAdapter {
_schemaCollection(): Promise<MongoSchemaCollection> {
return this.connect()
.then(() => this._adaptiveCollection(MongoSchemaCollectionName))
.then(collection => new MongoSchemaCollection(collection));
.then(collection => {
if (!this._stream && this.enableSchemaHooks) {
this._stream = collection._mongoCollection.watch();
this._stream.on('change', () => this._onchange());
}
return new MongoSchemaCollection(collection);
});
}
classExists(name: string) {

58
src/Adapters/Storage/Postgres/PostgresStorageAdapter.js
View File

@@ -4,6 +4,8 @@ import { createClient } from './PostgresClient';
import Parse from 'parse/node';
// @flow-disable-next
import _ from 'lodash';
// @flow-disable-next
import { v4 as uuidv4 } from 'uuid';
import sql from './sql';
const PostgresRelationDoesNotExistError = '42P01';
@@ -794,20 +796,33 @@ const buildWhereClause = ({ schema, query, index, caseInsensitive }): WhereClaus
export class PostgresStorageAdapter implements StorageAdapter {
canSortOnJoinTables: boolean;
enableSchemaHooks: boolean;
// Private
_collectionPrefix: string;
_client: any;
_onchange: any;
_pgp: any;
_stream: any;
_uuid: any;
constructor({ uri, collectionPrefix = '', databaseOptions }: any) {
constructor({ uri, collectionPrefix = '', databaseOptions = {} }: any) {
this._collectionPrefix = collectionPrefix;
this.enableSchemaHooks = !!databaseOptions.enableSchemaHooks;
delete databaseOptions.enableSchemaHooks;
const { client, pgp } = createClient(uri, databaseOptions);
this._client = client;
this._onchange = () => {};
this._pgp = pgp;
this._uuid = uuidv4();
this.canSortOnJoinTables = false;
}
watch(callback: () => void): void {
this._onchange = callback;
}
//Note that analyze=true will run the query, executing INSERTS, DELETES, etc.
createExplainableQuery(query: string, analyze: boolean = false) {
if (analyze) {
@@ -818,12 +833,39 @@ export class PostgresStorageAdapter implements StorageAdapter {
}
handleShutdown() {
if (this._stream) {
this._stream.done();
delete this._stream;
}
if (!this._client) {
return;
}
this._client.$pool.end();
}
async _listenToSchema() {
if (!this._stream && this.enableSchemaHooks) {
this._stream = await this._client.connect({ direct: true });
this._stream.client.on('notification', data => {
const payload = JSON.parse(data.payload);
if (payload.senderId !== this._uuid) {
this._onchange();
}
});
await this._stream.none('LISTEN $1~', 'schema.change');
}
}
_notifySchemaChange() {
if (this._stream) {
this._stream
.none('NOTIFY $1~, $2', ['schema.change', { senderId: this._uuid }])
.catch(error => {
console.log('Failed to Notify:', error); // unlikely to ever happen
});
}
}
async _ensureSchemaCollectionExists(conn: any) {
conn = conn || this._client;
await conn
@@ -859,6 +901,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
values
);
});
this._notifySchemaChange();
}
async setIndexesWithSchemaFormat(
@@ -920,11 +963,12 @@ export class PostgresStorageAdapter implements StorageAdapter {
[className, 'schema', 'indexes', JSON.stringify(existingIndexes)]
);
});
this._notifySchemaChange();
}
async createClass(className: string, schema: SchemaType, conn: ?any) {
conn = conn || this._client;
return conn
const parseSchema = await conn
.tx('create-class', async t => {
await this.createTable(className, schema, t);
await t.none(
@@ -940,6 +984,8 @@ export class PostgresStorageAdapter implements StorageAdapter {
}
throw err;
});
this._notifySchemaChange();
return parseSchema;
}
// Just create a table, do not insert in schema
@@ -1073,6 +1119,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
);
}
});
this._notifySchemaChange();
}
// Drops a collection. Resolves with true if it was a Parse Schema (eg. _User, Custom, etc.)
@@ -1085,9 +1132,12 @@ export class PostgresStorageAdapter implements StorageAdapter {
values: [className],
},
];
return this._client
const response = await this._client
.tx(t => t.none(this._pgp.helpers.concat(operations)))
.then(() => className.indexOf('_Join:') != 0); // resolves with false when _Join table
this._notifySchemaChange();
return response;
}
// Delete all data known to this adapter. Used for testing.
@@ -1173,6 +1223,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
await t.none(`ALTER TABLE $1:name DROP COLUMN IF EXISTS ${columns}`, values);
}
});
this._notifySchemaChange();
}
// Return a promise for all schemas known to this adapter, in Parse format. In case the
@@ -2237,6 +2288,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
})
.then(() => this.schemaUpgrade(schema.className, schema));
});
promises.push(this._listenToSchema());
return Promise.all(promises)
.then(() => {
return this._client.tx('perform-initialization', async t => {

1
src/Adapters/Storage/StorageAdapter.js
View File

@@ -111,6 +111,7 @@ export interface StorageAdapter {
explain?: boolean
): Promise<any>;
performInitialization(options: ?any): Promise<void>;
watch(callback: () => void): void;
// Indexing
createIndexes(className: string, indexes: any, conn: ?any): Promise<void>;

8
src/Config.js
View File

@@ -3,7 +3,6 @@
// mount is the URL for the root of the API; includes http, domain, etc.
import AppCache from './cache';
import SchemaCache from './Controllers/SchemaCache';
import DatabaseController from './Controllers/DatabaseController';
import net from 'net';
import {
@@ -35,12 +34,7 @@ export class Config {
config.applicationId = applicationId;
Object.keys(cacheInfo).forEach(key => {
if (key == 'databaseController') {
const schemaCache = new SchemaCache(
cacheInfo.cacheController,
cacheInfo.schemaCacheTTL,
cacheInfo.enableSingleSchemaCache
);
config.database = new DatabaseController(cacheInfo.databaseController.adapter, schemaCache);
config.database = new DatabaseController(cacheInfo.databaseController.adapter);
} else {
config[key] = cacheInfo[key];
}

149
src/Controllers/DatabaseController.js
View File

@@ -13,6 +13,9 @@ import deepcopy from 'deepcopy';
import logger from '../logger';
import * as SchemaController from './SchemaController';
import { StorageAdapter } from '../Adapters/Storage/StorageAdapter';
import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
import SchemaCache from '../Adapters/Cache/SchemaCache';
import type { LoadSchemaOptions } from './types';
import type { QueryOptions, FullQueryOptions } from '../Adapters/Storage/StorageAdapter';
function addWriteACL(query, acl) {
@@ -230,9 +233,6 @@ const filterSensitiveData = (
return object;
};
import type { LoadSchemaOptions } from './types';
import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
// Runs an update on the database.
// Returns a promise for an object with the new values for field
// modifications that don't know their results ahead of time, like
@@ -398,9 +398,8 @@ class DatabaseController {
schemaPromise: ?Promise<SchemaController.SchemaController>;
_transactionalSession: ?any;
constructor(adapter: StorageAdapter, schemaCache: any) {
constructor(adapter: StorageAdapter) {
this.adapter = adapter;
this.schemaCache = schemaCache;
// We don't want a mutable this.schema, because then you could have
// one request that uses different schemas for different parts of
// it. Instead, use loadSchema to get a schema.
@@ -434,7 +433,7 @@ class DatabaseController {
if (this.schemaPromise != null) {
return this.schemaPromise;
}
this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options);
this.schemaPromise = SchemaController.load(this.adapter, options);
this.schemaPromise.then(
() => delete this.schemaPromise,
() => delete this.schemaPromise
@@ -916,7 +915,8 @@ class DatabaseController {
*/
deleteEverything(fast: boolean = false): Promise<any> {
this.schemaPromise = null;
return Promise.all([this.adapter.deleteAllClasses(fast), this.schemaCache.clear()]);
SchemaCache.clear();
return this.adapter.deleteAllClasses(fast);
}
// Returns a promise for a list of related ids given an owning id.
@@ -1325,8 +1325,12 @@ class DatabaseController {
}
deleteSchema(className: string): Promise<void> {
let schemaController;
return this.loadSchema({ clearCache: true })
.then(schemaController => schemaController.getOneSchema(className, true))
.then(s => {
schemaController = s;
return schemaController.getOneSchema(className, true);
})
.catch(error => {
if (error === undefined) {
return { fields: {} };
@@ -1356,7 +1360,8 @@ class DatabaseController {
this.adapter.deleteClass(joinTableName(className, name))
)
).then(() => {
return;
SchemaCache.del(className);
return schemaController.reloadData();
});
} else {
return Promise.resolve();
@@ -1688,108 +1693,64 @@ class DatabaseController {
...SchemaController.defaultColumns._Idempotency,
},
};
await this.loadSchema().then(schema => schema.enforceClassExists('_User'));
await this.loadSchema().then(schema => schema.enforceClassExists('_Role'));
if (this.adapter instanceof MongoStorageAdapter) {
await this.loadSchema().then(schema => schema.enforceClassExists('_Idempotency'));
}
const userClassPromise = this.loadSchema().then(schema => schema.enforceClassExists('_User'));
const roleClassPromise = this.loadSchema().then(schema => schema.enforceClassExists('_Role'));
const idempotencyClassPromise =
this.adapter instanceof MongoStorageAdapter
? this.loadSchema().then(schema => schema.enforceClassExists('_Idempotency'))
: Promise.resolve();
await this.adapter.ensureUniqueness('_User', requiredUserFields, ['username']).catch(error => {
logger.warn('Unable to ensure uniqueness for usernames: ', error);
throw error;
});
const usernameUniqueness = userClassPromise
.then(() => this.adapter.ensureUniqueness('_User', requiredUserFields, ['username']))
await this.adapter
.ensureIndex('_User', requiredUserFields, ['username'], 'case_insensitive_username', true)
.catch(error => {
logger.warn('Unable to ensure uniqueness for usernames: ', error);
logger.warn('Unable to create case insensitive username index: ', error);
throw error;
});
const usernameCaseInsensitiveIndex = userClassPromise
.then(() =>
this.adapter.ensureIndex(
'_User',
requiredUserFields,
['username'],
'case_insensitive_username',
true
)
)
await this.adapter
.ensureIndex('_User', requiredUserFields, ['username'], 'case_insensitive_username', true)
.catch(error => {
logger.warn('Unable to create case insensitive username index: ', error);
throw error;
});
const emailUniqueness = userClassPromise
.then(() => this.adapter.ensureUniqueness('_User', requiredUserFields, ['email']))
.catch(error => {
logger.warn('Unable to ensure uniqueness for user email addresses: ', error);
throw error;
});
await this.adapter.ensureUniqueness('_User', requiredUserFields, ['email']).catch(error => {
logger.warn('Unable to ensure uniqueness for user email addresses: ', error);
throw error;
});
const emailCaseInsensitiveIndex = userClassPromise
.then(() =>
this.adapter.ensureIndex(
'_User',
requiredUserFields,
['email'],
'case_insensitive_email',
true
)
)
await this.adapter
.ensureIndex('_User', requiredUserFields, ['email'], 'case_insensitive_email', true)
.catch(error => {
logger.warn('Unable to create case insensitive email index: ', error);
throw error;
});
const roleUniqueness = roleClassPromise
.then(() => this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']))
.catch(error => {
logger.warn('Unable to ensure uniqueness for role name: ', error);
throw error;
});
await this.adapter.ensureUniqueness('_Role', requiredRoleFields, ['name']).catch(error => {
logger.warn('Unable to ensure uniqueness for role name: ', error);
throw error;
});
if (this.adapter instanceof MongoStorageAdapter) {
await this.adapter
.ensureUniqueness('_Idempotency', requiredIdempotencyFields, ['reqId'])
.catch(error => {
logger.warn('Unable to ensure uniqueness for idempotency request ID: ', error);
throw error;
});
const idempotencyRequestIdIndex =
this.adapter instanceof MongoStorageAdapter
? idempotencyClassPromise
.then(() =>
this.adapter.ensureUniqueness('_Idempotency', requiredIdempotencyFields, ['reqId'])
)
.catch(error => {
logger.warn('Unable to ensure uniqueness for idempotency request ID: ', error);
throw error;
})
: Promise.resolve();
const idempotencyExpireIndex =
this.adapter instanceof MongoStorageAdapter
? idempotencyClassPromise
.then(() =>
this.adapter.ensureIndex(
'_Idempotency',
requiredIdempotencyFields,
['expire'],
'ttl',
false,
{ ttl: 0 }
)
)
.catch(error => {
logger.warn('Unable to create TTL index for idempotency expire date: ', error);
throw error;
})
: Promise.resolve();
const indexPromise = this.adapter.updateSchemaWithIndexes();
return Promise.all([
usernameUniqueness,
usernameCaseInsensitiveIndex,
emailUniqueness,
emailCaseInsensitiveIndex,
roleUniqueness,
idempotencyRequestIdIndex,
idempotencyExpireIndex,
indexPromise,
]);
await this.adapter
.ensureIndex('_Idempotency', requiredIdempotencyFields, ['expire'], 'ttl', false, {
ttl: 0,
})
.catch(error => {
logger.warn('Unable to create TTL index for idempotency expire date: ', error);
throw error;
});
}
await this.adapter.updateSchemaWithIndexes();
}
static _validateQuery: any => void;

55
src/Controllers/SchemaCache.js
View File

@@ -1,55 +0,0 @@
const MAIN_SCHEMA = '__MAIN_SCHEMA';
const SCHEMA_CACHE_PREFIX = '__SCHEMA';
import { randomString } from '../cryptoUtils';
import defaults from '../defaults';
export default class SchemaCache {
cache: Object;
constructor(cacheController, ttl = defaults.schemaCacheTTL, singleCache = false) {
this.ttl = ttl;
if (typeof ttl == 'string') {
this.ttl = parseInt(ttl);
}
this.cache = cacheController;
this.prefix = SCHEMA_CACHE_PREFIX;
if (!singleCache) {
this.prefix += randomString(20);
}
}
getAllClasses() {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.get(this.prefix + MAIN_SCHEMA);
}
setAllClasses(schema) {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.put(this.prefix + MAIN_SCHEMA, schema, this.ttl);
}
getOneSchema(className) {
if (!this.ttl) {
return Promise.resolve(null);
}
return this.cache.get(this.prefix + MAIN_SCHEMA).then(cachedSchemas => {
cachedSchemas = cachedSchemas || [];
const schema = cachedSchemas.find(cachedSchema => {
return cachedSchema.className === className;
});
if (schema) {
return Promise.resolve(schema);
}
return Promise.resolve(null);
});
}
clear() {
return this.cache.del(this.prefix + MAIN_SCHEMA);
}
}

125
src/Controllers/SchemaController.js
View File

@@ -17,6 +17,7 @@
// @flow-disable-next
const Parse = require('parse/node').Parse;
import { StorageAdapter } from '../Adapters/Storage/StorageAdapter';
import SchemaCache from '../Adapters/Cache/SchemaCache';
import DatabaseController from './DatabaseController';
import Config from '../Config';
// @flow-disable-next
@@ -682,15 +683,13 @@ const typeToString = (type: SchemaField | string): string => {
export default class SchemaController {
_dbAdapter: StorageAdapter;
schemaData: { [string]: Schema };
_cache: any;
reloadDataPromise: ?Promise<any>;
protectedFields: any;
userIdRegEx: RegExp;
constructor(databaseAdapter: StorageAdapter, schemaCache: any) {
constructor(databaseAdapter: StorageAdapter) {
this._dbAdapter = databaseAdapter;
this._cache = schemaCache;
this.schemaData = new SchemaData();
this.schemaData = new SchemaData(SchemaCache.all(), this.protectedFields);
this.protectedFields = Config.get(Parse.applicationId).protectedFields;
const customIds = Config.get(Parse.applicationId).allowCustomObjectId;
@@ -699,6 +698,10 @@ export default class SchemaController {
const autoIdRegEx = /^[a-zA-Z0-9]{1,}$/;
this.userIdRegEx = customIds ? customIdRegEx : autoIdRegEx;
this._dbAdapter.watch(() => {
this.reloadData({ clearCache: true });
});
}
reloadData(options: LoadSchemaOptions = { clearCache: false }): Promise<any> {
@@ -725,12 +728,11 @@ export default class SchemaController {
if (options.clearCache) {
return this.setAllClasses();
}
return this._cache.getAllClasses().then(allClasses => {
if (allClasses && allClasses.length) {
return Promise.resolve(allClasses);
}
return this.setAllClasses();
});
const cached = SchemaCache.all();
if (cached && cached.length) {
return Promise.resolve(cached);
}
return this.setAllClasses();
}
setAllClasses(): Promise<Array<Schema>> {
@@ -738,11 +740,7 @@ export default class SchemaController {
.getAllClasses()
.then(allSchemas => allSchemas.map(injectDefaultSchema))
.then(allSchemas => {
/* eslint-disable no-console */
this._cache
.setAllClasses(allSchemas)
.catch(error => console.error('Error saving schema to cache:', error));
/* eslint-enable no-console */
SchemaCache.put(allSchemas);
return allSchemas;
});
}
@@ -752,32 +750,28 @@ export default class SchemaController {
allowVolatileClasses: boolean = false,
options: LoadSchemaOptions = { clearCache: false }
): Promise<Schema> {
let promise = Promise.resolve();
if (options.clearCache) {
promise = this._cache.clear();
SchemaCache.clear();
}
return promise.then(() => {
if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) {
const data = this.schemaData[className];
return Promise.resolve({
className,
fields: data.fields,
classLevelPermissions: data.classLevelPermissions,
indexes: data.indexes,
});
}
return this._cache.getOneSchema(className).then(cached => {
if (cached && !options.clearCache) {
return Promise.resolve(cached);
}
return this.setAllClasses().then(allSchemas => {
const oneSchema = allSchemas.find(schema => schema.className === className);
if (!oneSchema) {
return Promise.reject(undefined);
}
return oneSchema;
});
if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) {
const data = this.schemaData[className];
return Promise.resolve({
className,
fields: data.fields,
classLevelPermissions: data.classLevelPermissions,
indexes: data.indexes,
});
}
const cached = SchemaCache.get(className);
if (cached && !options.clearCache) {
return Promise.resolve(cached);
}
return this.setAllClasses().then(allSchemas => {
const oneSchema = allSchemas.find(schema => schema.className === className);
if (!oneSchema) {
return Promise.reject(undefined);
}
return oneSchema;
});
}
@@ -788,7 +782,7 @@ export default class SchemaController {
// on success, and rejects with an error on fail. Ensure you
// have authorization (master key, or client class creation
// enabled) before calling this function.
addClassIfNotExists(
async addClassIfNotExists(
className: string,
fields: SchemaFields = {},
classLevelPermissions: any,
@@ -803,9 +797,8 @@ export default class SchemaController {
}
return Promise.reject(validationError);
}
return this._dbAdapter
.createClass(
try {
const adapterSchema = await this._dbAdapter.createClass(
className,
convertSchemaToAdapterSchema({
fields,
@@ -813,18 +806,18 @@ export default class SchemaController {
indexes,
className,
})
)
.then(convertAdapterSchemaToParseSchema)
.catch(error => {
if (error && error.code === Parse.Error.DUPLICATE_VALUE) {
throw new Parse.Error(
Parse.Error.INVALID_CLASS_NAME,
`Class ${className} already exists.`
);
} else {
throw error;
}
});
);
// TODO: Remove by updating schema cache directly
await this.reloadData({ clearCache: true });
const parseSchema = convertAdapterSchemaToParseSchema(adapterSchema);
return parseSchema;
} catch (error) {
if (error && error.code === Parse.Error.DUPLICATE_VALUE) {
throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`);
} else {
throw error;
}
}
}
updateClass(
@@ -938,9 +931,8 @@ export default class SchemaController {
}
// We don't have this class. Update the schema
return (
// The schema update succeeded. Reload the schema
this.addClassIfNotExists(className)
// The schema update succeeded. Reload the schema
.then(() => this.reloadData({ clearCache: true }))
.catch(() => {
// The schema update failed. This can be okay - it might
// have failed because there's a race condition and a different
@@ -1050,12 +1042,16 @@ export default class SchemaController {
}
// Sets the Class-level permissions for a given className, which must exist.
setPermissions(className: string, perms: any, newSchema: SchemaFields) {
async setPermissions(className: string, perms: any, newSchema: SchemaFields) {
if (typeof perms === 'undefined') {
return Promise.resolve();
}
validateCLP(perms, newSchema, this.userIdRegEx);
return this._dbAdapter.setClassLevelPermissions(className, perms);
await this._dbAdapter.setClassLevelPermissions(className, perms);
const cached = SchemaCache.get(className);
if (cached) {
cached.classLevelPermissions = perms;
}
}
// Returns a promise that resolves successfully to the new schema
@@ -1203,7 +1199,9 @@ export default class SchemaController {
);
});
})
.then(() => this._cache.clear());
.then(() => {
SchemaCache.clear();
});
}
// Validates an object provided in REST format.
@@ -1245,6 +1243,7 @@ export default class SchemaController {
const enforceFields = results.filter(result => !!result);
if (enforceFields.length !== 0) {
// TODO: Remove by updating schema cache directly
await this.reloadData({ clearCache: true });
}
this.ensureFields(enforceFields);
@@ -1413,12 +1412,8 @@ export default class SchemaController {
}
// Returns a promise for a new Schema.
const load = (
dbAdapter: StorageAdapter,
schemaCache: any,
options: any
): Promise<SchemaController> => {
const schema = new SchemaController(dbAdapter, schemaCache);
const load = (dbAdapter: StorageAdapter, options: any): Promise<SchemaController> => {
const schema = new SchemaController(dbAdapter);
return schema.reloadData(options).then(() => schema);
};

23
src/Controllers/index.js
View File

@@ -15,7 +15,6 @@ import { PushController } from './PushController';
import { PushQueue } from '../Push/PushQueue';
import { PushWorker } from '../Push/PushWorker';
import DatabaseController from './DatabaseController';
import SchemaCache from './SchemaCache';
// Adapters
import { GridFSBucketAdapter } from '../Adapters/Files/GridFSBucketAdapter';
@@ -26,6 +25,7 @@ import MongoStorageAdapter from '../Adapters/Storage/Mongo/MongoStorageAdapter';
import PostgresStorageAdapter from '../Adapters/Storage/Postgres/PostgresStorageAdapter';
import ParsePushAdapter from '@parse/push-adapter';
import ParseGraphQLController from './ParseGraphQLController';
import SchemaCache from '../Adapters/Cache/SchemaCache';
export function getControllers(options: ParseServerOptions) {
const loggerController = getLoggerController(options);
@@ -41,7 +41,7 @@ export function getControllers(options: ParseServerOptions) {
const cacheController = getCacheController(options);
const analyticsController = getAnalyticsController(options);
const liveQueryController = getLiveQueryController(options);
const databaseController = getDatabaseController(options, cacheController);
const databaseController = getDatabaseController(options);
const hooksController = getHooksController(options, databaseController);
const authDataManager = getAuthDataManager(options);
const parseGraphQLController = getParseGraphQLController(options, {
@@ -64,6 +64,7 @@ export function getControllers(options: ParseServerOptions) {
databaseController,
hooksController,
authDataManager,
schemaCache: SchemaCache,
};
}
@@ -141,17 +142,8 @@ export function getLiveQueryController(options: ParseServerOptions): LiveQueryCo
return new LiveQueryController(options.liveQuery);
}
export function getDatabaseController(
options: ParseServerOptions,
cacheController: CacheController
): DatabaseController {
const {
databaseURI,
databaseOptions,
collectionPrefix,
schemaCacheTTL,
enableSingleSchemaCache,
} = options;
export function getDatabaseController(options: ParseServerOptions): DatabaseController {
const { databaseURI, collectionPrefix, databaseOptions } = options;
let { databaseAdapter } = options;
if (
(databaseOptions ||
@@ -165,10 +157,7 @@ export function getDatabaseController(
} else {
databaseAdapter = loadAdapter(databaseAdapter);
}
return new DatabaseController(
databaseAdapter,
new SchemaCache(cacheController, schemaCacheTTL, enableSingleSchemaCache)
);
return new DatabaseController(databaseAdapter);
}
export function getHooksController(

3
src/GraphQL/ParseGraphQLSchema.js
View File

@@ -11,6 +11,7 @@ import * as defaultGraphQLQueries from './loaders/defaultGraphQLQueries';
import * as defaultGraphQLMutations from './loaders/defaultGraphQLMutations';
import ParseGraphQLController, { ParseGraphQLConfig } from '../Controllers/ParseGraphQLController';
import DatabaseController from '../Controllers/DatabaseController';
import SchemaCache from '../Adapters/Cache/SchemaCache';
import { toGraphQLError } from './parseGraphQLUtils';
import * as schemaDirectives from './loaders/schemaDirectives';
import * as schemaTypes from './loaders/schemaTypes';
@@ -66,6 +67,7 @@ class ParseGraphQLSchema {
log: any;
appId: string;
graphQLCustomTypeDefs: ?(string | GraphQLSchema | DocumentNode | GraphQLNamedType[]);
schemaCache: any;
constructor(
params: {
@@ -85,6 +87,7 @@ class ParseGraphQLSchema {
this.log = params.log || requiredParameter('You must provide a log instance!');
this.graphQLCustomTypeDefs = params.graphQLCustomTypeDefs;
this.appId = params.appId || requiredParameter('You must provide the appId!');
this.schemaCache = SchemaCache;
}
async load() {

25
src/Options/Definitions.js
View File

@@ -100,7 +100,7 @@ module.exports.ParseServerOptions = {
},
databaseOptions: {
env: 'PARSE_SERVER_DATABASE_OPTIONS',
help: 'Options to pass to the mongodb client',
help: 'Options to pass to the database client',
action: parsers.objectParser,
},
databaseURI: {
@@ -149,13 +149,6 @@ module.exports.ParseServerOptions = {
action: parsers.booleanParser,
default: false,
},
enableSingleSchemaCache: {
env: 'PARSE_SERVER_ENABLE_SINGLE_SCHEMA_CACHE',
help:
'Use a single schema cache shared across requests. Reduces number of queries made to _SCHEMA, defaults to false, i.e. unique schema cache per request.',
action: parsers.booleanParser,
default: false,
},
encryptionKey: {
env: 'PARSE_SERVER_ENCRYPTION_KEY',
help: 'Key for encrypting your files',
@@ -366,13 +359,6 @@ module.exports.ParseServerOptions = {
action: parsers.booleanParser,
default: false,
},
schemaCacheTTL: {
env: 'PARSE_SERVER_SCHEMA_CACHE_TTL',
help:
'The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 5000; set 0 to disable.',
action: parsers.numberParser('schemaCacheTTL'),
default: 5000,
},
security: {
env: 'PARSE_SERVER_SECURITY',
help: 'The security options to identify and report weak security settings.',
@@ -788,3 +774,12 @@ module.exports.FileUploadOptions = {
default: false,
},
};
module.exports.DatabaseOptions = {
enableSchemaHooks: {
env: 'PARSE_SERVER_DATABASE_ENABLE_SCHEMA_HOOKS',
help:
'Enables database hooks to update single schema cache. Set to true if using multiple Parse Servers instances connected to the same database.',
action: parsers.booleanParser,
default: false,
},
};

9
src/Options/docs.js
View File

@@ -18,7 +18,7 @@
* @property {String} collectionPrefix A collection prefix for the classes
* @property {CustomPagesOptions} customPages custom pages for password validation and reset
* @property {Adapter<StorageAdapter>} databaseAdapter Adapter module for the database
* @property {Any} databaseOptions Options to pass to the mongodb client
* @property {DatabaseOptions} databaseOptions Options to pass to the database client
* @property {String} databaseURI The full URI to your database. Supported databases are mongodb or postgres.
* @property {Boolean} directAccess Replace HTTP Interface when using JS SDK in current node runtime, defaults to false. Caution, this is an experimental feature that may not be appropriate for production.
* @property {String} dotNetKey Key for Unity and .Net SDK
@@ -27,7 +27,6 @@
* @property {Number} emailVerifyTokenValidityDuration Email verification token validity duration, in seconds
* @property {Boolean} enableAnonymousUsers Enable (or disable) anonymous users, defaults to true
* @property {Boolean} enableExpressErrorHandler Enables the default express error handler for all errors
* @property {Boolean} enableSingleSchemaCache Use a single schema cache shared across requests. Reduces number of queries made to _SCHEMA, defaults to false, i.e. unique schema cache per request.
* @property {String} encryptionKey Key for encrypting your files
* @property {Boolean} expireInactiveSessions Sets wether we should expire the inactive sessions, defaults to true
* @property {String} fileKey Key for your files
@@ -67,7 +66,6 @@
* @property {String} restAPIKey Key for REST calls
* @property {Boolean} revokeSessionOnPasswordReset When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.
* @property {Boolean} scheduledPush Configuration for push scheduling, defaults to false.
* @property {Number} schemaCacheTTL The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 5000; set 0 to disable.
* @property {SecurityOptions} security The security options to identify and report weak security settings.
* @property {Function} serverCloseComplete Callback when server has closed
* @property {Function} serverStartComplete Callback when server has started
@@ -190,3 +188,8 @@
* @property {Boolean} enableForAuthenticatedUser Is true if file upload should be allowed for authenticated users.
* @property {Boolean} enableForPublic Is true if file upload should be allowed for anyone, regardless of user authentication.
*/
/**
* @interface DatabaseOptions
* @property {Boolean} enableSchemaHooks Enables database hooks to update single schema cache. Set to true if using multiple Parse Servers instances connected to the same database.
*/

17
src/Options/index.js
View File

@@ -63,8 +63,9 @@ export interface ParseServerOptions {
/* The full URI to your database. Supported databases are mongodb or postgres.
:DEFAULT: mongodb://localhost:27017/parse */
databaseURI: string;
/* Options to pass to the mongodb client */
databaseOptions: ?any;
/* Options to pass to the database client
:ENV: PARSE_SERVER_DATABASE_OPTIONS */
databaseOptions: ?DatabaseOptions;
/* Adapter module for the database */
databaseAdapter: ?Adapter<StorageAdapter>;
/* Full path to your cloud code main.js */
@@ -158,9 +159,6 @@ export interface ParseServerOptions {
/* When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.
:DEFAULT: true */
revokeSessionOnPasswordReset: ?boolean;
/* The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 5000; set 0 to disable.
:DEFAULT: 5000 */
schemaCacheTTL: ?number;
/* Sets the TTL for the in memory cache (in ms), defaults to 5000 (5 seconds)
:DEFAULT: 5000 */
cacheTTL: ?number;
@@ -171,9 +169,6 @@ export interface ParseServerOptions {
:ENV: PARSE_SERVER_ENABLE_EXPERIMENTAL_DIRECT_ACCESS
:DEFAULT: false */
directAccess: ?boolean;
/* Use a single schema cache shared across requests. Reduces number of queries made to _SCHEMA, defaults to false, i.e. unique schema cache per request.
:DEFAULT: false */
enableSingleSchemaCache: ?boolean;
/* Enables the default express error handler for all errors
:DEFAULT: false */
enableExpressErrorHandler: ?boolean;
@@ -416,3 +411,9 @@ export interface FileUploadOptions {
:DEFAULT: false */
enableForPublic: ?boolean;
}
export interface DatabaseOptions {
/* Enables database hooks to update single schema cache. Set to true if using multiple Parse Servers instances connected to the same database.
:DEFAULT: false */
enableSchemaHooks: ?boolean;
}

10
src/PromiseRouter.js
View File

@@ -150,7 +150,6 @@ function makeExpressHandler(appId, promiseHandler) {
promiseHandler(req)
.then(
result => {
clearSchemaCache(req);
if (!result.response && !result.location && !result.text) {
log.error('the handler did not include a "response" or a "location" field');
throw 'control should not get here';
@@ -184,17 +183,14 @@ function makeExpressHandler(appId, promiseHandler) {
res.json(result.response);
},
error => {
clearSchemaCache(req);
next(error);
}
)
.catch(e => {
clearSchemaCache(req);
log.error(`Error generating response. ${inspect(e)}`, { error: e });
next(e);
});
} catch (e) {
clearSchemaCache(req);
log.error(`Error handling request: ${inspect(e)}`, { error: e });
next(e);
}
@@ -212,9 +208,3 @@ function maskSensitiveUrl(req) {
}
return maskUrl;
}
function clearSchemaCache(req) {
if (req.config && !req.config.enableSingleSchemaCache) {
req.config.database.schemaCache.clear();
}
}