From 56e8e1f4b3bd9bd47c7d5405dc0d6c589386436a Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Tue, 23 Aug 2022 15:54:12 -0700 Subject: [PATCH] (core) Randomize ActiveDoc interval delays Summary: When large numbers of documents were restarted simultaneously, they had a tendency to schedule intervals to begin at roughly the same moment in time, causing periodic spikes in load. This randomizes the delay of each interval to help avoid such spikes. Test Plan: Tested manually. Reviewers: alexmojaki Subscribers: alexmojaki Differential Revision: https://phab.getgrist.com/D3600 --- app/common/RandomizedTimer.ts | 48 ++++++++++++++++++++++++++++ app/server/lib/ActiveDoc.ts | 59 ++++++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 app/common/RandomizedTimer.ts diff --git a/app/common/RandomizedTimer.ts b/app/common/RandomizedTimer.ts new file mode 100644 index 00000000..17ef25da --- /dev/null +++ b/app/common/RandomizedTimer.ts @@ -0,0 +1,48 @@ +/** + * RandomizedTimer takes a function to execute, and calls it on a randomized interval + * between the minimum and maximum delay. The interval delay is randomized between + * each scheduled call. + */ +export class RandomizedTimer { + private _timeout?: NodeJS.Timeout | null; + + constructor( + private _callback: () => void, + private _minDelayMs: number, + private _maxDelayMs: number, + ) {} + + /** + * Sets the timeout and schedules the callback to be called. + */ + public enable(): void { + this._setTimeout(); + } + + /** + * Clears the timeout and prevents the callback from being called. + */ + public disable(): void { + this._clearTimeout(); + } + + private _clearTimeout() { + if (!this._timeout) { return; } + + clearTimeout(this._timeout); + this._timeout = null; + } + + private _setTimeout() { + this._clearTimeout(); + const [min, max] = [this._minDelayMs, this._maxDelayMs]; + const delay = Math.floor(Math.random() * (max - min + 1)) + min; + this._timeout = setTimeout(() => this._onTimeoutTriggered(), delay); + } + + private _onTimeoutTriggered() { + this._clearTimeout(); + this._callback(); + this._setTimeout(); + } +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 7b115d24..78e6764d 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -60,6 +60,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess import {parseUrlId} from 'app/common/gristUrls'; import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; +import {RandomizedTimer} from 'app/common/RandomizedTimer'; import * as roles from 'app/common/roles'; import {schema, SCHEMA_VERSION} from 'app/common/schema'; import {MetaRowRecord, SingleCell} from 'app/common/TableData'; @@ -219,23 +220,8 @@ export class ActiveDoc extends EventEmitter { private _recoveryMode: boolean = false; private _shuttingDown: boolean = false; - // Intervals to clear on shutdown - private _intervals = [ - // Cleanup expired attachments every hour (also happens when shutting down) - setInterval( - () => this.removeUnusedAttachments(true), - REMOVE_UNUSED_ATTACHMENTS_INTERVAL_MS, - ), - setInterval( - () => this._applyUserActions(makeExceptionalDocSession('system'), [["UpdateCurrentTime"]]), - UPDATE_CURRENT_TIME_INTERVAL_MS, - ), - // Measure and broadcast data size every 5 minutes - setInterval( - () => this._checkDataSizeLimitRatio(makeExceptionalDocSession('system')), - UPDATE_DATA_SIZE_INTERVAL_MS, - ), - ]; + // Randomized timers to clear on shutdown. + private _randomizedTimers: RandomizedTimer[] = []; constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) { super(); @@ -486,8 +472,8 @@ export class ActiveDoc extends EventEmitter { // Clear the MapWithTTL to remove all timers from the event loop. this._fetchCache.clear(); - for (const interval of this._intervals) { - clearInterval(interval); + for (const timer of this._randomizedTimers) { + timer.disable(); } // We'll defer syncing usage until everything is calculated. const usageOptions = {syncUsageToDatabase: false, broadcastUsageToClients: false}; @@ -2075,6 +2061,7 @@ export class ActiveDoc extends EventEmitter { this._inactivityTimer.setDelay(closeTimeout); this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`); void this._initializeDocUsage(docSession); + void this._scheduleBackgroundJobs(); } catch (err) { this._fullyLoaded = true; if (!this._shuttingDown) { @@ -2136,6 +2123,40 @@ export class ActiveDoc extends EventEmitter { } } + private _scheduleBackgroundJobs() { + /* In cases where large numbers of documents are restarted simultaneously + * (like during deployments), there's a tendency for scheduled intervals to + * execute at roughly the same moment in time, which causes spikes in load. + * + * To mitigate this, we use randomized timers that re-compute their delay + * in-between intervals, with a maximum variance of 30 seconds. */ + const VARIANCE_MS = 30000; + this._randomizedTimers = [ + // Cleanup expired attachments every hour (also happens when shutting down). + new RandomizedTimer( + () => this.removeUnusedAttachments(true), + REMOVE_UNUSED_ATTACHMENTS_INTERVAL_MS - VARIANCE_MS, + REMOVE_UNUSED_ATTACHMENTS_INTERVAL_MS + VARIANCE_MS, + ), + // Update the time in formulas every hour. + new RandomizedTimer( + () => this._applyUserActions(makeExceptionalDocSession('system'), [['UpdateCurrentTime']]), + UPDATE_CURRENT_TIME_INTERVAL_MS - VARIANCE_MS, + UPDATE_CURRENT_TIME_INTERVAL_MS + VARIANCE_MS, + ), + // Measure and broadcast data size every 5 minutes. + new RandomizedTimer( + () => this._checkDataSizeLimitRatio(makeExceptionalDocSession('system')), + UPDATE_DATA_SIZE_INTERVAL_MS - VARIANCE_MS, + UPDATE_DATA_SIZE_INTERVAL_MS + VARIANCE_MS, + ), + ]; + + for (const timer of this._randomizedTimers) { + timer.enable(); + } + } + /** * Called before a migration. Makes sure a back-up is made. */