Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

Shutdown Doc worker when it is not considered as available in Redis #831 (#856)

Browse Source 
* Shutdown Doc worker when it is not considered as available in Redis
* Use isAffirmative for GRIST_MANAGED_WORKERS
* Upgrade Sinon for the tests
* Run Smoke test with pages in English
* Add logic in /status endpoint
This commit is contained in:
Florent
2024-04-04 16:25:42 +02:00
committed by GitHub
parent dd83b7f678
commit 4a9b6fea9d
12 changed files with 176 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/gen-server/lib/DocWorkerMap.ts
View File

@@ -24,6 +24,9 @@ const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000; // 24 hours
// How long do permits stored in redis last, in milliseconds.
const PERMIT_TTL_MSEC = 1 * 60 * 1000; // 1 minute
// Default doc worker group.
const DEFAULT_GROUP = 'default';
class DummyDocWorkerMap implements IDocWorkerMap {
private _worker?: DocWorkerInfo;
private _available: boolean = false;
@@ -62,6 +65,10 @@ class DummyDocWorkerMap implements IDocWorkerMap {
this._available = available;
}
public async isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean> {
return Promise.resolve(true);
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
// nothing to do
}
@@ -241,7 +248,7 @@ export class DocWorkerMap implements IDocWorkerMap {
try {
// Drop out of available set first.
await this._client.sremAsync('workers-available', workerId);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
const group = await this._client.getAsync(`worker-${workerId}-group`) || DEFAULT_GROUP;
await this._client.sremAsync(`workers-available-${group}`, workerId);
// At this point, this worker should no longer be receiving new doc assignments, though
// clients may still be directed to the worker.
@@ -290,7 +297,7 @@ export class DocWorkerMap implements IDocWorkerMap {
public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
const group = await this._client.getAsync(`worker-${workerId}-group`) || DEFAULT_GROUP;
if (available) {
const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
if (!docWorker) { throw new Error('no doc worker contact info available'); }
@@ -306,6 +313,11 @@ export class DocWorkerMap implements IDocWorkerMap {
}
}
public async isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean> {
const group = workerInfo.group || DEFAULT_GROUP;
return Boolean(await this._client.sismemberAsync(`workers-available-${group}`, workerInfo.id));
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
const op = this._client.multi();
op.del(`doc-${docId}`);
@@ -352,7 +364,7 @@ export class DocWorkerMap implements IDocWorkerMap {
if (docId === 'import') {
const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
try {
const _workerId = await this._client.srandmemberAsync(`workers-available-default`);
const _workerId = await this._client.srandmemberAsync(`workers-available-${DEFAULT_GROUP}`);
if (!_workerId) { throw new Error('no doc worker available'); }
const docWorker = await this._client.hgetallAsync(`worker-${_workerId}`) as DocWorkerInfo|null;
if (!docWorker) { throw new Error('no doc worker contact info available'); }
@@ -383,7 +395,7 @@ export class DocWorkerMap implements IDocWorkerMap {
if (!workerId) {
// Check if document has a preferred worker group set.
const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
const group = await this._client.getAsync(`doc-${docId}-group`) || DEFAULT_GROUP;
// Let's start off by assigning documents to available workers randomly.
// TODO: use a smarter algorithm.

4
app/server/lib/DocWorkerMap.ts
View File

@@ -6,7 +6,7 @@
import { IChecksumStore } from 'app/server/lib/IChecksumStore';
import { IElectionStore } from 'app/server/lib/IElectionStore';
import { IPermitStores } from 'app/server/lib/Permit';
import {RedisClient} from 'redis';
import { RedisClient } from 'redis';
export interface DocWorkerInfo {
id: string;
@@ -57,6 +57,8 @@ export interface IDocWorkerMap extends IPermitStores, IElectionStore, IChecksumS
// release existing assignments.
setWorkerAvailability(workerId: string, available: boolean): Promise<void>;
isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean>;
// Releases doc from worker, freeing it to be assigned elsewhere.
// Assignments should only be released for workers that are now unavailable.
releaseAssignment(workerId: string, docId: string): Promise<void>;

12
app/server/lib/FlexServer.ts
View File

@@ -445,7 +445,8 @@ export class FlexServer implements GristServer {
// /status/hooks allows the tests to wait for them to be ready.
// If db=1 query parameter is included, status will include the status of DB connection.
// If redis=1 query parameter is included, status will include the status of the Redis connection.
// If ready=1 query parameter is included, status will include whether the server is fully ready.
// If docWorkerRegistered=1 query parameter is included, status will include the status of the
// doc worker registration in Redis.
this.app.get('/status(/hooks)?', async (req, res) => {
const checks = new Map<string, Promise<boolean>|boolean>();
const timeout = optIntegerParam(req.query.timeout, 'timeout') || 10_000;
@@ -467,6 +468,15 @@ export class FlexServer implements GristServer {
if (isParameterOn(req.query.redis)) {
checks.set('redis', asyncCheck(this._docWorkerMap.getRedisClient()?.pingAsync()));
}
if (isParameterOn(req.query.docWorkerRegistered) && this.worker) {
// Only check whether the doc worker is registered if we have a worker.
// The Redis client may not be connected, but in this case this has to
// be checked with the 'redis' parameter (the user may want to avoid
// removing workers when connection is unstable).
if (this._docWorkerMap.getRedisClient()?.connected) {
checks.set('docWorkerRegistered', asyncCheck(this._docWorkerMap.isWorkerRegistered(this.worker)));
}
}
if (isParameterOn(req.query.ready)) {
checks.set('ready', this._isReady);
}