Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

Shutdown Doc worker when it is not considered as available in Redis #831 (#856)

Browse Source 
* Shutdown Doc worker when it is not considered as available in Redis
* Use isAffirmative for GRIST_MANAGED_WORKERS
* Upgrade Sinon for the tests
* Run Smoke test with pages in English
* Add logic in /status endpoint
This commit is contained in:
Florent
2024-04-04 16:25:42 +02:00
committed by GitHub
parent dd83b7f678
commit 4a9b6fea9d
12 changed files with 176 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/gen-server/lib/DocWorkerMap.ts
View File

@@ -24,6 +24,9 @@ const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000; // 24 hours
// How long do permits stored in redis last, in milliseconds.
const PERMIT_TTL_MSEC = 1 * 60 * 1000; // 1 minute
// Default doc worker group.
const DEFAULT_GROUP = 'default';
class DummyDocWorkerMap implements IDocWorkerMap {
private _worker?: DocWorkerInfo;
private _available: boolean = false;
@@ -62,6 +65,10 @@ class DummyDocWorkerMap implements IDocWorkerMap {
this._available = available;
}
public async isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean> {
return Promise.resolve(true);
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
// nothing to do
}
@@ -241,7 +248,7 @@ export class DocWorkerMap implements IDocWorkerMap {
try {
// Drop out of available set first.
await this._client.sremAsync('workers-available', workerId);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
const group = await this._client.getAsync(`worker-${workerId}-group`) || DEFAULT_GROUP;
await this._client.sremAsync(`workers-available-${group}`, workerId);
// At this point, this worker should no longer be receiving new doc assignments, though
// clients may still be directed to the worker.
@@ -290,7 +297,7 @@ export class DocWorkerMap implements IDocWorkerMap {
public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
const group = await this._client.getAsync(`worker-${workerId}-group`) || DEFAULT_GROUP;
if (available) {
const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
if (!docWorker) { throw new Error('no doc worker contact info available'); }
@@ -306,6 +313,11 @@ export class DocWorkerMap implements IDocWorkerMap {
}
}
public async isWorkerRegistered(workerInfo: DocWorkerInfo): Promise<boolean> {
const group = workerInfo.group || DEFAULT_GROUP;
return Boolean(await this._client.sismemberAsync(`workers-available-${group}`, workerInfo.id));
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
const op = this._client.multi();
op.del(`doc-${docId}`);
@@ -352,7 +364,7 @@ export class DocWorkerMap implements IDocWorkerMap {
if (docId === 'import') {
const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
try {
const _workerId = await this._client.srandmemberAsync(`workers-available-default`);
const _workerId = await this._client.srandmemberAsync(`workers-available-${DEFAULT_GROUP}`);
if (!_workerId) { throw new Error('no doc worker available'); }
const docWorker = await this._client.hgetallAsync(`worker-${_workerId}`) as DocWorkerInfo|null;
if (!docWorker) { throw new Error('no doc worker contact info available'); }
@@ -383,7 +395,7 @@ export class DocWorkerMap implements IDocWorkerMap {
if (!workerId) {
// Check if document has a preferred worker group set.
const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
const group = await this._client.getAsync(`doc-${docId}-group`) || DEFAULT_GROUP;
// Let's start off by assigning documents to available workers randomly.
// TODO: use a smarter algorithm.