2020-07-21 13:20:51 +00:00
|
|
|
/**
|
|
|
|
* Defines the IDocWorkerMap interface we need to assign a DocWorker to a doc, and to look it up.
|
|
|
|
* TODO This is not yet implemented, there is only a hard-coded stub.
|
|
|
|
*/
|
|
|
|
|
2020-10-30 16:53:23 +00:00
|
|
|
import { IChecksumStore } from 'app/server/lib/IChecksumStore';
|
2020-07-21 13:20:51 +00:00
|
|
|
import { IElectionStore } from 'app/server/lib/IElectionStore';
|
2021-08-16 15:11:17 +00:00
|
|
|
import { IPermitStores } from 'app/server/lib/Permit';
|
2020-07-21 13:20:51 +00:00
|
|
|
|
|
|
|
export interface DocWorkerInfo {
|
|
|
|
id: string;
|
|
|
|
|
|
|
|
// The public base URL for the docWorker, which tells the browser how to connect to it. E.g.
|
|
|
|
// https://docworker-17.getgrist.com/ or http://localhost:8080/v/gtag/
|
|
|
|
publicUrl: string;
|
|
|
|
|
|
|
|
// The internal base URL for the docWorker.
|
|
|
|
internalUrl: string;
|
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group
Summary:
In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that.
* Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them.
* These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`.
* The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group.
* An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments.
The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that.
For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical:
* Set prod2 tag in docker to desired codebase for that document
* Set desired_count for grist-docs-prod2 service to non-zero
* Set doc-<docid>-group for that doc in redis to secondary
* Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2
(If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff).
Test Plan: added tests
Reviewers: dsagal
Reviewed By: dsagal
Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
|
|
|
|
|
|
|
// If set, worker should accept work only for this named group.
|
|
|
|
group?: string;
|
2020-07-21 13:20:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
export interface DocStatus {
|
|
|
|
// MD5 hash of the SQLite file for this document as stored on S3. We use MD5 because it is
|
|
|
|
// automatically computed by S3 (except for multipart uploads). Null indicates a new file.
|
|
|
|
docMD5: string|null;
|
|
|
|
|
|
|
|
// DocWorker most recently, or currently, responsible for the file.
|
|
|
|
docWorker: DocWorkerInfo;
|
|
|
|
|
|
|
|
// Whether the file is currently open on this DocWorker.
|
|
|
|
isActive: boolean;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Assignment of documents to workers, and other storage related to distributed work.
|
|
|
|
*/
|
2021-08-16 15:11:17 +00:00
|
|
|
export interface IDocWorkerMap extends IPermitStores, IElectionStore, IChecksumStore {
|
2020-07-21 13:20:51 +00:00
|
|
|
// Looks up which DocWorker is responsible for this docId.
|
|
|
|
getDocWorker(docId: string): Promise<DocStatus|null>;
|
|
|
|
|
|
|
|
// Assigns a DocWorker to this docId if one is not yet assigned.
|
|
|
|
assignDocWorker(docId: string): Promise<DocStatus>;
|
|
|
|
|
|
|
|
// Assigns a particular DocWorker to this docId if one is not yet assigned.
|
|
|
|
getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus>;
|
|
|
|
|
|
|
|
updateDocStatus(docId: string, checksum: string): Promise<void>;
|
|
|
|
|
|
|
|
addWorker(info: DocWorkerInfo): Promise<void>;
|
|
|
|
|
|
|
|
removeWorker(workerId: string): Promise<void>;
|
|
|
|
|
|
|
|
// Set whether worker is accepting new assignments. This does not automatically
|
|
|
|
// release existing assignments.
|
|
|
|
setWorkerAvailability(workerId: string, available: boolean): Promise<void>;
|
|
|
|
|
|
|
|
// Releases doc from worker, freeing it to be assigned elsewhere.
|
2022-02-19 09:46:49 +00:00
|
|
|
// Assignments should only be released for workers that are now unavailable.
|
2020-07-21 13:20:51 +00:00
|
|
|
releaseAssignment(workerId: string, docId: string): Promise<void>;
|
|
|
|
|
|
|
|
// Get all assignments for a worker. Should only be queried for a worker that
|
|
|
|
// is currently unavailable.
|
|
|
|
getAssignments(workerId: string): Promise<string[]>;
|
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group
Summary:
In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that.
* Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them.
* These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`.
* The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group.
* An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments.
The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that.
For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical:
* Set prod2 tag in docker to desired codebase for that document
* Set desired_count for grist-docs-prod2 service to non-zero
* Set doc-<docid>-group for that doc in redis to secondary
* Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2
(If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff).
Test Plan: added tests
Reviewers: dsagal
Reviewed By: dsagal
Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
|
|
|
|
|
|
|
getWorkerGroup(workerId: string): Promise<string|null>;
|
|
|
|
getDocGroup(docId: string): Promise<string|null>;
|
2022-03-21 20:22:35 +00:00
|
|
|
|
|
|
|
incrementDocApiUsage(key: string): Promise<number|null>;
|
2020-07-21 13:20:51 +00:00
|
|
|
}
|