diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts new file mode 100644 index 00000000..3696eb88 --- /dev/null +++ b/app/gen-server/lib/Housekeeper.ts @@ -0,0 +1,162 @@ +import { Document } from 'app/gen-server/entity/Document'; +import { Workspace } from 'app/gen-server/entity/Workspace'; +import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager'; +import { fromNow } from 'app/gen-server/sqlUtils'; +import { GristServer } from 'app/server/lib/GristServer'; +import { IElectionStore } from 'app/server/lib/IElectionStore'; +import * as log from 'app/server/lib/log'; +import { IPermitStore } from 'app/server/lib/Permit'; +import fetch from 'node-fetch'; + +const HOUSEKEEPER_PERIOD_MS = 1 * 60 * 60 * 1000; // operate every 1 hour +const AGE_THRESHOLD_OFFSET = '-30 days'; // should be an interval known by postgres + sqlite + +/** + * Take care of periodic tasks: + * + * - deleting old soft-deleted documents + * - deleting old soft-deleted workspaces + * + * Call start(), keep the object around, and call stop() when shutting down. + * + * Some care is taken to elect a single server to do the housekeeping, so if there are + * multiple home servers, there will be no competition or duplication of effort. + */ +export class Housekeeper { + private _interval?: NodeJS.Timeout; + private _electionKey?: string; + + public constructor(private _dbManager: HomeDBManager, private _server: GristServer, + private _permitStore: IPermitStore, private _electionStore: IElectionStore) { + } + + /** + * Start a ticker to launch housekeeping tasks from time to time. + */ + public async start() { + await this.stop(); + this._interval = setInterval(() => this.deleteTrashExclusively().catch(log.warn.bind(log)), HOUSEKEEPER_PERIOD_MS); + } + + /** + * Stop scheduling housekeeping tasks. Note: doesn't wait for any housekeeping task in progress. + */ + public async stop() { + if (this._interval) { + clearInterval(this._interval); + this._interval = undefined; + } + } + + /** + * Deletes old trash if no other server is working on it or worked on it recently. + */ + public async deleteTrashExclusively(): Promise { + const electionKey = await this._electionStore.getElection('trash', HOUSEKEEPER_PERIOD_MS / 2.0); + if (!electionKey) { + log.info('Skipping deleteTrash since another server is working on it or worked on it recently'); + return false; + } + this._electionKey = electionKey; + await this.deleteTrash(); + return true; + } + + /** + * Deletes old trash regardless of what other servers may be doing. + */ + public async deleteTrash() { + // Delete old soft-deleted docs + const docs = await this._getDocsToDelete(); + for (const doc of docs) { + // Last minute check - is the doc really soft-deleted? + if (doc.removedAt === null && doc.workspace.removedAt === null) { + throw new Error(`attempted to hard-delete a document that was not soft-deleted: ${doc.id}`); + } + // In general, documents can only be manipulated with the coordination of the + // document worker to which they are assigned. For an old soft-deleted doc, + // we could probably get away with ensuring the document is closed/unloaded + // and then deleting it without ceremony. But, for consistency, and because + // it will be useful for other purposes, we work through the api using special + // temporary permits. + const permitKey = await this._permitStore.setPermit({docId: doc.id}); + try { + const result = await fetch(await this._server.getHomeUrlByDocId(doc.id, `/api/docs/${doc.id}`), { + method: 'DELETE', + headers: { + Permit: permitKey + } + }); + if (result.status !== 200) { + log.error(`failed to delete document ${doc.id}: error status ${result.status}`); + } + } finally { + await this._permitStore.removePermit(permitKey); + } + } + + // Delete old soft-deleted workspaces + const workspaces = await this._getWorkspacesToDelete(); + // Note: there's a small chance a workspace could be undeleted right under the wire, + // and a document added, in which case the method we call here would not yet clean + // up the docs in s3. TODO: deal with this. + for (const workspace of workspaces) { + // Last minute check - is the workspace really soft-deleted? + if (workspace.removedAt === null) { + throw new Error(`attempted to hard-delete a workspace that was not soft-deleted: ${workspace.id}`); + } + const scope: Scope = { + userId: this._dbManager.getPreviewerUserId(), + specialPermit: { + workspaceId: workspace.id + } + }; + await this._dbManager.deleteWorkspace(scope, workspace.id); + } + } + + /** + * For test purposes, removes any exclusive lock on housekeeping. + */ + public async testClearExclusivity(): Promise { + if (this._electionKey) { + await this._electionStore.removeElection('trash', this._electionKey); + this._electionKey = undefined; + } + } + + private async _getDocsToDelete() { + const docs = await this._dbManager.connection.createQueryBuilder() + .select('docs') + .from(Document, 'docs') + .leftJoinAndSelect('docs.workspace', 'workspaces') + .where(`COALESCE(docs.removed_at, workspaces.removed_at) <= ${this._getThreshold()}`) + // the following has no effect (since null <= date is false) but added for clarity + .andWhere('COALESCE(docs.removed_at, workspaces.removed_at) IS NOT NULL') + .getMany(); + return docs; + } + + private async _getWorkspacesToDelete() { + const docs = await this._dbManager.connection.createQueryBuilder() + .select('workspaces') + .from(Workspace, 'workspaces') + .leftJoin('workspaces.docs', 'docs') + .where(`workspaces.removed_at <= ${this._getThreshold()}`) + // the following has no effect (since null <= date is false) but added for clarity + .andWhere('workspaces.removed_at IS NOT NULL') + // wait for workspace to be empty + .andWhere('docs.id IS NULL') + .getMany(); + return docs; + } + + /** + * TypeORM isn't very adept at handling date representation for + * comparisons, so we construct the threshold date in SQL so that we + * don't have to deal with its caprices. + */ + private _getThreshold() { + return fromNow(this._dbManager.connection.driver.options.type, AGE_THRESHOLD_OFFSET); + } +} diff --git a/app/server/lib/IElectionStore.ts b/app/server/lib/IElectionStore.ts new file mode 100644 index 00000000..b3fe7acc --- /dev/null +++ b/app/server/lib/IElectionStore.ts @@ -0,0 +1,24 @@ +/** + * Get a revokable named exclusive lock with a TTL. This is convenient for housekeeping + * tasks, which can be done by any server, but should preferably be only done by one + * at a time. + */ +export interface IElectionStore { + /** + * Try to get a lock called for a specified duration. If the named lock + * has already been taken, null is returned, otherwise a secret is returned. + * The secret can be used to remove the lock before the duration has expired. + */ + getElection(name: string, durationInMs: number): Promise; + + /** + * Remove a named lock, presenting the secret returned by getElection() as + * a cross-check. + */ + removeElection(name: string, electionKey: string): Promise; + + /** + * Close down access to the store. + */ + close(): Promise; +} diff --git a/app/server/lib/Permit.ts b/app/server/lib/Permit.ts new file mode 100644 index 00000000..585ea34b --- /dev/null +++ b/app/server/lib/Permit.ts @@ -0,0 +1,58 @@ +/** + * An exceptional grant of rights on a resource, for when work needs to be + * initiated by Grist systems rather than a user. Cases where this may happen: + * + * - Deletion of documents and workspaces in the trash + * + * Permits are stored in redis (or, in a single-process dev environment, in memory) + * as json, in keys that expire within minutes. The keys should be effectively + * unguessable. + * + * To use a permit: + * + * - Prepare a Permit object that includes the id of the document or + * workspace to be operated on. + * + * - It the operation you care about involves the database, check + * that "allowSpecialPermit" is enabled for it in HomeDBManager + * (currently only deletion of docs/workspaces has this enabled). + * + * - Save the permit in the permit store, with setPermit, noting its + * generated key. + * + * - Call the API with a "Permit: " header. + * + * - Optionally, remove the permit with removePermit(). + */ +export interface Permit { + docId?: string; + workspaceId?: number; + org?: string|number; +} + +/* A store of permits */ +export interface IPermitStore { + + // Store a permit, and return the key it is stored in. + // Permits are transient, and will expire. + setPermit(permit: Permit): Promise; + + // Get any permit associated with the given key, or null if none. + getPermit(permitKey: string): Promise; + + // Remove any permit associated with the given key. + removePermit(permitKey: string): Promise; + + // Close down the permit store. + close(): Promise; +} + +// Create a well formatted permit key from a seed string. +export function formatPermitKey(seed: string) { + return `permit-${seed}`; +} + +// Check that permit key is well formatted. +export function checkPermitKey(key: string): boolean { + return key.startsWith('permit-'); +}