From 68a682f876dd37ca74676d2d81fc475302e2aa18 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 5 Jan 2021 09:35:14 -0500 Subject: [PATCH] (core) add housekeeping endpoints for cleaning doc snapshots+state Summary: This adds endpoints that allow the support user to remove unlisted snapshots for a document, and to remove all action history for a document. This does increase what the support user can do, but not in a way that would be particularly valuable to attack. It would have some destructive value, for removing history (removing unlisted snapshots doesn't impact the user, by contrast). This would simplify some maintenance operations. Test Plan: added test for snapshots; tested states manually Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2699 --- app/gen-server/lib/Housekeeper.ts | 62 +++++++++++++++++++++++++++++++ app/server/lib/FlexServer.ts | 3 +- app/server/mergedServerMain.ts | 4 +- 3 files changed, 65 insertions(+), 4 deletions(-) diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts index 3696eb88..10e836ee 100644 --- a/app/gen-server/lib/Housekeeper.ts +++ b/app/gen-server/lib/Housekeeper.ts @@ -1,12 +1,18 @@ +import { ApiError } from 'app/common/ApiError'; import { Document } from 'app/gen-server/entity/Document'; import { Workspace } from 'app/gen-server/entity/Workspace'; import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager'; import { fromNow } from 'app/gen-server/sqlUtils'; +import { getAuthorizedUserId } from 'app/server/lib/Authorizer'; +import { expressWrap } from 'app/server/lib/expressWrap'; import { GristServer } from 'app/server/lib/GristServer'; import { IElectionStore } from 'app/server/lib/IElectionStore'; import * as log from 'app/server/lib/log'; import { IPermitStore } from 'app/server/lib/Permit'; +import { stringParam } from 'app/server/lib/requestUtils'; +import * as express from 'express'; import fetch from 'node-fetch'; +import * as Fetch from 'node-fetch'; const HOUSEKEEPER_PERIOD_MS = 1 * 60 * 60 * 1000; // operate every 1 hour const AGE_THRESHOLD_OFFSET = '-30 days'; // should be an interval known by postgres + sqlite @@ -115,6 +121,38 @@ export class Housekeeper { } } + public addEndpoints(app: express.Application) { + // Allow support user to perform housekeeping tasks for a specific + // document. The tasks necessarily bypass user access controls. + // As such, it would be best if these endpoints not offer ways to + // read or write the content of a document. + + // Remove unlisted snapshots that are not recorded in inventory. + // Once all such snapshots have been removed, there should be no + // further need for this endpoint. + app.post('/api/housekeeping/docs/:docId/snapshots/clean', this._withSupport(async (docId, headers) => { + const url = await this._server.getHomeUrlByDocId(docId, `/api/docs/${docId}/snapshots/remove`); + return fetch(url, { + method: 'POST', + body: JSON.stringify({ select: 'unlisted' }), + headers, + }); + })); + + // Remove action history from document. This may be of occasional + // use, for allowing support to help users looking to purge some + // information that leaked into document history that they'd + // prefer not be there, until there's an alternative. + app.post('/api/housekeeping/docs/:docId/states/remove', this._withSupport(async (docId, headers) => { + const url = await this._server.getHomeUrlByDocId(docId, `/api/docs/${docId}/states/remove`); + return fetch(url, { + method: 'POST', + body: JSON.stringify({ keep: 1 }), + headers, + }); + })); + } + /** * For test purposes, removes any exclusive lock on housekeeping. */ @@ -159,4 +197,28 @@ export class Housekeeper { private _getThreshold() { return fromNow(this._dbManager.connection.driver.options.type, AGE_THRESHOLD_OFFSET); } + + // Call a document endpoint with a permit, cleaning up after the call. + // Checks that the user is the support user. + private _withSupport(callback: (docId: string, headers: Record) => Promise): express.RequestHandler { + return expressWrap(async (req, res) => { + const userId = getAuthorizedUserId(req); + if (userId !== this._dbManager.getSupportUserId()) { + throw new ApiError('access denied', 403); + } + const docId = stringParam(req.params.docId); + const permitKey = await this._permitStore.setPermit({docId}); + try { + const result = await callback(docId, { + Permit: permitKey, + 'Content-Type': 'application/json', + }); + res.status(result.status); + // Return JSON result, or an empty object if no result provided. + res.json(await result.json().catch(() => ({}))); + } finally { + await this._permitStore.removePermit(permitKey); + } + }); + } } diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 59fd680c..5061e655 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1106,9 +1106,10 @@ export class FlexServer implements GristServer { } public async addHousekeeper() { - if (this._check('housekeeper', 'start', 'homedb', 'map')) { return; } + if (this._check('housekeeper', 'start', 'homedb', 'map', 'json', 'api-mw')) { return; } const store = this._docWorkerMap; this.housekeeper = new Housekeeper(this.dbManager, this, store, store); + this.housekeeper.addEndpoints(this.app); await this.housekeeper.start(); } diff --git a/app/server/mergedServerMain.ts b/app/server/mergedServerMain.ts index 089717a4..cd401813 100644 --- a/app/server/mergedServerMain.ts +++ b/app/server/mergedServerMain.ts @@ -104,7 +104,6 @@ export async function main(port: number, serverTypes: ServerType[], if (includeHome) { if (!includeApp) { server.addUsage(); - await server.addHousekeeper(); } if (!includeDocs) { server.addDocApiForwarder(); @@ -115,9 +114,8 @@ export async function main(port: number, serverTypes: ServerType[], if (!includeApp) { server.addHomeApi(); server.addBillingApi(); - } - if (!includeApp) { server.addNotifier(); + await server.addHousekeeper(); } server.addLoginRoutes(); server.addBillingPages();