From ec8460b772cdbc48f49e68393af186047553ed68 Mon Sep 17 00:00:00 2001 From: Alex Hall Date: Tue, 15 Mar 2022 12:45:20 +0200 Subject: [PATCH] (core) Prune snapshots outside the window in product features Summary: - Add a method `getSnapshotWindow` to `IInventory` and `DocSnapshotInventory`. It returns a `SnapshotWindow`, which represents a duration of time for which we keep backups for a particular document. - `DocSnapshotPruner` calls this method and passes the window to `shouldKeepSnapshots` to determine which document versions have fallen outside the window and should be pruned. - The implementation passed to `DocSnapshotInventory` uses a new method `getDocProduct` in `HomeDBManager` which directly returns the `Product` associated with a document, given only the document ID. Other methods in `HomeDBManager` require passing more information, especially about a user, but `DocSnapshotPruner` only knows about document IDs. Test Plan: Added a test for `getDocProduct` and a test for `DocSnapshotPruner` where `getSnapshotWindow` is specified. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3322 --- app/common/Features.ts | 10 ++++--- app/gen-server/entity/Product.ts | 6 +++- app/gen-server/lib/HomeDBManager.ts | 12 ++++++++ app/server/lib/DocSnapshots.ts | 38 +++++++++++++++++++------- app/server/lib/HostedStorageManager.ts | 19 +++++++++---- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/app/common/Features.ts b/app/common/Features.ts index 9c0cadf6..21577386 100644 --- a/app/common/Features.ts +++ b/app/common/Features.ts @@ -1,3 +1,8 @@ +export interface SnapshotWindow { + count: number; + unit: 'month' | 'year'; +} + // A product is essentially a list of flags and limits that we may enforce/support. export interface Features { vanityDomain?: boolean; // are user-selected domains allowed (unenforced) (default: true) @@ -35,10 +40,7 @@ export interface Features { readOnlyDocs?: boolean; // if set, docs can only be read, not written. - snapshotWindow?: { // if set, controls how far back snapshots are kept. - count: number; // TODO: not honored at time of writing. - unit: 'month'|'year'; - }; + snapshotWindow?: SnapshotWindow; // if set, controls how far back snapshots are kept. baseMaxRowsPerDocument?: number; // If set, establishes a default maximum on the // number of rows (total) in a single document. diff --git a/app/gen-server/entity/Product.ts b/app/gen-server/entity/Product.ts index f86c35e2..3c7b35f9 100644 --- a/app/gen-server/entity/Product.ts +++ b/app/gen-server/entity/Product.ts @@ -1,7 +1,8 @@ import {Features} from 'app/common/Features'; import {nativeValues} from 'app/gen-server/lib/values'; import * as assert from 'assert'; -import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm'; +import {BillingAccount} from 'app/gen-server/entity/BillingAccount'; +import {BaseEntity, Column, Connection, Entity, OneToMany, PrimaryGeneratedColumn} from 'typeorm'; /** * A summary of features used in 'starter' plans. @@ -149,6 +150,9 @@ export class Product extends BaseEntity { @Column({type: nativeValues.jsonEntityType}) public features: Features; + + @OneToMany(type => BillingAccount, account => account.product) + public accounts: BillingAccount[]; } /** diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index f3f7cf64..a6fd5722 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -2325,6 +2325,18 @@ export class HomeDBManager extends EventEmitter { }); } + public async getDocProduct(docId: string): Promise { + return await this._connection.createQueryBuilder() + .select('product') + .from(Product, 'product') + .leftJoinAndSelect('product.accounts', 'account') + .leftJoinAndSelect('account.orgs', 'org') + .leftJoinAndSelect('org.workspaces', 'workspace') + .leftJoinAndSelect('workspace.docs', 'doc') + .where('doc.id = :docId', {docId}) + .getOne(); + } + /** * Get the anonymous user, as a constructed object rather than a database lookup. */ diff --git a/app/server/lib/DocSnapshots.ts b/app/server/lib/DocSnapshots.ts index 5aeacf83..1fc77ed2 100644 --- a/app/server/lib/DocSnapshots.ts +++ b/app/server/lib/DocSnapshots.ts @@ -1,7 +1,8 @@ -import { ObjSnapshotWithMetadata } from 'app/common/DocSnapshot'; -import { KeyedOps } from 'app/common/KeyedOps'; -import { KeyedMutex } from 'app/common/KeyedMutex'; -import { ExternalStorage } from 'app/server/lib/ExternalStorage'; +import {ObjSnapshotWithMetadata} from 'app/common/DocSnapshot'; +import {SnapshotWindow} from 'app/common/Features'; +import {KeyedMutex} from 'app/common/KeyedMutex'; +import {KeyedOps} from 'app/common/KeyedOps'; +import {ExternalStorage} from 'app/server/lib/ExternalStorage'; import * as log from 'app/server/lib/log'; import * as fse from 'fs-extra'; import * as moment from 'moment-timezone'; @@ -10,6 +11,7 @@ import * as moment from 'moment-timezone'; * A subset of the ExternalStorage interface, focusing on maintaining a list of versions. */ export interface IInventory { + getSnapshotWindow?: (key: string) => Promise; versions(key: string): Promise; remove(key: string, snapshotIds: string[]): Promise; } @@ -59,8 +61,9 @@ export class DocSnapshotPruner { // Get all snapshots for a document, and whether they should be kept or pruned. public async classify(key: string): Promise> { + const snapshotWindow = await this._ext.getSnapshotWindow?.(key); const versions = await this._ext.versions(key); - return shouldKeepSnapshots(versions).map((keep, index) => ({keep, snapshot: versions[index]})); + return shouldKeepSnapshots(versions, snapshotWindow).map((keep, index) => ({keep, snapshot: versions[index]})); } // Prune the specified document immediately. If no snapshotIds are provided, they @@ -107,8 +110,12 @@ export class DocSnapshotInventory implements IInventory { * Expects to be given the store for documents, a store for metadata, and a method * for naming cache files on the local filesystem. The stores should be consistent. */ - constructor(private _doc: ExternalStorage, private _meta: ExternalStorage, - private _getFilename: (key: string) => Promise) {} + constructor( + private _doc: ExternalStorage, + private _meta: ExternalStorage, + private _getFilename: (key: string) => Promise, + public getSnapshotWindow: (key: string) => Promise, + ) {} /** * Start keeping inventory for a new document. @@ -312,7 +319,7 @@ export class DocSnapshotInventory implements IInventory { * - Anything with a label, for up to 32 days before the current version. * Calculations done in UTC, Gregorian calendar, ISO weeks (week starts with Monday). */ -export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[]): boolean[] { +export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[], snapshotWindow?: SnapshotWindow): boolean[] { // Get current version const current = snapshots[0]; if (!current) { return []; } @@ -334,15 +341,26 @@ export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[]): boole // For each snapshot starting with newest, check if it is worth saving by comparing // it with the last saved snapshot based on hour, day, week, month, year return snapshots.map((snapshot, index) => { - let keep = index < 5; // Keep 5 most recent versions + // Just to make extra sure we don't delete everything + if (index === 0) { + return true; + } + const date = moment.tz(snapshot.lastModified, tz); + + // Limit snapshots to the given window corresponding to what the user has paid for + if (snapshotWindow && start.diff(date, snapshotWindow.unit, true) > snapshotWindow.count) { + return false; + } + + let keep = index < 5; // Keep 5 most recent versions for (const bucket of buckets) { if (updateAndCheckRange(date, bucket)) { keep = true; } } // Preserve recent labelled snapshots in a naive and limited way. No doubt this will // be elaborated on if we make this a user-facing feature. if (snapshot.metadata?.label && - start.diff(moment.tz(snapshot.lastModified, tz), 'days') < 32) { keep = true; } + start.diff(date, 'days') < 32) { keep = true; } return keep; }); } diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts index 2f3e3f88..18efeb6b 100644 --- a/app/server/lib/HostedStorageManager.ts +++ b/app/server/lib/HostedStorageManager.ts @@ -162,12 +162,19 @@ export class HostedStorageManager implements IDocStorageManager { this._latestMetaVersions, options); - this._inventory = new DocSnapshotInventory(this._ext, this._extMeta, - async docId => { - const dir = this.getAssetPath(docId); - await fse.mkdirp(dir); - return path.join(dir, 'meta.json'); - }); + this._inventory = new DocSnapshotInventory( + this._ext, + this._extMeta, + async docId => { + const dir = this.getAssetPath(docId); + await fse.mkdirp(dir); + return path.join(dir, 'meta.json'); + }, + async docId => { + const product = await dbManager.getDocProduct(docId); + return product?.features.snapshotWindow; + }, + ); // The pruner could use an inconsistent store without any real loss overall, // but tests are easier if it is consistent.