(core) Prune snapshots outside the window in product features

Summary:
- Add a method `getSnapshotWindow` to `IInventory` and `DocSnapshotInventory`. It returns a `SnapshotWindow`, which represents a duration of time for which we keep backups for a particular document.
- `DocSnapshotPruner` calls this method and passes the window to `shouldKeepSnapshots` to determine which document versions have fallen outside the window and should be pruned.
- The implementation passed to `DocSnapshotInventory` uses a new method `getDocProduct` in `HomeDBManager` which directly returns the `Product` associated with a document, given only the document ID. Other methods in `HomeDBManager` require passing more information, especially about a user, but `DocSnapshotPruner` only knows about document IDs.

Test Plan: Added a test for `getDocProduct` and a test for `DocSnapshotPruner` where `getSnapshotWindow` is specified.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3322

app/common/Features.ts

@@ -1,3 +1,8 @@
+export interface SnapshotWindow {
+  count: number;
+  unit: 'month' | 'year';
+}
+
 // A product is essentially a list of flags and limits that we may enforce/support.
 export interface Features {
   vanityDomain?: boolean;   // are user-selected domains allowed (unenforced) (default: true)
@@ -35,10 +40,7 @@ export interface Features {
 
   readOnlyDocs?: boolean;   // if set, docs can only be read, not written.
 
-  snapshotWindow?: {        // if set, controls how far back snapshots are kept.
+  snapshotWindow?: SnapshotWindow;  // if set, controls how far back snapshots are kept.
-    count: number;          // TODO: not honored at time of writing.
-    unit: 'month'|'year';
-  };
 
   baseMaxRowsPerDocument?: number;  // If set, establishes a default maximum on the
                                  // number of rows (total) in a single document.

app/gen-server/entity/Product.ts

@@ -1,7 +1,8 @@
 import {Features} from 'app/common/Features';
 import {nativeValues} from 'app/gen-server/lib/values';
 import * as assert from 'assert';
-import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {BaseEntity, Column, Connection, Entity, OneToMany, PrimaryGeneratedColumn} from 'typeorm';
 
 /**
  * A summary of features used in 'starter' plans.
@@ -149,6 +150,9 @@ export class Product extends BaseEntity {
 
   @Column({type: nativeValues.jsonEntityType})
   public features: Features;
+
+  @OneToMany(type => BillingAccount, account => account.product)
+  public accounts: BillingAccount[];
 }
 
 /**

app/gen-server/lib/HomeDBManager.ts

@@ -2325,6 +2325,18 @@ export class HomeDBManager extends EventEmitter {
     });
   }
 
+  public async getDocProduct(docId: string): Promise<Product | undefined> {
+    return await this._connection.createQueryBuilder()
+      .select('product')
+      .from(Product, 'product')
+      .leftJoinAndSelect('product.accounts', 'account')
+      .leftJoinAndSelect('account.orgs', 'org')
+      .leftJoinAndSelect('org.workspaces', 'workspace')
+      .leftJoinAndSelect('workspace.docs', 'doc')
+      .where('doc.id = :docId', {docId})
+      .getOne();
+  }
+
   /**
    * Get the anonymous user, as a constructed object rather than a database lookup.
    */

app/server/lib/DocSnapshots.ts

@@ -1,7 +1,8 @@
-import { ObjSnapshotWithMetadata } from 'app/common/DocSnapshot';
+import {ObjSnapshotWithMetadata} from 'app/common/DocSnapshot';
-import { KeyedOps } from 'app/common/KeyedOps';
+import {SnapshotWindow} from 'app/common/Features';
-import { KeyedMutex } from 'app/common/KeyedMutex';
+import {KeyedMutex} from 'app/common/KeyedMutex';
-import { ExternalStorage } from 'app/server/lib/ExternalStorage';
+import {KeyedOps} from 'app/common/KeyedOps';
+import {ExternalStorage} from 'app/server/lib/ExternalStorage';
 import * as log from 'app/server/lib/log';
 import * as fse from 'fs-extra';
 import * as moment from 'moment-timezone';
@@ -10,6 +11,7 @@ import * as moment from 'moment-timezone';
  * A subset of the ExternalStorage interface, focusing on maintaining a list of versions.
  */
 export interface IInventory {
+  getSnapshotWindow?: (key: string) => Promise<SnapshotWindow|undefined>;
   versions(key: string): Promise<ObjSnapshotWithMetadata[]>;
   remove(key: string, snapshotIds: string[]): Promise<void>;
 }
@@ -59,8 +61,9 @@ export class DocSnapshotPruner {
 
   // Get all snapshots for a document, and whether they should be kept or pruned.
   public async classify(key: string): Promise<Array<{snapshot: ObjSnapshotWithMetadata, keep: boolean}>> {
+    const snapshotWindow = await this._ext.getSnapshotWindow?.(key);
     const versions = await this._ext.versions(key);
-    return shouldKeepSnapshots(versions).map((keep, index) => ({keep, snapshot: versions[index]}));
+    return shouldKeepSnapshots(versions, snapshotWindow).map((keep, index) => ({keep, snapshot: versions[index]}));
   }
 
   // Prune the specified document immediately.  If no snapshotIds are provided, they
@@ -107,8 +110,12 @@ export class DocSnapshotInventory implements IInventory {
    * Expects to be given the store for documents, a store for metadata, and a method
    * for naming cache files on the local filesystem.  The stores should be consistent.
    */
-  constructor(private _doc: ExternalStorage, private _meta: ExternalStorage,
+  constructor(
-              private _getFilename: (key: string) => Promise<string>) {}
+    private _doc: ExternalStorage,
+    private _meta: ExternalStorage,
+    private _getFilename: (key: string) => Promise<string>,
+    public getSnapshotWindow: (key: string) => Promise<SnapshotWindow|undefined>,
+  ) {}
 
   /**
    * Start keeping inventory for a new document.
@@ -312,7 +319,7 @@ export class DocSnapshotInventory implements IInventory {
  *   - Anything with a label, for up to 32 days before the current version.
  * Calculations done in UTC, Gregorian calendar, ISO weeks (week starts with Monday).
  */
-export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[]): boolean[] {
+export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[], snapshotWindow?: SnapshotWindow): boolean[] {
   // Get current version
   const current = snapshots[0];
   if (!current) { return []; }
@@ -334,15 +341,26 @@ export function shouldKeepSnapshots(snapshots: ObjSnapshotWithMetadata[]): boole
   // For each snapshot starting with newest, check if it is worth saving by comparing
   // it with the last saved snapshot based on hour, day, week, month, year
   return snapshots.map((snapshot, index) => {
-    let keep = index < 5;   // Keep 5 most recent versions
+    // Just to make extra sure we don't delete everything
+    if (index === 0) {
+      return true;
+    }
+
     const date = moment.tz(snapshot.lastModified, tz);
+
+    // Limit snapshots to the given window corresponding to what the user has paid for
+    if (snapshotWindow && start.diff(date, snapshotWindow.unit, true) > snapshotWindow.count) {
+      return false;
+    }
+
+    let keep = index < 5;   // Keep 5 most recent versions
     for (const bucket of buckets) {
       if (updateAndCheckRange(date, bucket)) { keep = true; }
     }
     // Preserve recent labelled snapshots in a naive and limited way.  No doubt this will
     // be elaborated on if we make this a user-facing feature.
     if (snapshot.metadata?.label &&
-        start.diff(moment.tz(snapshot.lastModified, tz), 'days') < 32) { keep = true; }
+        start.diff(date, 'days') < 32) { keep = true; }
     return keep;
   });
 }

app/server/lib/HostedStorageManager.ts

@@ -162,12 +162,19 @@ export class HostedStorageManager implements IDocStorageManager {
                                                           this._latestMetaVersions,
                                                           options);
 
-      this._inventory = new DocSnapshotInventory(this._ext, this._extMeta,
+      this._inventory = new DocSnapshotInventory(
-                                                 async docId => {
+        this._ext,
-                                                   const dir = this.getAssetPath(docId);
+        this._extMeta,
-                                                   await fse.mkdirp(dir);
+        async docId => {
-                                                   return path.join(dir, 'meta.json');
+          const dir = this.getAssetPath(docId);
-                                                 });
+          await fse.mkdirp(dir);
+          return path.join(dir, 'meta.json');
+        },
+        async docId => {
+          const product = await dbManager.getDocProduct(docId);
+          return product?.features.snapshotWindow;
+        },
+      );
 
       // The pruner could use an inconsistent store without any real loss overall,
       // but tests are easier if it is consistent.