(core) Grace period and delete-only mode when exceeding row limit

Summary:
Builds upon https://phab.getgrist.com/D3328

- Add HomeDB column `Document.gracePeriodStart`
- When the row count moves above the limit, set it to the current date. When it moves below, set it to null.
- Add DataLimitStatus type indicating if the document is approaching the limit, is in a grace period, or is in delete only mode if the grace period started at least 14 days ago. Compute it in ActiveDoc and send it to client when opening.
- Only allow certain user actions when in delete-only mode.

Follow-up tasks related to this diff:

- When DataLimitStatus in the client is non-empty, show a banner to the appropriate users.
- Only send DataLimitStatus to users with the appropriate access. There's no risk landing this now since real users will only see null until free team sites are released.
- Update DataLimitStatus immediately in the client when it changes, e.g. when user actions are applied or the product is changed. Right now it's only sent when the document loads.
- Update row limit, grace period start, and data limit status in ActiveDoc when the product changes, i.e. the user upgrades/downgrades.
- Account for data size when computing data limit status, not just row counts.

See also the tasks mentioned in https://phab.getgrist.com/D3331

Test Plan: Extended FreeTeam nbrowser test, testing the 4 statuses.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3331

app/server/lib/ActiveDoc.ts

@@ -10,6 +10,7 @@ import {ActionSummary} from "app/common/ActionSummary";
 import {
   ApplyUAOptions,
   ApplyUAResult,
+  DataLimitStatus,
   DataSourceTransformed,
   ForkResult,
   ImportOptions,
@@ -34,6 +35,7 @@ import {DocData} from 'app/common/DocData';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentSettings} from 'app/common/DocumentSettings';
 import {normalizeEmail} from 'app/common/emails';
+import {Features} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
 import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
@@ -51,6 +53,7 @@ import {Authorizer} from 'app/server/lib/Authorizer';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
 import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager';
+import {ICreateActiveDocOptions} from 'app/server/lib/ICreate';
 import {makeForkIds} from 'app/server/lib/idUtils';
 import {GRIST_DOC_SQL, GRIST_DOC_WITH_TABLE1_SQL} from 'app/server/lib/initialDocSql';
 import {ISandbox} from 'app/server/lib/ISandbox';
@@ -160,18 +163,21 @@ export class ActiveDoc extends EventEmitter {
   private _lastMemoryMeasurement: number = 0;   // Timestamp when memory was last measured.
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
   private _rowCount?: number;
+  private _productFeatures?: Features;
+  private _gracePeriodStart: Date|null = null;
 
   // Timer for shutting down the ActiveDoc a bit after all clients are gone.
   private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000);
   private _recoveryMode: boolean = false;
   private _shuttingDown: boolean = false;
 
-  constructor(docManager: DocManager, docName: string, private _options?: {
-    safeMode?: boolean,
-    docUrl?: string
-  }) {
+  constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
     super();
     if (_options?.safeMode) { this._recoveryMode = true; }
+    if (_options?.doc) {
+      this._productFeatures = _options.doc.workspace.org.billingAccount?.product.features;
+      this._gracePeriodStart = _options.doc.gracePeriodStart;
+    }
     this._docManager = docManager;
     this._docName = docName;
     this.docStorage = new DocStorage(docManager.storageManager, docName);
@@ -219,6 +225,24 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
+  public async getDataLimitStatus(): Promise<DataLimitStatus> {
+    if (this._rowLimit && this._rowCount) {
+      const ratio = this._rowCount / this._rowLimit;
+      if (ratio > 1) {
+        const start = this._gracePeriodStart;
+        const days = this._productFeatures?.gracePeriodDays;
+        if (start && days && moment().diff(moment(start), 'days') >= days) {
+          return 'deleteOnly';
+        } else {
+          return 'gracePeriod';
+        }
+      } else if (ratio > 0.9) {
+        return 'approachingLimit';
+      }
+    }
+    return null;
+  }
+
   public async getUserOverride(docSession: OptDocSession) {
     return this._granularAccess.getUserOverride(docSession);
   }
@@ -900,6 +924,17 @@ export class ActiveDoc extends EventEmitter {
     // Be careful not to sneak into user action queue before Calculate action, otherwise
     // there'll be a deadlock.
     await this.waitForInitialization();
+
+    if (
+      await this.getDataLimitStatus() === "deleteOnly" &&
+      !actions.every(action => [
+          'RemoveTable', 'RemoveColumn', 'RemoveRecord', 'BulkRemoveRecord',
+          'RemoveViewSection', 'RemoveView', 'ApplyUndoActions',
+        ].includes(action[0] as string))
+    ) {
+      throw new Error("Document is in delete-only mode");
+    }
+
     // Granular access control implemented in _applyUserActions.
     return await this._applyUserActions(docSession, actions, options);
   }
@@ -1223,7 +1258,7 @@ export class ActiveDoc extends EventEmitter {
         ...this.getLogMeta(docSession),
         rowCount: sandboxActionBundle.rowCount
       });
-      this._rowCount = sandboxActionBundle.rowCount;
+      await this._updateRowCount(sandboxActionBundle.rowCount);
       await this._reportDataEngineMemory();
     } else {
       // Create default SandboxActionBundle to use if the data engine is not called.
@@ -1451,6 +1486,25 @@ export class ActiveDoc extends EventEmitter {
     };
   }
 
+  private get _rowLimit(): number | undefined {
+    return this._productFeatures?.baseMaxRowsPerDocument;
+  }
+
+  private async _updateGracePeriodStart(gracePeriodStart: Date | null) {
+    this._gracePeriodStart = gracePeriodStart;
+    await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart);
+  }
+
+  private async _updateRowCount(rowCount: number) {
+    this._rowCount = rowCount;
+    const exceedingRowLimit = this._rowLimit && rowCount > this._rowLimit;
+    if (exceedingRowLimit && !this._gracePeriodStart) {
+      await this._updateGracePeriodStart(new Date());
+    } else if (!exceedingRowLimit && this._gracePeriodStart) {
+      await this._updateGracePeriodStart(null);
+    }
+  }
+
   /**
    * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply.
    */

app/server/lib/DocManager.ts

@@ -313,11 +313,12 @@ export class DocManager extends EventEmitter {
         }
       }
 
-      const [metaTables, recentActions, userOverride, rowCount] = await Promise.all([
+      const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([
         activeDoc.fetchMetaTables(docSession),
         activeDoc.getRecentMinimalActions(docSession),
         activeDoc.getUserOverride(docSession),
         activeDoc.getRowCount(docSession),
+        activeDoc.getDataLimitStatus(),
       ]);
 
       const result = {
@@ -328,6 +329,7 @@ export class DocManager extends EventEmitter {
         recoveryMode: activeDoc.recoveryMode,
         userOverride,
         rowCount,
+        dataLimitStatus,
       } as OpenLocalDocResult;
 
       if (!activeDoc.muted) {
@@ -510,7 +512,7 @@ export class DocManager extends EventEmitter {
     const doc = await this._getDoc(docSession, docName);
     // Get URL for document for use with SELF_HYPERLINK().
     const docUrl = doc && await this._getDocUrl(doc);
-    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode});
+    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode, doc});
   }
 
   /**

app/server/lib/ICreate.ts

@@ -1,3 +1,4 @@
+import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DocManager} from 'app/server/lib/DocManager';
@@ -32,4 +33,5 @@ export interface ICreate {
 export interface ICreateActiveDocOptions {
   safeMode?: boolean;
   docUrl?: string;
+  doc?: Document;
 }

app/server/lib/requestUtils.ts

@@ -18,9 +18,10 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
   parseInt(process.env.GRIST_TEST_HTTPS_OFFSET, 10) : undefined;
 
 // Database fields that we permit in entities but don't want to cross the api.
-const INTERNAL_FIELDS = new Set(['apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId',
-                                 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId',
-                                 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin']);
+const INTERNAL_FIELDS = new Set([
+  'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId',
+  'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
+]);
 
 /**
  * Adapt a home-server or doc-worker URL to match the hostname in the request URL. For custom