(core) Grace period and delete-only mode when exceeding row limit

Summary:
Builds upon https://phab.getgrist.com/D3328

- Add HomeDB column `Document.gracePeriodStart`
- When the row count moves above the limit, set it to the current date. When it moves below, set it to null.
- Add DataLimitStatus type indicating if the document is approaching the limit, is in a grace period, or is in delete only mode if the grace period started at least 14 days ago. Compute it in ActiveDoc and send it to client when opening.
- Only allow certain user actions when in delete-only mode.

Follow-up tasks related to this diff:

- When DataLimitStatus in the client is non-empty, show a banner to the appropriate users.
- Only send DataLimitStatus to users with the appropriate access. There's no risk landing this now since real users will only see null until free team sites are released.
- Update DataLimitStatus immediately in the client when it changes, e.g. when user actions are applied or the product is changed. Right now it's only sent when the document loads.
- Update row limit, grace period start, and data limit status in ActiveDoc when the product changes, i.e. the user upgrades/downgrades.
- Account for data size when computing data limit status, not just row counts.

See also the tasks mentioned in https://phab.getgrist.com/D3331

Test Plan: Extended FreeTeam nbrowser test, testing the 4 statuses.

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3331

app/client/components/GristDoc.ts

@@ -157,7 +157,7 @@ export class GristDoc extends DisposableWithEvents {
     } = {}
   ) {
     super();
-    console.log("RECEIVED DOC RESPONSE", openDocResponse.doc);
+    console.log("RECEIVED DOC RESPONSE", openDocResponse);
     this.docData = new DocData(this.docComm, openDocResponse.doc);
     this.docModel = new DocModel(this.docData);
     this.querySetManager = QuerySetManager.create(this, this.docModel, this.docComm);

app/client/models/DocPageModel.ts

@@ -14,6 +14,7 @@ import {bigBasicButton} from 'app/client/ui2018/buttons';
 import {testId} from 'app/client/ui2018/cssVars';
 import {menu, menuDivider, menuIcon, menuItem, menuText} from 'app/client/ui2018/menus';
 import {confirmModal} from 'app/client/ui2018/modals';
+import {DataLimitStatus} from 'app/common/ActiveDocAPI';
 import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow';
 import {delay} from 'app/common/delay';
 import {OpenDocMode, UserOverride} from 'app/common/DocListAPI';
@@ -66,6 +67,7 @@ export interface DocPageModel {
   gristDoc: Observable<GristDoc|null>;             // Instance of GristDoc once it exists.
 
   rowCount: Observable<number|undefined>;
+  dataLimitStatus: Observable<DataLimitStatus|undefined>;
 
   createLeftPane(leftPanelOpen: Observable<boolean>): DomArg;
   renameDoc(value: string): Promise<void>;
@@ -108,6 +110,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
   public readonly gristDoc = Observable.create<GristDoc|null>(this, null);
 
   public readonly rowCount = Observable.create<number|undefined>(this, undefined);
+  public readonly dataLimitStatus = Observable.create<DataLimitStatus|undefined>(this, null);
 
   // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the
   // URL, and when it changes, we need to re-open.
@@ -258,6 +261,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
       this.currentDoc.set({...doc});
     }
     this.rowCount.set(openDocResponse.rowCount);
+    this.dataLimitStatus.set(openDocResponse.dataLimitStatus);
     const gdModule = await gristDocModulePromise;
     const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier);
     flow.checkIfCancelled();

app/common/ActiveDocAPI.ts

@@ -153,6 +153,8 @@ export interface PermissionDataWithExtraUsers extends PermissionData {
   exampleUsers: UserAccessData[];
 }
 
+export type DataLimitStatus = null | 'approachingLimit' | 'gracePeriod' | 'deleteOnly';
+
 export interface ActiveDocAPI {
   /**
    * Closes a document, and unsubscribes from its userAction events.

app/common/DocListAPI.ts

@@ -1,4 +1,5 @@
 import {MinimalActionGroup} from 'app/common/ActionGroup';
+import {DataLimitStatus} from 'app/common/ActiveDocAPI';
 import {TableDataAction} from 'app/common/DocActions';
 import {Role} from 'app/common/roles';
 import {StringUnion} from 'app/common/StringUnion';
@@ -45,6 +46,7 @@ export interface OpenLocalDocResult {
   recoveryMode?: boolean;
   userOverride?: UserOverride;
   rowCount?: number;
+  dataLimitStatus?: DataLimitStatus;
 }
 
 export interface UserOverride {

app/common/Features.ts

@@ -45,9 +45,9 @@ export interface Features {
   baseMaxRowsPerDocument?: number;  // If set, establishes a default maximum on the
                                  // number of rows (total) in a single document.
                                  // Actual max for a document may be higher.
-                                 // TODO: not honored at time of writing.
-                                 // TODO: nuances about how rows are counted.
   baseMaxApiUnitsPerDocumentPerDay?: number;  // Similar for api calls.
+
+  gracePeriodDays?: number;  // Duration of the grace period in days, before entering delete-only mode
 }
 
 // Check whether it is possible to add members at the org level.  There's no flag

app/gen-server/entity/Document.ts

@@ -53,6 +53,9 @@ export class Document extends Resource {
   @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
   public removedAt: Date|null;
 
+  @Column({name: 'grace_period_start', type: nativeValues.dateTimeType, nullable: true})
+  public gracePeriodStart: Date|null;
+
   @OneToMany(type => Alias, alias => alias.doc)
   public aliases: Alias[];
 

app/gen-server/entity/Product.ts

@@ -37,7 +37,8 @@ export const teamFreeFeatures: Features = {
   maxDocsPerOrg: 20,
   snapshotWindow: { count: 1, unit: 'month' },
   baseMaxRowsPerDocument: 5000,
-  baseMaxApiUnitsPerDocumentPerDay: 5000
+  baseMaxApiUnitsPerDocumentPerDay: 5000,
+  gracePeriodDays: 14,
 };
 
 /**

app/gen-server/lib/HomeDBManager.ts

@@ -2362,6 +2362,14 @@ export class HomeDBManager extends EventEmitter {
     });
   }
 
+  public async setDocGracePeriodStart(docId: string, gracePeriodStart: Date | null) {
+    return await this._connection.createQueryBuilder()
+      .update(Document)
+      .set({gracePeriodStart})
+      .where({id: docId})
+      .execute();
+  }
+
   public async getDocProduct(docId: string): Promise<Product | undefined> {
     return await this._connection.createQueryBuilder()
       .select('product')

app/gen-server/migration/1647883793388-GracePeriodStart.ts

@@ -0,0 +1,18 @@
+import {nativeValues} from "app/gen-server/lib/values";
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class GracePeriodStart1647883793388 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn("docs", new TableColumn({
+      name: "grace_period_start",
+      type: nativeValues.dateTimeType,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn("docs", "grace_period_start");
+  }
+
+}

app/server/lib/ActiveDoc.ts

@@ -10,6 +10,7 @@ import {ActionSummary} from "app/common/ActionSummary";
 import {
   ApplyUAOptions,
   ApplyUAResult,
+  DataLimitStatus,
   DataSourceTransformed,
   ForkResult,
   ImportOptions,
@@ -34,6 +35,7 @@ import {DocData} from 'app/common/DocData';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentSettings} from 'app/common/DocumentSettings';
 import {normalizeEmail} from 'app/common/emails';
+import {Features} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
 import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
@@ -51,6 +53,7 @@ import {Authorizer} from 'app/server/lib/Authorizer';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
 import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager';
+import {ICreateActiveDocOptions} from 'app/server/lib/ICreate';
 import {makeForkIds} from 'app/server/lib/idUtils';
 import {GRIST_DOC_SQL, GRIST_DOC_WITH_TABLE1_SQL} from 'app/server/lib/initialDocSql';
 import {ISandbox} from 'app/server/lib/ISandbox';
@@ -160,18 +163,21 @@ export class ActiveDoc extends EventEmitter {
   private _lastMemoryMeasurement: number = 0;   // Timestamp when memory was last measured.
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
   private _rowCount?: number;
+  private _productFeatures?: Features;
+  private _gracePeriodStart: Date|null = null;
 
   // Timer for shutting down the ActiveDoc a bit after all clients are gone.
   private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000);
   private _recoveryMode: boolean = false;
   private _shuttingDown: boolean = false;
 
-  constructor(docManager: DocManager, docName: string, private _options?: {
+  constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
-    safeMode?: boolean,
-    docUrl?: string
-  }) {
     super();
     if (_options?.safeMode) { this._recoveryMode = true; }
+    if (_options?.doc) {
+      this._productFeatures = _options.doc.workspace.org.billingAccount?.product.features;
+      this._gracePeriodStart = _options.doc.gracePeriodStart;
+    }
     this._docManager = docManager;
     this._docName = docName;
     this.docStorage = new DocStorage(docManager.storageManager, docName);
@@ -219,6 +225,24 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
+  public async getDataLimitStatus(): Promise<DataLimitStatus> {
+    if (this._rowLimit && this._rowCount) {
+      const ratio = this._rowCount / this._rowLimit;
+      if (ratio > 1) {
+        const start = this._gracePeriodStart;
+        const days = this._productFeatures?.gracePeriodDays;
+        if (start && days && moment().diff(moment(start), 'days') >= days) {
+          return 'deleteOnly';
+        } else {
+          return 'gracePeriod';
+        }
+      } else if (ratio > 0.9) {
+        return 'approachingLimit';
+      }
+    }
+    return null;
+  }
+
   public async getUserOverride(docSession: OptDocSession) {
     return this._granularAccess.getUserOverride(docSession);
   }
@@ -900,6 +924,17 @@ export class ActiveDoc extends EventEmitter {
     // Be careful not to sneak into user action queue before Calculate action, otherwise
     // there'll be a deadlock.
     await this.waitForInitialization();
+
+    if (
+      await this.getDataLimitStatus() === "deleteOnly" &&
+      !actions.every(action => [
+          'RemoveTable', 'RemoveColumn', 'RemoveRecord', 'BulkRemoveRecord',
+          'RemoveViewSection', 'RemoveView', 'ApplyUndoActions',
+        ].includes(action[0] as string))
+    ) {
+      throw new Error("Document is in delete-only mode");
+    }
+
     // Granular access control implemented in _applyUserActions.
     return await this._applyUserActions(docSession, actions, options);
   }
@@ -1223,7 +1258,7 @@ export class ActiveDoc extends EventEmitter {
         ...this.getLogMeta(docSession),
         rowCount: sandboxActionBundle.rowCount
       });
-      this._rowCount = sandboxActionBundle.rowCount;
+      await this._updateRowCount(sandboxActionBundle.rowCount);
       await this._reportDataEngineMemory();
     } else {
       // Create default SandboxActionBundle to use if the data engine is not called.
@@ -1451,6 +1486,25 @@ export class ActiveDoc extends EventEmitter {
     };
   }
 
+  private get _rowLimit(): number | undefined {
+    return this._productFeatures?.baseMaxRowsPerDocument;
+  }
+
+  private async _updateGracePeriodStart(gracePeriodStart: Date | null) {
+    this._gracePeriodStart = gracePeriodStart;
+    await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart);
+  }
+
+  private async _updateRowCount(rowCount: number) {
+    this._rowCount = rowCount;
+    const exceedingRowLimit = this._rowLimit && rowCount > this._rowLimit;
+    if (exceedingRowLimit && !this._gracePeriodStart) {
+      await this._updateGracePeriodStart(new Date());
+    } else if (!exceedingRowLimit && this._gracePeriodStart) {
+      await this._updateGracePeriodStart(null);
+    }
+  }
+
   /**
    * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply.
    */

app/server/lib/DocManager.ts

@@ -313,11 +313,12 @@ export class DocManager extends EventEmitter {
         }
       }
 
-      const [metaTables, recentActions, userOverride, rowCount] = await Promise.all([
+      const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([
         activeDoc.fetchMetaTables(docSession),
         activeDoc.getRecentMinimalActions(docSession),
         activeDoc.getUserOverride(docSession),
         activeDoc.getRowCount(docSession),
+        activeDoc.getDataLimitStatus(),
       ]);
 
       const result = {
@@ -328,6 +329,7 @@ export class DocManager extends EventEmitter {
         recoveryMode: activeDoc.recoveryMode,
         userOverride,
         rowCount,
+        dataLimitStatus,
       } as OpenLocalDocResult;
 
       if (!activeDoc.muted) {
@@ -510,7 +512,7 @@ export class DocManager extends EventEmitter {
     const doc = await this._getDoc(docSession, docName);
     // Get URL for document for use with SELF_HYPERLINK().
     const docUrl = doc && await this._getDocUrl(doc);
-    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode});
+    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode, doc});
   }
 
   /**

app/server/lib/ICreate.ts

@@ -1,3 +1,4 @@
+import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DocManager} from 'app/server/lib/DocManager';
@@ -32,4 +33,5 @@ export interface ICreate {
 export interface ICreateActiveDocOptions {
   safeMode?: boolean;
   docUrl?: string;
+  doc?: Document;
 }

app/server/lib/requestUtils.ts

@@ -18,9 +18,10 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
   parseInt(process.env.GRIST_TEST_HTTPS_OFFSET, 10) : undefined;
 
 // Database fields that we permit in entities but don't want to cross the api.
-const INTERNAL_FIELDS = new Set(['apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId',
+const INTERNAL_FIELDS = new Set([
-                                 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId',
+  'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId',
-                                 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin']);
+  'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
+]);
 
 /**
  * Adapt a home-server or doc-worker URL to match the hostname in the request URL. For custom