(core) Add API endpoint to get site usage summary

Summary:
The summary includes a count of documents that are approaching
limits, in grace period, or delete-only. The endpoint is only accessible
to site owners, and is currently unused. A follow-up diff will add usage
banners to the site home page, which will use the response from the
endpoint to communicate usage information to owners.

Test Plan: Browser and server tests.

Reviewers: alexmojaki

Reviewed By: alexmojaki

Differential Revision: https://phab.getgrist.com/D3420

app/client/components/DocComm.ts

@@ -5,7 +5,7 @@ import {ActionGroup} from 'app/common/ActionGroup';
 import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI';
 import {DocAction, UserAction} from 'app/common/DocActions';
 import {OpenLocalDocResult} from 'app/common/DocListAPI';
-import {DocUsage} from 'app/common/DocUsage';
+import {FilteredDocUsageSummary} from 'app/common/DocUsage';
 import {docUrl} from 'app/common/urlUtils';
 import {Events as BackboneEvents} from 'backbone';
 import {Disposable, Emitter} from 'grainjs';
@@ -18,7 +18,7 @@ export interface DocUserAction extends CommMessage {
   data: {
     docActions: DocAction[];
     actionGroup: ActionGroup;
-    docUsage: DocUsage;
+    docUsage: FilteredDocUsageSummary;
     error?: string;
   };
 }

app/client/components/DocUsageBanner.ts

@@ -11,12 +11,12 @@ export class DocUsageBanner extends Disposable {
   // Whether the banner is vertically expanded on narrow screens.
   private readonly _isExpanded = Observable.create(this, true);
 
-  private readonly _currentDoc = this._docPageModel.currentDoc;
   private readonly _currentDocId = this._docPageModel.currentDocId;
-  private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus;
+  private readonly _currentDocUsage = this._docPageModel.currentDocUsage;
+  private readonly _currentOrg = this._docPageModel.currentOrg;
 
-  private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => {
+  private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => {
-    return doc?.workspace.org ?? null;
+    return usage?.dataLimitStatus ?? null;
   });
 
   private readonly _shouldShowBanner: Computed<boolean> =

app/client/components/DocumentUsage.ts

@@ -30,13 +30,23 @@ const ACCESS_DENIED_MESSAGE = 'Usage statistics are only available to users with
  */
 export class DocumentUsage extends Disposable {
   private readonly _currentDoc = this._docPageModel.currentDoc;
-  private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus;
+  private readonly _currentDocUsage = this._docPageModel.currentDocUsage;
-  private readonly _rowCount = this._docPageModel.rowCount;
+  private readonly _currentOrg = this._docPageModel.currentOrg;
-  private readonly _dataSizeBytes = this._docPageModel.dataSizeBytes;
-  private readonly _attachmentsSizeBytes = this._docPageModel.attachmentsSizeBytes;
 
-  private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => {
+  private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => {
-    return doc?.workspace.org ?? null;
+    return usage?.dataLimitStatus ?? null;
+  });
+
+  private readonly _rowCount = Computed.create(this, this._currentDocUsage, (_use, usage) => {
+    return usage?.rowCount;
+  });
+
+  private readonly _dataSizeBytes = Computed.create(this, this._currentDocUsage, (_use, usage) => {
+    return usage?.dataSizeBytes;
+  });
+
+  private readonly _attachmentsSizeBytes = Computed.create(this, this._currentDocUsage, (_use, usage) => {
+    return usage?.attachmentsSizeBytes;
   });
 
   private readonly _rowMetrics: Computed<MetricOptions | null> =
@@ -102,7 +112,9 @@ export class DocumentUsage extends Disposable {
     Computed.create(
       this, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes,
       (_use, doc, rowCount, dataSize, attachmentsSize) => {
-        return !doc || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'pending');
+        return !doc || [rowCount, dataSize, attachmentsSize].some(metric => {
+          return metric === 'pending' || metric === undefined;
+        });
       }
     );
 

app/client/components/GristDoc.ts

@@ -478,7 +478,7 @@ export class GristDoc extends DisposableWithEvents {
       if (schemaUpdated) {
         this.trigger('schemaUpdateAction', docActions);
       }
-      this.docPageModel.updateDocUsage(message.data.docUsage);
+      this.docPageModel.updateCurrentDocUsage(message.data.docUsage);
     }
   }
 

app/client/models/DocPageModel.ts

@@ -17,7 +17,7 @@ import {confirmModal} from 'app/client/ui2018/modals';
 import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow';
 import {delay} from 'app/common/delay';
 import {OpenDocMode, UserOverride} from 'app/common/DocListAPI';
-import {AttachmentsSize, DataLimitStatus, DataSize, DocUsage, RowCount} from 'app/common/DocUsage';
+import {FilteredDocUsageSummary} from 'app/common/DocUsage';
 import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls';
 import {getReconnectTimeout} from 'app/common/gutil';
 import {canEdit} from 'app/common/roles';
@@ -44,6 +44,7 @@ export interface DocPageModel {
 
   appModel: AppModel;
   currentDoc: Observable<DocInfo|null>;
+  currentDocUsage: Observable<FilteredDocUsageSummary|null>;
 
   // This block is to satisfy previous interface, but usable as this.currentDoc.get().id, etc.
   currentDocId: Observable<string|undefined>;
@@ -66,16 +67,11 @@ export interface DocPageModel {
 
   gristDoc: Observable<GristDoc|null>;             // Instance of GristDoc once it exists.
 
-  dataLimitStatus: Observable<DataLimitStatus>;
-  rowCount: Observable<RowCount>;
-  dataSizeBytes: Observable<DataSize>;
-  attachmentsSizeBytes: Observable<AttachmentsSize>;
-
   createLeftPane(leftPanelOpen: Observable<boolean>): DomArg;
   renameDoc(value: string): Promise<void>;
   updateCurrentDoc(urlId: string, openMode: OpenDocMode): Promise<Document>;
   refreshCurrentDoc(doc: DocInfo): Promise<Document>;
-  updateDocUsage(docUsage: DocUsage): void;
+  updateCurrentDocUsage(docUsage: FilteredDocUsageSummary): void;
 }
 
 export interface ImportSource {
@@ -88,6 +84,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
   public readonly pageType = "doc";
 
   public readonly currentDoc = Observable.create<DocInfo|null>(this, null);
+  public readonly currentDocUsage = Observable.create<FilteredDocUsageSummary|null>(this, null);
 
   public readonly currentUrlId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.urlId : undefined);
   public readonly currentDocId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.id : undefined);
@@ -112,11 +109,6 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
   // Observable set to the instance of GristDoc once it's created.
   public readonly gristDoc = Observable.create<GristDoc|null>(this, null);
 
-  public readonly dataLimitStatus = Observable.create<DataLimitStatus>(this, null);
-  public readonly rowCount = Observable.create<RowCount>(this, 'pending');
-  public readonly dataSizeBytes = Observable.create<DataSize>(this, 'pending');
-  public readonly attachmentsSizeBytes = Observable.create<AttachmentsSize>(this, 'pending');
-
   // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the
   // URL, and when it changes, we need to re-open.
   // If making a comparison, the id of the document we are comparing with is also included
@@ -199,6 +191,10 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
     return this.updateCurrentDoc(doc.urlId || doc.id, doc.openMode);
   }
 
+  public updateCurrentDocUsage(docUsage: FilteredDocUsageSummary) {
+    this.currentDocUsage.set(docUsage);
+  }
+
   // Replace the URL without reloading the doc.
   public updateUrlNoReload(urlId: string, urlOpenMode: OpenDocMode, options: {replace: boolean}) {
     const state = urlState().state.get();
@@ -208,13 +204,6 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
     return urlState().pushUrl(nextState, {avoidReload: true, ...options});
   }
 
-  public updateDocUsage(docUsage: DocUsage) {
-    this.rowCount.set(docUsage.rowCount);
-    this.dataLimitStatus.set(docUsage.dataLimitStatus);
-    this.dataSizeBytes.set(docUsage.dataSizeBytes);
-    this.attachmentsSizeBytes.set(docUsage.attachmentsSizeBytes);
-  }
-
   private _onOpenError(err: Error) {
     if (err instanceof CancelledError) {
       // This means that we started loading a new doc before the previous one finished loading.
@@ -273,7 +262,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
       this.currentDoc.set({...doc});
     }
     if (openDocResponse.docUsage) {
-      this.updateDocUsage(openDocResponse.docUsage);
+      this.updateCurrentDocUsage(openDocResponse.docUsage);
     }
     const gdModule = await gristDocModulePromise;
     const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier);

app/common/DocLimits.ts

@@ -0,0 +1,74 @@
+import {ApiError} from 'app/common/ApiError';
+import {APPROACHING_LIMIT_RATIO, DataLimitStatus, DocumentUsage, getUsageRatio} from 'app/common/DocUsage';
+import {Features} from 'app/common/Features';
+import * as moment from 'moment-timezone';
+
+/**
+ * Error class indicating failure due to limits being exceeded.
+ */
+export class LimitExceededError extends ApiError {
+  constructor(message: string) {
+    super(message, 413);
+  }
+}
+
+export interface GetDataLimitStatusParams {
+  docUsage: DocumentUsage | null;
+  productFeatures: Features | undefined;
+  gracePeriodStart: Date | null;
+}
+
+/**
+ * Given a set of params that includes document usage, current product features, and
+ * a grace-period start (if any), returns the data limit status of a document.
+ */
+export function getDataLimitStatus(params: GetDataLimitStatusParams): DataLimitStatus {
+  const {docUsage, productFeatures, gracePeriodStart} = params;
+  const ratio = getDataLimitRatio(docUsage, productFeatures);
+  if (ratio > 1) {
+    const start = gracePeriodStart;
+    const days = productFeatures?.gracePeriodDays;
+    if (start && days && moment().diff(moment(start), 'days') >= days) {
+      return 'deleteOnly';
+    } else {
+      return 'gracePeriod';
+    }
+  } else if (ratio > APPROACHING_LIMIT_RATIO) {
+    return 'approachingLimit';
+  } else {
+    return null;
+  }
+}
+
+/**
+ * Given `docUsage` and `productFeatures`, returns the highest usage ratio
+ * across all data-related limits (currently only row count and data size).
+ */
+export function getDataLimitRatio(
+  docUsage: DocumentUsage | null,
+  productFeatures: Features | undefined
+): number {
+  if (!docUsage) { return 0; }
+
+  const {rowCount, dataSizeBytes} = docUsage;
+  const maxRows = productFeatures?.baseMaxRowsPerDocument;
+  const maxDataSize = productFeatures?.baseMaxDataSizePerDocument;
+  const rowRatio = getUsageRatio(rowCount, maxRows);
+  const dataSizeRatio = getUsageRatio(dataSizeBytes, maxDataSize);
+  return Math.max(rowRatio, dataSizeRatio);
+}
+
+/**
+ * Maps `dataLimitStatus` status to an integer and returns it; larger integer
+ * values indicate a more "severe" status.
+ *
+ * Useful for relatively comparing the severity of two statuses.
+ */
+export function getSeverity(dataLimitStatus: DataLimitStatus): number {
+  switch (dataLimitStatus) {
+    case null: { return 0; }
+    case 'approachingLimit': { return 1; }
+    case 'gracePeriod': { return 2; }
+    case 'deleteOnly': { return 3; }
+  }
+}

app/common/DocListAPI.ts

@@ -1,6 +1,6 @@
 import {MinimalActionGroup} from 'app/common/ActionGroup';
 import {TableDataAction} from 'app/common/DocActions';
-import {DocUsage} from 'app/common/DocUsage';
+import {FilteredDocUsageSummary} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {StringUnion} from 'app/common/StringUnion';
 import {FullUser} from 'app/common/UserAPI';
@@ -45,7 +45,7 @@ export interface OpenLocalDocResult {
   log: MinimalActionGroup[];
   recoveryMode?: boolean;
   userOverride?: UserOverride;
-  docUsage?: DocUsage;
+  docUsage?: FilteredDocUsageSummary;
 }
 
 export interface UserOverride {

app/common/DocUsage.ts

@@ -1,29 +1,60 @@
-import {ApiError} from 'app/common/ApiError';
+export interface DocumentUsage {
-
+  rowCount?: number;
-export interface DocUsage {
+  dataSizeBytes?: number;
-  dataLimitStatus: DataLimitStatus;
+  attachmentsSizeBytes?: number;
-  rowCount: RowCount;
-  dataSizeBytes: DataSize;
-  attachmentsSizeBytes: AttachmentsSize;
 }
 
-type NumberOrStatus = number | 'hidden' | 'pending';
-
-export type RowCount = NumberOrStatus;
-
-export type DataSize = NumberOrStatus;
-
-export type AttachmentsSize = NumberOrStatus;
-
 export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null;
 
-export type NonHidden<T> = Exclude<T, 'hidden'>;
+type DocUsageOrPending = {
+  [Metric in keyof Required<DocumentUsage>]: Required<DocumentUsage>[Metric] | 'pending'
+}
+
+export interface DocUsageSummary extends DocUsageOrPending {
+  dataLimitStatus: DataLimitStatus;
+}
+
+// Count of non-removed documents in an org, grouped by data limit status.
+export type OrgUsageSummary = Record<NonNullable<DataLimitStatus>, number>;
+
+type FilteredDocUsage = {
+  [Metric in keyof DocUsageOrPending]: DocUsageOrPending[Metric] | 'hidden'
+}
+
+export interface FilteredDocUsageSummary extends FilteredDocUsage {
+  dataLimitStatus: DataLimitStatus;
+}
 
 // Ratio of usage at which we start telling users that they're approaching limits.
 export const APPROACHING_LIMIT_RATIO = 0.9;
 
-export class LimitExceededError extends ApiError {
+/**
-  constructor(message: string) {
+ * Computes a ratio of `usage` to `limit`, if possible. Returns 0 if `usage` or `limit`
-    super(message, 413);
+ * is invalid or undefined.
+ */
+export function getUsageRatio(usage: number | undefined, limit: number | undefined): number {
+  if (!isEnforceableLimit(limit) || usage === undefined || usage < 0) {
+    // Treat undefined or invalid values as having 0 usage.
+    return 0;
   }
+
+  return usage / limit;
+}
+
+/**
+ * Returns an empty org usage summary with values initialized to 0.
+ */
+ export function createEmptyOrgUsageSummary(): OrgUsageSummary {
+  return {
+    approachingLimit: 0,
+    gracePeriod: 0,
+    deleteOnly: 0,
+  };
+}
+
+/**
+ * Returns true if `limit` is defined and is a valid, positive number.
+ */
+function isEnforceableLimit(limit: number | undefined): limit is number {
+  return limit !== undefined && limit > 0;
 }

app/common/UserAPI.ts

@@ -5,6 +5,7 @@ import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI';
 import {BrowserSettings} from 'app/common/BrowserSettings';
 import {BulkColValues, TableColValues, TableRecordValue, TableRecordValues, UserAction} from 'app/common/DocActions';
 import {DocCreationInfo, OpenDocMode} from 'app/common/DocListAPI';
+import {OrgUsageSummary} from 'app/common/DocUsage';
 import {Features} from 'app/common/Features';
 import {ICustomWidget} from 'app/common/CustomWidget';
 import {isClient} from 'app/common/gristUrls';
@@ -288,6 +289,7 @@ export interface UserAPI {
   getWorkspace(workspaceId: number): Promise<Workspace>;
   getOrg(orgId: number|string): Promise<Organization>;
   getOrgWorkspaces(orgId: number|string): Promise<Workspace[]>;
+  getOrgUsageSummary(orgId: number|string): Promise<OrgUsageSummary>;
   getTemplates(onlyFeatured?: boolean): Promise<Workspace[]>;
   getDoc(docId: string): Promise<Document>;
   newOrg(props: Partial<OrganizationProperties>): Promise<number>;
@@ -448,6 +450,10 @@ export class UserAPIImpl extends BaseAPI implements UserAPI {
       { method: 'GET' });
   }
 
+  public async getOrgUsageSummary(orgId: number|string): Promise<OrgUsageSummary> {
+    return this.requestJson(`${this._url}/api/orgs/${orgId}/usage`, { method: 'GET' });
+  }
+
   public async getTemplates(onlyFeatured: boolean = false): Promise<Workspace[]> {
     return this.requestJson(`${this._url}/api/templates?onlyFeatured=${onlyFeatured ? 1 : 0}`, { method: 'GET' });
   }

app/gen-server/ApiServer.ts

@@ -146,6 +146,15 @@ export class ApiServer {
       return sendReply(req, res, query);
     }));
 
+    // GET /api/orgs/:oid/usage
+    // Get usage summary of all un-deleted documents in the organization.
+    // Only accessible to org owners.
+    this._app.get('/api/orgs/:oid/usage', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const usage = await this._dbManager.getOrgUsageSummary(getScope(req), org);
+      return sendOkReply(req, res, usage);
+    }));
+
     // POST /api/orgs
     // Body params: name (required), domain
     // Create a new org.
@@ -194,7 +203,7 @@ export class ApiServer {
       return sendReply(req, res, query);
     }));
 
-    // // DELETE /api/workspaces/:wid
+    // DELETE /api/workspaces/:wid
     // Delete the specified workspace and all included docs.
     this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => {
       const wsId = integerParam(req.params.wid, 'wid');

app/gen-server/entity/Document.ts

@@ -1,4 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
+import {DocumentUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {DocumentOptions, DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
 import {nativeValues} from 'app/gen-server/lib/values';
@@ -65,6 +66,9 @@ export class Document extends Resource {
   @OneToMany(_type => Secret, secret => secret.doc)
   public secrets: Secret[];
 
+  @Column({name: 'usage', type: nativeValues.jsonEntityType, nullable: true})
+  public usage: DocumentUsage | null;
+
   public checkProperties(props: any): props is Partial<DocumentProperties> {
     return super.checkProperties(props, documentPropertyKeys);
   }

app/gen-server/lib/HomeDBManager.ts

@@ -1,5 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate';
+import {getDataLimitStatus} from 'app/common/DocLimits';
+import {createEmptyOrgUsageSummary, DocumentUsage, OrgUsageSummary} from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {canAddOrgMembers, Features} from 'app/common/Features';
 import {buildUrlId, MIN_URLID_PREFIX_LENGTH, parseUrlId} from 'app/common/gristUrls';
@@ -231,6 +233,12 @@ function stringifyUrlIdOrg(urlId: string, org?: string): string {
   return `${urlId}:${org}`;
 }
 
+export interface DocumentMetadata {
+  // ISO 8601 UTC date (e.g. the output of new Date().toISOString()).
+  updatedAt?: string;
+  usage?: DocumentUsage|null;
+}
+
 /**
  * HomeDBManager handles interaction between the ApiServer and the Home database,
  * encapsulating the typeorm logic.
@@ -922,6 +930,44 @@ export class HomeDBManager extends EventEmitter {
     return result;
   }
 
+  /**
+   * Returns an organization's usage summary (e.g. count of documents that are approaching or exceeding
+   * limits).
+   */
+  public async getOrgUsageSummary(scope: Scope, orgKey: string|number): Promise<OrgUsageSummary> {
+    // Check that an owner of the org is making the request.
+    const markPermissions = Permissions.OWNER;
+    let orgQuery = this.org(scope, orgKey, {
+      markPermissions,
+      needRealOrg: true
+    });
+    orgQuery = this._addFeatures(orgQuery);
+    const orgQueryResult = await verifyIsPermitted(orgQuery);
+    const org: Organization = this.unwrapQueryResult(orgQueryResult);
+    const productFeatures = org.billingAccount.product.features;
+
+    // Grab all the non-removed documents in the org.
+    let docsQuery = this._docs()
+      .innerJoin('docs.workspace', 'workspaces')
+      .innerJoin('workspaces.org', 'orgs')
+      .where('docs.workspace_id = workspaces.id')
+      .andWhere('workspaces.removed_at IS NULL AND docs.removed_at IS NULL');
+    docsQuery = this._whereOrg(docsQuery, orgKey);
+    if (this.isMergedOrg(orgKey)) {
+      docsQuery = docsQuery.andWhere('orgs.owner_id = :userId', {userId: scope.userId});
+    }
+    const docsQueryResult = await this._verifyAclPermissions(docsQuery, { scope, emptyAllowed: true });
+    const docs: Document[] = this.unwrapQueryResult(docsQueryResult);
+
+    // Return an aggregate count of documents, grouped by data limit status.
+    const summary = createEmptyOrgUsageSummary();
+    for (const {usage: docUsage, gracePeriodStart} of docs) {
+      const dataLimitStatus = getDataLimitStatus({docUsage, gracePeriodStart, productFeatures});
+      if (dataLimitStatus) { summary[dataLimitStatus] += 1; }
+    }
+    return summary;
+  }
+
   /**
    * Compute the best access option for an organization, from the
    * users available to the client.  If none of the options can access
@@ -2364,12 +2410,13 @@ export class HomeDBManager extends EventEmitter {
   }
 
   /**
-   * Updates the updatedAt values for several docs. Takes a map where each entry maps a docId to
+   * Updates the updatedAt and usage values for several docs. Takes a map where each entry maps
-   * an ISO date string representing the new updatedAt time. This is not a part of the API, it
+   * a docId to a metadata object containing the updatedAt and/or usage values. This is not a part
-   * should be called only by the HostedMetadataManager when a change is made to a doc.
+   * of the API, it should be called only by the HostedMetadataManager when a change is made to a
+   * doc.
    */
-  public async setDocsUpdatedAt(
+  public async setDocsMetadata(
-    docUpdateMap: {[docId: string]: string}
+    docUpdateMap: {[docId: string]: DocumentMetadata}
   ): Promise<QueryResult<void>> {
     if (!docUpdateMap || Object.keys(docUpdateMap).length === 0) {
       return {
@@ -2382,7 +2429,7 @@ export class HomeDBManager extends EventEmitter {
       const updateTasks = docIds.map(docId => {
         return manager.createQueryBuilder()
           .update(Document)
-          .set({updatedAt: docUpdateMap[docId]})
+          .set(docUpdateMap[docId])
           .where("id = :docId", {docId})
           .execute();
       });

app/gen-server/migration/1651469582887-DocumentUsage.ts

@@ -0,0 +1,18 @@
+import {nativeValues} from "app/gen-server/lib/values";
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class DocumentUsage1651469582887 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn("docs", new TableColumn({
+      name: "usage",
+      type: nativeValues.jsonType,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn("docs", "usage");
+  }
+
+}

app/server/lib/ActiveDoc.ts

@@ -33,21 +33,26 @@ import {
   UserAction
 } from 'app/common/DocActions';
 import {DocData} from 'app/common/DocData';
+import {
+  getDataLimitRatio,
+  getDataLimitStatus,
+  getSeverity,
+  LimitExceededError,
+} from 'app/common/DocLimits';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentSettings} from 'app/common/DocumentSettings';
 import {
   APPROACHING_LIMIT_RATIO,
-  AttachmentsSize,
   DataLimitStatus,
-  DataSize,
+  DocumentUsage,
-  DocUsage,
+  DocUsageSummary,
-  LimitExceededError,
+  FilteredDocUsageSummary,
-  NonHidden,
+  getUsageRatio,
-  RowCount
 } from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {Features} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
+import {parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
@@ -180,12 +185,12 @@ export class ActiveDoc extends EventEmitter {
   private _fullyLoaded: boolean = false;  // Becomes true once all columns are loaded/computed.
   private _lastMemoryMeasurement: number = 0;    // Timestamp when memory was last measured.
   private _lastDataSizeMeasurement: number = 0;  // Timestamp when dbstat data size was last measured.
+  private _lastDataLimitStatus?: DataLimitStatus;
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
-  private _rowCount: NonHidden<RowCount> = 'pending';
+  private _docUsage: DocumentUsage|null = null;
-  private _dataSize: NonHidden<DataSize> = 'pending';
-  private _attachmentsSize: NonHidden<AttachmentsSize> = 'pending';
   private _productFeatures?: Features;
   private _gracePeriodStart: Date|null = null;
+  private _isForkOrSnapshot: boolean = false;
 
   // Timer for shutting down the ActiveDoc a bit after all clients are gone.
   private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000);
@@ -207,10 +212,30 @@ export class ActiveDoc extends EventEmitter {
 
   constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
     super();
+    const {forkId, snapshotId} = parseUrlId(docName);
+    this._isForkOrSnapshot = Boolean(forkId || snapshotId);
     if (_options?.safeMode) { this._recoveryMode = true; }
     if (_options?.doc) {
-      this._productFeatures = _options.doc.workspace.org.billingAccount?.product.features;
+      const {gracePeriodStart, workspace, usage} = _options.doc;
-      this._gracePeriodStart = _options.doc.gracePeriodStart;
+      this._productFeatures = workspace.org.billingAccount?.product.features;
+      this._gracePeriodStart = gracePeriodStart;
+
+      if (!this._isForkOrSnapshot) {
+        /* Note: We don't currently persist usage for forks or snapshots anywhere, so
+         * we need to hold off on setting _docUsage here. Normally, usage is set shortly
+         * after initialization finishes, after data/attachments size has finished
+         * calculating. However, this leaves a narrow window where forks can circumvent
+         * delete-only restrictions and replace the trunk document (even when the trunk
+         * is delete-only). This isn't very concerning today as the window is typically
+         * too narrow to easily exploit, and there are other ways to work around limits,
+         * like resetting gracePeriodStart by momentarily lowering usage. Regardless, it
+         * would be good to fix this eventually (perhaps around the same time we close
+         * up the gracePeriodStart loophole).
+         *
+         * TODO: Revisit this later and patch up the loophole. */
+        this._docUsage = usage;
+        this._lastDataLimitStatus = this.dataLimitStatus;
+      }
     }
     this._docManager = docManager;
     this._docName = docName;
@@ -253,55 +278,46 @@ export class ActiveDoc extends EventEmitter {
 
   public get isShuttingDown(): boolean { return this._shuttingDown; }
 
-  public get rowLimitRatio() {
-    if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') {
-      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
-      return 0;
-    }
 
-    return this._rowCount / this._rowLimit;
+  public get rowLimitRatio(): number {
+    return getUsageRatio(
+      this._docUsage?.rowCount,
+      this._productFeatures?.baseMaxRowsPerDocument
+    );
   }
 
-  public get dataSizeLimitRatio() {
+  public get dataSizeLimitRatio(): number {
-    if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') {
+    return getUsageRatio(
-      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
+      this._docUsage?.dataSizeBytes,
-      return 0;
+      this._productFeatures?.baseMaxDataSizePerDocument
-    }
+    );
-
-    return this._dataSize / this._dataSizeLimit;
   }
 
-  public get dataLimitRatio() {
+  public get dataLimitRatio(): number {
-    return Math.max(this.rowLimitRatio, this.dataSizeLimitRatio);
+    return getDataLimitRatio(this._docUsage, this._productFeatures);
   }
 
   public get dataLimitStatus(): DataLimitStatus {
-    const ratio = this.dataLimitRatio;
+    return getDataLimitStatus({
-    if (ratio > 1) {
+      docUsage: this._docUsage,
-      const start = this._gracePeriodStart;
+      productFeatures: this._productFeatures,
-      const days = this._productFeatures?.gracePeriodDays;
+      gracePeriodStart: this._gracePeriodStart,
-      if (start && days && moment().diff(moment(start), 'days') >= days) {
+    });
-        return 'deleteOnly';
-      } else {
-        return 'gracePeriod';
-      }
-    } else if (ratio > APPROACHING_LIMIT_RATIO) {
-      return 'approachingLimit';
-    }
-    return null;
   }
 
-  public get docUsage(): DocUsage {
+  public getDocUsageSummary(): DocUsageSummary {
     return {
       dataLimitStatus: this.dataLimitStatus,
-      rowCount: this._rowCount,
+      rowCount: this._docUsage?.rowCount ?? 'pending',
-      dataSizeBytes: this._dataSize,
+      dataSizeBytes: this._docUsage?.dataSizeBytes ?? 'pending',
-      attachmentsSizeBytes: this._attachmentsSize,
+      attachmentsSizeBytes: this._docUsage?.attachmentsSizeBytes ?? 'pending',
     };
   }
 
-  public getFilteredDocUsage(docSession: OptDocSession): Promise<DocUsage> {
+  public async getFilteredDocUsageSummary(
-    return this._granularAccess.filterDocUsage(docSession, this.docUsage);
+    docSession: OptDocSession
+  ): Promise<FilteredDocUsageSummary> {
+    return this._granularAccess.filterDocUsageSummary(docSession, this.getDocUsageSummary());
   }
 
   public async getUserOverride(docSession: OptDocSession) {
@@ -431,15 +447,29 @@ export class ActiveDoc extends EventEmitter {
         clearInterval(interval);
       }
 
+      // Remove expired attachments, i.e. attachments that were soft deleted a while ago. This
+      // needs to happen periodically, and doing it here means we can guarantee that it happens
+      // even if the doc is only ever opened briefly, without having to slow down startup.
+      const removeAttachmentsPromise = this.removeUnusedAttachments(true, {syncUsageToDatabase: false});
+
+      // Update data size as well. We'll schedule a sync to the database once both this and the
+      // above promise settle.
+      const updateDataSizePromise = this._updateDataSize({syncUsageToDatabase: false});
+
       try {
-        // Remove expired attachments, i.e. attachments that were soft deleted a while ago.
+        await removeAttachmentsPromise;
-        // This needs to happen periodically, and doing it here means we can guarantee that it happens even if
-        // the doc is only ever opened briefly, without having to slow down startup.
-        await this.removeUnusedAttachments(true);
       } catch (e) {
         this._log.error(docSession, "Failed to remove expired attachments", e);
       }
 
+      try {
+        await updateDataSizePromise;
+      } catch (e) {
+        this._log.error(docSession, "Failed to update data size", e);
+      }
+
+      this._syncDocUsageToDatabase(true);
+
       try {
         await this._docManager.storageManager.closeDocument(this.docName);
       } catch (err) {
@@ -715,7 +745,7 @@ export class ActiveDoc extends EventEmitter {
           // is potentially expensive, so this optimises for the common case of not exceeding the limit.
           const hadChanges = await this.updateUsedAttachmentsIfNeeded();
           if (hadChanges) {
-            await this._updateAttachmentsSize();
+            await this._updateAttachmentsSize({syncUsageToDatabase: false});
           } else {
             // No point in retrying if nothing changed.
             throw new LimitExceededError("Exceeded attachments limit for document");
@@ -1401,10 +1431,13 @@ export class ActiveDoc extends EventEmitter {
   /**
    * Delete unused attachments from _grist_Attachments and gristsys_Files.
    * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago.
+   * @param options.syncUsageToDatabase: if true, schedule an update to the usage column of the docs table, if
+   * any unused attachments were soft-deleted. defaults to true.
    */
-  public async removeUnusedAttachments(expiredOnly: boolean) {
+  public async removeUnusedAttachments(expiredOnly: boolean, options: {syncUsageToDatabase?: boolean} = {}) {
+    const {syncUsageToDatabase = true} = options;
     const hadChanges = await this.updateUsedAttachmentsIfNeeded();
-    if (hadChanges) { await this._updateAttachmentsSize(); }
+    if (hadChanges) { await this._updateAttachmentsSize({syncUsageToDatabase}); }
     const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly);
     if (rowIds.length) {
       const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
@@ -1468,7 +1501,7 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public async updateRowCount(rowCount: number, docSession: OptDocSession | null) {
-    this._rowCount = rowCount;
+    this._updateDocUsage({rowCount});
     log.rawInfo('Sandbox row count', {...this.getLogMeta(docSession), rowCount});
     await this._checkDataLimitRatio();
 
@@ -1649,17 +1682,50 @@ export class ActiveDoc extends EventEmitter {
     };
   }
 
-  private get _rowLimit(): number | undefined {
+  /**
-    return this._productFeatures?.baseMaxRowsPerDocument;
+   * Applies all metrics from `usage` to the current document usage state.
+   * Syncs updated usage to the home database by default, unless
+   * `options.syncUsageToDatabase` is set to false.
+   */
+  private _updateDocUsage(
+    usage: Partial<DocumentUsage>,
+    options: {
+      syncUsageToDatabase?: boolean
+    } = {}
+  ) {
+    const {syncUsageToDatabase = true} = options;
+    this._docUsage = {...(this._docUsage || {}), ...usage};
+    if (this._lastDataLimitStatus === this.dataLimitStatus) {
+      // If status is unchanged, there's no need to sync usage to the database, as it currently
+      // won't result in any noticeable difference to site usage banners. On shutdown, we'll
+      // still schedule a sync so that the latest usage is persisted.
+      return;
+    }
+
+    const lastStatus = this._lastDataLimitStatus;
+    this._lastDataLimitStatus = this.dataLimitStatus;
+    if (!syncUsageToDatabase) { return; }
+
+    // If status decreased, we'll want to update usage in the DB with minimal delay, so that site
+    // usage banners show up-to-date statistics. If status increased or stayed the same, we'll
+    // schedule a delayed update, since it's less critical for such banners to update quickly
+    // when usage grows.
+    const didStatusDecrease = (
+      lastStatus !== undefined &&
+      getSeverity(this.dataLimitStatus) < getSeverity(lastStatus)
+    );
+    this._syncDocUsageToDatabase(didStatusDecrease);
   }
 
-  private get _dataSizeLimit(): number | undefined {
+  private _syncDocUsageToDatabase(minimizeDelay = false) {
-    return this._productFeatures?.baseMaxDataSizePerDocument;
+    this._docManager.storageManager.scheduleUsageUpdate(this._docName, this._docUsage, minimizeDelay);
   }
 
   private async _updateGracePeriodStart(gracePeriodStart: Date | null) {
     this._gracePeriodStart = gracePeriodStart;
-    await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart);
+    if (!this._isForkOrSnapshot) {
+      await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart);
+    }
   }
 
   private async _checkDataLimitRatio() {
@@ -1673,13 +1739,29 @@ export class ActiveDoc extends EventEmitter {
 
   private async _checkDataSizeLimitRatio(docSession: OptDocSession | null) {
     const start = Date.now();
-    const dataSize = await this.docStorage.getDataSize();
+    const dataSizeBytes = await this._updateDataSize();
     const timeToMeasure = Date.now() - start;
-    this._dataSize = dataSize;
+    log.rawInfo('Data size from dbstat...', {
-    log.rawInfo('Data size from dbstat...', {...this.getLogMeta(docSession), dataSize, timeToMeasure});
+      ...this.getLogMeta(docSession),
+      dataSizeBytes,
+      timeToMeasure,
+    });
     await this._checkDataLimitRatio();
   }
 
+  /**
+   * Calculates the total data size in bytes and sets it in _docUsage. Schedules
+   * a sync to the database, unless `options.syncUsageToDatabase` is set to false.
+   *
+   * Returns the calculated data size.
+   */
+  private async _updateDataSize(options: {syncUsageToDatabase?: boolean} = {}): Promise<number> {
+    const {syncUsageToDatabase = true} = options;
+    const dataSizeBytes = await this.docStorage.getDataSize();
+    this._updateDocUsage({dataSizeBytes}, {syncUsageToDatabase});
+    return dataSizeBytes;
+  }
+
   /**
    * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply.
    */
@@ -1840,10 +1922,7 @@ export class ActiveDoc extends EventEmitter {
       const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT;
       this._inactivityTimer.setDelay(closeTimeout);
       this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`);
-      // TODO: Initialize data and attachments size from Document.usage once it's available.
+      this._initializeDocUsageIfNeeded(docSession);
-      this._updateAttachmentsSize().catch(e => {
-        this._log.warn(docSession, 'failed to update attachments size', e);
-      });
     } catch (err) {
       this._fullyLoaded = true;
       if (!this._shuttingDown) {
@@ -1884,6 +1963,21 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
+  private _initializeDocUsageIfNeeded(docSession: OptDocSession) {
+    // TODO: Broadcast a message to clients after usage is fully calculated.
+    if (this._docUsage?.dataSizeBytes === undefined) {
+      this._updateDataSize().catch(e => {
+        this._log.warn(docSession, 'failed to update data size', e);
+      });
+    }
+
+    if (this._docUsage?.attachmentsSizeBytes === undefined) {
+      this._updateAttachmentsSize().catch(e => {
+        this._log.warn(docSession, 'failed to update attachments size', e);
+      });
+    }
+  }
+
   /**
    * Called before a migration.  Makes sure a back-up is made.
    */
@@ -1990,14 +2084,22 @@ export class ActiveDoc extends EventEmitter {
     const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
     if (!maxSize) { return true; }
 
-    const currentSize = this._attachmentsSize !== 'pending'
+    let currentSize = this._docUsage?.attachmentsSizeBytes;
-      ? this._attachmentsSize
+    currentSize = currentSize ?? await this._updateAttachmentsSize({syncUsageToDatabase: false});
-      : await this.docStorage.getTotalAttachmentFileSizes();
     return currentSize + uploadSizeBytes <= maxSize;
   }
 
-  private async _updateAttachmentsSize() {
+  /**
-    this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes();
+   * Calculates the total attachments size in bytes and sets it in _docUsage. Schedules
+   * a sync to the database, unless `options.syncUsageToDatabase` is set to false.
+   *
+   * Returns the calculated attachments size.
+   */
+  private async _updateAttachmentsSize(options: {syncUsageToDatabase?: boolean} = {}): Promise<number> {
+    const {syncUsageToDatabase = true} = options;
+    const attachmentsSizeBytes = await this.docStorage.getTotalAttachmentFileSizes();
+    this._updateDocUsage({attachmentsSizeBytes}, {syncUsageToDatabase});
+    return attachmentsSizeBytes;
   }
 }
 
@@ -2023,8 +2125,3 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table
   }
   return tableRefs[tableRowIndex];
 }
-
-// Helper that returns true if `limit` is set to a valid, positive number.
-function isEnforceableLimit(limit: number | undefined): limit is number {
-  return limit !== undefined && limit > 0;
-}

app/server/lib/DocApi.ts

@@ -851,6 +851,9 @@ export class DocWorkerApi {
       return true;
     }
 
+    // If Redis isn't configured, this is as far as we can go with checks.
+    if (!process.env.REDIS_URL) { return false; }
+
     // Note the increased API usage on redis and in our local cache.
     // Update redis in the background so that the rest of the request can continue without waiting for redis.
     const multi = this._docWorkerMap.getRedisClient().multi();

app/server/lib/DocManager.ts

@@ -9,7 +9,7 @@ import {ApiError} from 'app/common/ApiError';
 import {mapSetOrClear} from 'app/common/AsyncCreate';
 import {BrowserSettings} from 'app/common/BrowserSettings';
 import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
-import {DocUsage} from 'app/common/DocUsage';
+import {FilteredDocUsageSummary} from 'app/common/DocUsage';
 import {Invite} from 'app/common/sharing';
 import {tbind} from 'app/common/tbind';
 import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
@@ -320,9 +320,9 @@ export class DocManager extends EventEmitter {
         activeDoc.getUserOverride(docSession),
       ]);
 
-      let docUsage: DocUsage | undefined;
+      let docUsage: FilteredDocUsageSummary | undefined;
       try {
-        docUsage = await activeDoc.getFilteredDocUsage(docSession);
+        docUsage = await activeDoc.getFilteredDocUsageSummary(docSession);
       } catch (e) {
         log.warn("DocManager.openDoc failed to get doc usage", e);
       }

app/server/lib/DocStorageManager.ts

@@ -6,6 +6,7 @@ import * as path from 'path';
 
 import {DocEntry, DocEntryTag} from 'app/common/DocListAPI';
 import {DocSnapshots} from 'app/common/DocSnapshot';
+import {DocumentUsage} from 'app/common/DocUsage';
 import * as gutil from 'app/common/gutil';
 import * as Comm from 'app/server/lib/Comm';
 import * as docUtils from 'app/server/lib/docUtils';
@@ -217,6 +218,14 @@ export class DocStorageManager implements IDocStorageManager {
     // nothing to do
   }
 
+  public scheduleUsageUpdate(
+    docName: string,
+    docUsage: DocumentUsage,
+    minimizeDelay = false
+  ): void {
+    // nothing to do
+  }
+
   public testReopenStorage(): void {
     // nothing to do
   }

app/server/lib/GranularAccess.ts

@@ -10,7 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app
 import { TableDataAction, UserAction } from 'app/common/DocActions';
 import { DocData } from 'app/common/DocData';
 import { UserOverride } from 'app/common/DocListAPI';
-import { DocUsage } from 'app/common/DocUsage';
+import { DocUsageSummary, FilteredDocUsageSummary } from 'app/common/DocUsage';
 import { normalizeEmail } from 'app/common/emails';
 import { ErrorWithCode } from 'app/common/ErrorWithCode';
 import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause';
@@ -105,7 +105,7 @@ const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);
 interface DocUpdateMessage {
   actionGroup: ActionGroup;
   docActions: DocAction[];
-  docUsage: DocUsage;
+  docUsage: DocUsageSummary;
 }
 
 /**
@@ -115,7 +115,7 @@ export interface GranularAccessForBundle {
   canApplyBundle(): Promise<void>;
   appliedBundle(): Promise<void>;
   finishedBundle(): Promise<void>;
-  sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise<void>;
+  sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsageSummary): Promise<void>;
 }
 
 /**
@@ -395,14 +395,14 @@ export class GranularAccess implements GranularAccessForBundle {
   }
 
   /**
-   * Filter DocUsage to be sent to a client.
+   * Filter DocUsageSummary to be sent to a client.
    */
-  public async filterDocUsage(
+  public async filterDocUsageSummary(
     docSession: OptDocSession,
-    docUsage: DocUsage,
+    docUsage: DocUsageSummary,
     options: {role?: Role | null} = {}
-  ): Promise<DocUsage> {
+  ): Promise<FilteredDocUsageSummary> {
-    const result: DocUsage = { ...docUsage };
+    const result: FilteredDocUsageSummary = { ...docUsage };
     const role = options.role ?? await this.getNominalAccess(docSession);
     const hasEditRole = canEdit(role);
     if (!hasEditRole) { result.dataLimitStatus = null; }
@@ -747,7 +747,7 @@ export class GranularAccess implements GranularAccessForBundle {
   /**
    * Broadcast document changes to all clients, with appropriate filtering.
    */
-  public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) {
+  public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsageSummary) {
     if (!this._activeBundle) { throw new Error('no active bundle'); }
     const { docActions, docSession } = this._activeBundle;
     const client = docSession && docSession.client || null;
@@ -909,7 +909,7 @@ export class GranularAccess implements GranularAccessForBundle {
     const role = await this.getNominalAccess(docSession);
     const result = {
       ...message,
-      docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}),
+      docUsage: await this.filterDocUsageSummary(docSession, message.docUsage, {role}),
     };
     if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
       return result;

app/server/lib/HostedMetadataManager.ts

@@ -1,14 +1,14 @@
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {DocumentMetadata, HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import * as log from 'app/server/lib/log';
 
 /**
  * HostedMetadataManager handles pushing document metadata changes to the Home database when
- * a doc is updated. Currently only updates doc updatedAt time.
+ * a doc is updated. Currently updates doc updatedAt time and usage.
  */
 export class HostedMetadataManager {
 
-  // updatedAt times as UTC ISO strings mapped by docId.
+  // Document metadata mapped by docId.
-  private _updatedAt: {[docId: string]: string} = {};
+  private _metadata: {[docId: string]: DocumentMetadata} = {};
 
   // Set if the class holder is closing and no further pushes should be scheduled.
   private _closing: boolean = false;
@@ -22,60 +22,78 @@ export class HostedMetadataManager {
   // Maintains the update Promise to wait on it if the class is closing.
   private _push: Promise<any>|null;
 
+  // The default delay in milliseconds between metadata pushes to the database.
+  private readonly _minPushDelayMs: number;
+
   /**
    * Create an instance of HostedMetadataManager.
-   * The minPushDelay is the delay in seconds between metadata pushes to the database.
+   * The minPushDelay is the default delay in seconds between metadata pushes to the database.
    */
-  constructor(private _dbManager: HomeDBManager, private _minPushDelay: number = 60) {}
+  constructor(private _dbManager: HomeDBManager, minPushDelay: number = 60) {
+    this._minPushDelayMs = minPushDelay * 1000;
+  }
 
   /**
    * Close the manager. Send out any pending updates and prevent more from being scheduled.
    */
   public async close(): Promise<void> {
-    // Finish up everything outgoing
+    // Pushes will no longer be scheduled.
-    this._closing = true;  // Pushes will no longer be scheduled.
+    this._closing = true;
+    // Wait for outgoing pushes to finish before proceeding.
+    if (this._push) { await this._push; }
     if (this._timeout) {
-      clearTimeout(this._timeout);
-      this._timeout = null;
       // Since an update was scheduled, perform one final update now.
       this._update();
+      if (this._push) { await this._push; }
     }
-    if (this._push) { await this._push; }
   }
 
   /**
    * Schedule a call to _update some time from now.  When the update is made, it will
-   * store the given timestamp in the updated_at column of the docs table for the
+   * store the given metadata in the updated_at and usage columns of the docs table for
-   * specified document. Timestamp should be an ISO 8601 format time, in UTC, e.g.
+   * the specified document.
-   * the output of new Date().toISOString()
+   *
+   * If `minimizeDelay` is true, the push will be scheduled with minimum delay (0ms) and
+   * will cancel/overwrite an already scheduled push (if present).
    */
-  public scheduleUpdate(docId: string, timestamp: string): void {
+  public scheduleUpdate(docId: string, metadata: DocumentMetadata, minimizeDelay = false): void {
-    // Update updatedAt even if an update is already scheduled - if the update has not yet occurred,
+    if (this._closing) { return; }
-    // the more recent updatedAt time will be used.
+
-    this._updatedAt[docId] = timestamp;
+    // Update metadata even if an update is already scheduled - if the update has not yet occurred,
-    if (this._timeout || this._closing) { return; }
+    // the more recent metadata will be used.
-    const minDelay = this._minPushDelay * 1000;
+    this._setOrUpdateMetadata(docId, metadata);
-    // Set the push to occur at least the minDelay after the last push time.
+    if (this._timeout && !minimizeDelay) { return; }
-    const delay = Math.round(minDelay - (Date.now() - this._lastPushTime));
+
-    this._timeout = setTimeout(() => this._update(), delay < 0 ? 0 : delay);
+    this._schedulePush(minimizeDelay ? 0 : undefined);
   }
 
-  public setDocsUpdatedAt(docUpdateMap: {[docId: string]: string}): Promise<any> {
+  public setDocsMetadata(docUpdateMap: {[docId: string]: DocumentMetadata}): Promise<any> {
-    return this._dbManager.setDocsUpdatedAt(docUpdateMap);
+    return this._dbManager.setDocsMetadata(docUpdateMap);
   }
 
   /**
    * Push all metadata updates to the database.
    */
   private _update(): void {
-    if (this._push) { return; }
     if (this._timeout) {
       clearTimeout(this._timeout);
       this._timeout = null;
     }
+    if (this._push) { return; }
     this._push = this._performUpdate()
-    .catch(err => { log.error("HostedMetadataManager error performing update: ", err); })
+    .catch(err => {
-    .then(() => { this._push = null; });
+      log.error("HostedMetadataManager error performing update: ", err);
+    })
+    .then(() => {
+      this._push = null;
+      if (!this._closing && !this._timeout && Object.keys(this._metadata).length !== 0) {
+        // If we have metadata that hasn't been pushed up yet, but no push scheduled,
+        // go ahead and schedule an immediate push. This can happen if `scheduleUpdate`
+        // is called frequently with minimizeDelay set to true, particularly when
+        // _performUpdate is taking a bit longer than normal to complete.
+        this._schedulePush(0);
+      }
+    });
   }
 
   /**
@@ -84,9 +102,40 @@ export class HostedMetadataManager {
    */
   private async _performUpdate(): Promise<void> {
     // Await the database if it is not yet connected.
-    const docUpdates = this._updatedAt;
+    const docUpdates = this._metadata;
-    this._updatedAt = {};
+    this._metadata = {};
     this._lastPushTime = Date.now();
-    await this.setDocsUpdatedAt(docUpdates);
+    await this.setDocsMetadata(docUpdates);
+  }
+
+  /**
+   * Schedule a metadata push.
+   *
+   * If `delayMs` is specified, the push will be scheduled to occur at least that
+   * number of milliseconds in the future. If `delayMs` is unspecified, the push
+   * will be scheduled to occur at least `_minPushDelayMs` after the last push time.
+   *
+   * If called while a push is already scheduled, that push will be cancelled and
+   * replaced with this one.
+   */
+  private _schedulePush(delayMs?: number): void {
+    if (delayMs === undefined) {
+      delayMs = Math.round(this._minPushDelayMs - (Date.now() - this._lastPushTime));
+    }
+    if (this._timeout) { clearTimeout(this._timeout); }
+    this._timeout = setTimeout(() => this._update(), delayMs < 0 ? 0 : delayMs);
+  }
+
+  /**
+   * Adds `docId` and its `metadata` to the list of queued updates, merging any existing values.
+   */
+  private _setOrUpdateMetadata(docId: string, metadata: DocumentMetadata): void {
+    if (!this._metadata[docId]) {
+      this._metadata[docId] = metadata;
+    } else {
+      const {updatedAt, usage} = metadata;
+      if (updatedAt) { this._metadata[docId].updatedAt = updatedAt; }
+      if (usage !== undefined) { this._metadata[docId].usage = usage; }
+    }
   }
 }

app/server/lib/HostedStorageManager.ts

@@ -3,6 +3,7 @@ import {mapGetOrSet} from 'app/common/AsyncCreate';
 import {delay} from 'app/common/delay';
 import {DocEntry} from 'app/common/DocListAPI';
 import {DocSnapshots} from 'app/common/DocSnapshot';
+import {DocumentUsage} from 'app/common/DocUsage';
 import {buildUrlId, parseUrlId} from 'app/common/gristUrls';
 import {KeyedOps} from 'app/common/KeyedOps';
 import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
@@ -323,6 +324,8 @@ export class HostedStorageManager implements IDocStorageManager {
       // NOTE: fse.remove succeeds also when the file does not exist.
       await fse.remove(this._getHashFile(this.getPath(docId)));
       this.markAsChanged(docId, 'edit');
+      // Invalidate usage; it'll get re-computed the next time the document is opened.
+      this.scheduleUsageUpdate(docId, null, true);
     } catch (err) {
       this._log.error(docId, "problem replacing doc: %s", err);
       await fse.move(tmpPath, docPath, {overwrite: true});
@@ -483,6 +486,27 @@ export class HostedStorageManager implements IDocStorageManager {
     }
   }
 
+  /**
+   * Schedule an update to a document's usage column.
+   *
+   * If `minimizeDelay` is true, HostedMetadataManager will attempt to
+   * minimize delays by scheduling the update to occur as soon as possible.
+   */
+  public scheduleUsageUpdate(
+    docName: string,
+    docUsage: DocumentUsage|null,
+    minimizeDelay = false
+  ): void {
+    const {forkId, snapshotId} = parseUrlId(docName);
+    if (!this._metadataManager || forkId || snapshotId) { return; }
+
+    this._metadataManager.scheduleUpdate(
+      docName,
+      {usage: docUsage},
+      minimizeDelay
+    );
+  }
+
   /**
    * Check if there is a pending change to be pushed to S3.
    */
@@ -524,9 +548,9 @@ export class HostedStorageManager implements IDocStorageManager {
    * This is called when a document was edited by the user.
    */
   private _markAsEdited(docName: string, timestamp: string): void {
-    if (parseUrlId(docName).snapshotId) { return; }
+    if (parseUrlId(docName).snapshotId || !this._metadataManager) { return; }
     // Schedule a metadata update for the modified doc.
-    if (this._metadataManager) { this._metadataManager.scheduleUpdate(docName, timestamp); }
+    this._metadataManager.scheduleUpdate(docName, {updatedAt: timestamp});
   }
 
   /**

app/server/lib/IDocStorageManager.ts

@@ -1,5 +1,6 @@
 import {DocEntry} from 'app/common/DocListAPI';
 import {DocSnapshots} from 'app/common/DocSnapshot';
+import {DocumentUsage} from 'app/common/DocUsage';
 import {DocReplacementOptions} from 'app/common/UserAPI';
 
 export interface IDocStorageManager {
@@ -24,6 +25,7 @@ export interface IDocStorageManager {
   // Mark document as needing a backup (due to edits, migrations, etc).
   // If reason is set to 'edit' the user-facing timestamp on the document should be updated.
   markAsChanged(docName: string, reason?: 'edit'): void;
+  scheduleUsageUpdate(docName: string, usage: DocumentUsage|null, minimizeDelay?: boolean): void;
   testReopenStorage(): void;                // restart storage during tests
   addToStorage(docName: string): Promise<void>;  // add a new local document to storage
   prepareToCloseStorage(): void;            // speed up sync with remote store

app/server/lib/Sharing.ts

@@ -314,7 +314,7 @@ export class Sharing {
       });
       actionGroup.actionSummary = actionSummary;
       await accessControl.appliedBundle();
-      await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage);
+      await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.getDocUsageSummary());
       if (docSession) {
         docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0;
       }

app/server/lib/requestUtils.ts

@@ -21,7 +21,7 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
 const INTERNAL_FIELDS = new Set([
   'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId',
   'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
-  'authSubject',
+  'authSubject', 'usage'
 ]);
 
 /**