(core) Broadcast doc usage updates to clients

Summary:
Introduces a new message type, docUsage, that's broadcast to all connected
clients whenever document usage is updated in ActiveDoc.

Test Plan: Browser tests.

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3451

app/client/components/Comm.ts

@@ -63,6 +63,14 @@ import {Events as BackboneEvents} from 'backbone';
  * @property {Boolean} fromSelf - Flag to indicate whether the action originated from this client.
  */
 
+/**
+ * Event for a change to document usage. Sent to all clients that have this document open.
+ * @event docUsage
+ * @property {Number} docFD - The file descriptor of the open document, specific to each client.
+ * @property {FilteredDocUsageSummary} data.docUsage - Document usage summary.
+ * @property {Product} data.product - Product that was used to compute `data.docUsage`
+ */
+
 /**
  * Event for when a document is forcibly shutdown, and requires the client to re-open it.
  * @event docShutdown
@@ -111,7 +119,7 @@ import {Events as BackboneEvents} from 'backbone';
  */
 
 const ValidEvent = StringUnion('docListAction', 'docUserAction', 'docShutdown', 'docError',
-                               'clientConnect', 'clientLogout',
+                               'docUsage', 'clientConnect', 'clientLogout',
                                'profileFetch', 'userSettings', 'receiveInvites');
 type ValidEvent = typeof ValidEvent.type;
 

app/client/components/DocComm.ts

@@ -6,6 +6,7 @@ import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocA
 import {DocAction, UserAction} from 'app/common/DocActions';
 import {OpenLocalDocResult} from 'app/common/DocListAPI';
 import {FilteredDocUsageSummary} from 'app/common/DocUsage';
+import {Product} from 'app/common/Features';
 import {docUrl} from 'app/common/urlUtils';
 import {Events as BackboneEvents} from 'backbone';
 import {Disposable, Emitter} from 'grainjs';
@@ -13,7 +14,6 @@ import {Disposable, Emitter} from 'grainjs';
 // tslint:disable:no-console
 
 export interface DocUserAction extends CommMessage {
-  docFD: number;
   fromSelf?: boolean;
   data: {
     docActions: DocAction[];
@@ -23,6 +23,13 @@ export interface DocUserAction extends CommMessage {
   };
 }
 
+export interface DocUsageMessage extends CommMessage {
+  data: {
+    docUsage: FilteredDocUsageSummary;
+    product?: Product;
+  };
+}
+
 const SLOW_NOTIFICATION_TIMEOUT_MS = 1000; // applies to user actions only
 
 /**

app/client/components/DocumentUsage.ts

@@ -1,6 +1,7 @@
 import {DocPageModel} from 'app/client/models/DocPageModel';
 import {urlState} from 'app/client/models/gristUrlState';
 import {docListHeader} from 'app/client/ui/DocMenuCss';
+import {infoTooltip} from 'app/client/ui/tooltips';
 import {colors, mediaXSmall} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
 import {cssLink} from 'app/client/ui2018/links';
@@ -33,6 +34,7 @@ export class DocumentUsage extends Disposable {
   private readonly _currentDoc = this._docPageModel.currentDoc;
   private readonly _currentDocUsage = this._docPageModel.currentDocUsage;
   private readonly _currentOrg = this._docPageModel.currentOrg;
+  private readonly _currentProduct = this._docPageModel.currentProduct;
 
   private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => {
     return usage?.dataLimitStatus ?? null;
@@ -51,8 +53,8 @@ export class DocumentUsage extends Disposable {
   });
 
   private readonly _rowMetrics: Computed<MetricOptions | null> =
-    Computed.create(this, this._currentOrg, this._rowCount, (_use, org, rowCount) => {
-      const features = org?.billingAccount?.product.features;
+    Computed.create(this, this._currentProduct, this._rowCount, (_use, product, rowCount) => {
+      const features = product?.features;
       if (!features || typeof rowCount !== 'number') { return null; }
 
       const {baseMaxRowsPerDocument: maxRows} = features;
@@ -68,8 +70,8 @@ export class DocumentUsage extends Disposable {
     });
 
   private readonly _dataSizeMetrics: Computed<MetricOptions | null> =
-    Computed.create(this, this._currentOrg, this._dataSizeBytes, (_use, org, dataSize) => {
-      const features = org?.billingAccount?.product.features;
+    Computed.create(this, this._currentProduct, this._dataSizeBytes, (_use, product, dataSize) => {
+      const features = product?.features;
       if (!features || typeof dataSize !== 'number') { return null; }
 
       const {baseMaxDataSizePerDocument: maxSize} = features;
@@ -81,6 +83,10 @@ export class DocumentUsage extends Disposable {
         maximumValue: maxValue ?? DEFAULT_MAX_DATA_SIZE,
         unit: 'MB',
         shouldHideLimits: maxValue === undefined,
+        tooltipContent: () => cssTooltipBody(
+          dom('div', 'The total size of all data in this document, excluding attachments.'),
+          dom('div', 'Updates every 5 minutes.'),
+        ),
         formatValue: (val) => {
           // To display a nice, round number for `maximumValue`, we first convert
           // to KiBs (base-2), and then convert to MBs (base-10). Normally, we wouldn't
@@ -92,8 +98,8 @@ export class DocumentUsage extends Disposable {
     });
 
   private readonly _attachmentsSizeMetrics: Computed<MetricOptions | null> =
-    Computed.create(this, this._currentOrg, this._attachmentsSizeBytes, (_use, org, attachmentsSize) => {
-      const features = org?.billingAccount?.product.features;
+    Computed.create(this, this._currentProduct, this._attachmentsSizeBytes, (_use, product, attachmentsSize) => {
+      const features = product?.features;
       if (!features || typeof attachmentsSize !== 'number') { return null; }
 
       const {baseMaxAttachmentsBytesPerDocument: maxSize} = features;
@@ -154,10 +160,10 @@ export class DocumentUsage extends Disposable {
       if (isAccessDenied) { return buildMessage(ACCESS_DENIED_MESSAGE); }
 
       const org = use(this._currentOrg);
+      const product = use(this._currentProduct);
       const status = use(this._dataLimitStatus);
       if (!org || !status) { return null; }
 
-      const product = org.billingAccount?.product;
       return buildMessage([
         buildLimitStatusMessage(status, product?.features, {
           disableRawDataLink: true
@@ -259,6 +265,8 @@ interface MetricOptions {
   unit?: string;
   // If true, limits will always be hidden, even if `maximumValue` is a positive number.
   shouldHideLimits?: boolean;
+  // Shows an icon next to the metric name that displays a tooltip on hover.
+  tooltipContent?(): DomContents;
   formatValue?(value: number): string;
 }
 
@@ -274,12 +282,16 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
     maximumValue,
     unit,
     shouldHideLimits,
+    tooltipContent,
     formatValue = (val) => val.toString(),
   } = options;
   const ratioUsed = currentValue / (maximumValue || Infinity);
   const percentUsed = Math.min(100, Math.floor(ratioUsed * 100));
   return cssUsageMetric(
-    cssMetricName(name, testId('name')),
+    cssMetricName(
+      cssOverflowableText(name, testId('name')),
+      tooltipContent ? infoTooltip(tooltipContent()) : null,
+    ),
     cssProgressBarContainer(
       cssProgressBarFill(
         {style: `width: ${percentUsed}%`},
@@ -328,9 +340,18 @@ const cssIcon = styled(icon, `
 `);
 
 const cssMetricName = styled('div', `
+  display: flex;
+  align-items: center;
+  gap: 8px;
   font-weight: 700;
 `);
 
+const cssOverflowableText = styled('span', `
+  overflow: hidden;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+`);
+
 const cssHeader = styled(docListHeader, `
   margin-bottom: 0px;
 `);
@@ -385,3 +406,9 @@ const cssSpinner = styled('div', `
   justify-content: center;
   margin-top: 32px;
 `);
+
+const cssTooltipBody = styled('div', `
+  display: flex;
+  flex-direction: column;
+  gap: 8px;
+`);

app/client/components/GristDoc.ts

@@ -11,7 +11,7 @@ import {CodeEditorPanel} from 'app/client/components/CodeEditorPanel';
 import * as commands from 'app/client/components/commands';
 import {CursorPos} from 'app/client/components/Cursor';
 import {CursorMonitor, ViewCursorPos} from "app/client/components/CursorMonitor";
-import {DocComm, DocUserAction} from 'app/client/components/DocComm';
+import {DocComm, DocUsageMessage, DocUserAction} from 'app/client/components/DocComm';
 import * as DocConfigTab from 'app/client/components/DocConfigTab';
 import {Drafts} from "app/client/components/Drafts";
 import {EditorMonitor} from "app/client/components/EditorMonitor";
@@ -61,7 +61,8 @@ import {LocalPlugin} from "app/common/plugin";
 import {StringUnion} from 'app/common/StringUnion';
 import {TableData} from 'app/common/TableData';
 import {DocStateComparison} from 'app/common/UserAPI';
-import {Computed, dom, Emitter, Holder, IDisposable, IDomComponent, Observable, styled, subscribe, toKo} from 'grainjs';
+import {bundleChanges, Computed, dom, Emitter, Holder, IDisposable, IDomComponent, Observable,
+        styled, subscribe, toKo} from 'grainjs';
 import * as ko from 'knockout';
 import cloneDeepWith = require('lodash/cloneDeepWith');
 import isEqual = require('lodash/isEqual');
@@ -298,6 +299,8 @@ export class GristDoc extends DisposableWithEvents {
 
     this.listenTo(app.comm, 'docUserAction', this.onDocUserAction);
 
+    this.listenTo(app.comm, 'docUsage', this.onDocUsageMessage);
+
     this.autoDispose(DocConfigTab.create({gristDoc: this}));
 
     this.rightPanelTool = Computed.create(this, (use) => this._getToolContent(use(this._rightPanelTool)));
@@ -482,6 +485,19 @@ export class GristDoc extends DisposableWithEvents {
     }
   }
 
+  /**
+   * Process usage and product received from the server by updating their respective
+   * observables.
+   */
+  public onDocUsageMessage(message: DocUsageMessage) {
+    if (!this.docComm.isActionFromThisDoc(message)) { return; }
+
+    bundleChanges(() => {
+      this.docPageModel.updateCurrentDocUsage(message.data.docUsage);
+      this.docPageModel.currentProduct.set(message.data.product ?? null);
+    });
+  }
+
   public getTableModel(tableId: string): DataTableModel {
     return this.docModel.dataTables[tableId];
   }

app/client/components/SiteUsageBanner.ts

@@ -1,7 +1,7 @@
 import {Banner, buildBannerMessage} from 'app/client/components/Banner';
 import {buildUpgradeMessage} from 'app/client/components/DocumentUsage';
 import {sessionStorageBoolObs} from 'app/client/lib/localStorageObs';
-import {HomeModel} from 'app/client/models/HomeModel';
+import {AppModel} from 'app/client/models/AppModel';
 import {isFreeProduct} from 'app/common/Features';
 import {isOwner} from 'app/common/roles';
 import {Disposable, dom, makeTestId, Observable} from 'grainjs';
@@ -9,15 +9,15 @@ import {Disposable, dom, makeTestId, Observable} from 'grainjs';
 const testId = makeTestId('test-site-usage-banner-');
 
 export class SiteUsageBanner extends Disposable {
-  private readonly _currentOrg = this._homeModel.app.currentOrg;
-  private readonly _currentOrgUsage = this._homeModel.currentOrgUsage;
+  private readonly _currentOrg = this._app.currentOrg;
+  private readonly _currentOrgUsage = this._app.currentOrgUsage;
   private readonly _product = this._currentOrg?.billingAccount?.product;
-  private readonly _currentUser = this._homeModel.app.currentValidUser;
+  private readonly _currentUser = this._app.currentValidUser;
 
   // Session storage observable. Set to false to dismiss the banner for the session.
   private _showApproachingLimitBannerPref?: Observable<boolean>;
 
-  constructor(private _homeModel: HomeModel) {
+  constructor(private _app: AppModel) {
     super();
 
     if (this._currentUser && isOwner(this._currentOrg)) {

app/client/models/AppModel.ts

@@ -4,11 +4,13 @@ import {reportError, setErrorNotifier} from 'app/client/models/errors';
 import {urlState} from 'app/client/models/gristUrlState';
 import {Notifier} from 'app/client/models/NotifyModel';
 import {getFlavor, ProductFlavor} from 'app/client/ui/CustomThemes';
+import {OrgUsageSummary} from 'app/common/DocUsage';
 import {Features} from 'app/common/Features';
 import {GristLoadConfig} from 'app/common/gristUrls';
 import {FullUser} from 'app/common/LoginSessionAPI';
 import {LocalPlugin} from 'app/common/plugin';
 import {UserPrefs} from 'app/common/Prefs';
+import {isOwner} from 'app/common/roles';
 import {getTagManagerScript} from 'app/common/tagManager';
 import {getGristConfig} from 'app/common/urlUtils';
 import {getOrgName, Organization, OrgError, UserAPI, UserAPIImpl} from 'app/common/UserAPI';
@@ -60,6 +62,7 @@ export interface AppModel {
 
   currentOrg: Organization|null;        // null if no access to currentSubdomain
   currentOrgName: string;               // Our best guess for human-friendly name.
+  currentOrgUsage: Observable<OrgUsageSummary|null>;
   isPersonal: boolean;                  // Is it a personal site?
   isTeamSite: boolean;                  // Is it a team site?
   orgError?: OrgError;                  // If currentOrg is null, the error that caused it.
@@ -70,6 +73,8 @@ export interface AppModel {
   pageType: Observable<PageType>;
 
   notifier: Notifier;
+
+  refreshOrgUsage(): Promise<void>;
 }
 
 export class TopAppModelImpl extends Disposable implements TopAppModel {
@@ -182,6 +187,8 @@ export class AppModelImpl extends Disposable implements AppModel {
   // Figure out the org name, or blank if details are unavailable.
   public readonly currentOrgName = getOrgNameOrGuest(this.currentOrg, this.currentUser);
 
+  public readonly currentOrgUsage: Observable<OrgUsageSummary|null> = Observable.create(this, null);
+
   public readonly isPersonal = Boolean(this.currentOrg?.owner);
   public readonly isTeamSite = Boolean(this.currentOrg) && !this.isPersonal;
 
@@ -206,6 +213,23 @@ export class AppModelImpl extends Disposable implements AppModel {
     this._recordSignUpIfIsNewUser();
   }
 
+  /**
+   * Fetch and update the current org's usage.
+   */
+  public async refreshOrgUsage() {
+    const currentOrg = this.currentOrg;
+    if (!isOwner(currentOrg)) {
+      // Note: getOrgUsageSummary already checks for owner access; we do an early return
+      // here to skip making unnecessary API calls.
+      return;
+    }
+
+    const usage = await this.api.getOrgUsageSummary(currentOrg.id);
+    if (!this.isDisposed()) {
+      this.currentOrgUsage.set(usage);
+    }
+  }
+
   /**
    * If the current user is a new user, record a sign-up event via Google Tag Manager.
    */

app/client/models/DocPageModel.ts

@@ -18,6 +18,7 @@ import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow';
 import {delay} from 'app/common/delay';
 import {OpenDocMode, UserOverride} from 'app/common/DocListAPI';
 import {FilteredDocUsageSummary} from 'app/common/DocUsage';
+import {Product} from 'app/common/Features';
 import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls';
 import {getReconnectTimeout} from 'app/common/gutil';
 import {canEdit, isOwner} from 'app/common/roles';
@@ -46,6 +47,12 @@ export interface DocPageModel {
   currentDoc: Observable<DocInfo|null>;
   currentDocUsage: Observable<FilteredDocUsageSummary|null>;
 
+  /**
+   * Initially set to the product referenced by `currentDoc`, and updated whenever `currentDoc`
+   * changes, or a doc usage message is received from the server.
+   */
+  currentProduct: Observable<Product|null>;
+
   // This block is to satisfy previous interface, but usable as this.currentDoc.get().id, etc.
   currentDocId: Observable<string|undefined>;
   currentWorkspace: Observable<Workspace|null>;
@@ -90,6 +97,12 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
   public readonly currentDoc = Observable.create<DocInfo|null>(this, null);
   public readonly currentDocUsage = Observable.create<FilteredDocUsageSummary|null>(this, null);
 
+  /**
+   * Initially set to the product referenced by `currentDoc`, and updated whenever `currentDoc`
+   * changes, or a doc usage message is received from the server.
+   */
+  public readonly currentProduct = Observable.create<Product|null>(this, null);
+
   public readonly currentUrlId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.urlId : undefined);
   public readonly currentDocId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.id : undefined);
   public readonly currentWorkspace = Computed.create(this, this.currentDoc, (use, doc) => doc && doc.workspace);
@@ -146,6 +159,14 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
         }
       }
     }));
+
+    this.autoDispose(this.currentOrg.addListener((org) => {
+      // Whenever the current doc is updated, set the current product to be the
+      // one referenced by the updated doc.
+      if (org?.billingAccount?.product.name !== this.currentProduct.get()?.name) {
+        this.currentProduct.set(org?.billingAccount?.product ?? null);
+      }
+    }));
   }
 
   public createLeftPane(leftPanelOpen: Observable<boolean>) {

app/client/models/HomeModel.ts

@@ -7,7 +7,6 @@ import {AppModel, reportError} from 'app/client/models/AppModel';
 import {reportMessage, UserError} from 'app/client/models/errors';
 import {urlState} from 'app/client/models/gristUrlState';
 import {ownerName} from 'app/client/models/WorkspaceInfo';
-import {OrgUsageSummary} from 'app/common/DocUsage';
 import {IHomePage} from 'app/common/gristUrls';
 import {isLongerThan} from 'app/common/gutil';
 import {SortPref, UserOrgPrefs, ViewPref} from 'app/common/Prefs';
@@ -76,8 +75,6 @@ export interface HomeModel {
   // user isn't allowed to create a doc.
   newDocWorkspace: Observable<Workspace|null|"unsaved">;
 
-  currentOrgUsage: Observable<OrgUsageSummary|null>;
-
   createWorkspace(name: string): Promise<void>;
   renameWorkspace(id: number, name: string): Promise<void>;
   deleteWorkspace(id: number, forever: boolean): Promise<void>;
@@ -158,8 +155,6 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings
     wss.every((ws) => ws.isSupportWorkspace || ws.docs.length === 0) &&
     Boolean(use(this.newDocWorkspace))));
 
-  public readonly currentOrgUsage: Observable<OrgUsageSummary|null> = Observable.create(this, null);
-
   private _userOrgPrefs = Observable.create<UserOrgPrefs|undefined>(this, this._app.currentOrg?.userOrgPrefs);
 
   constructor(private _app: AppModel, clientScope: ClientScope) {
@@ -193,7 +188,7 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings
     const importSources = ImportSourceElement.fromArray(pluginManager.pluginsList);
     this.importSources.set(importSources);
 
-    this._updateCurrentOrgUsage().catch(reportError);
+    this._app.refreshOrgUsage().catch(reportError);
   }
 
   // Accessor for the AppModel containing this HomeModel.
@@ -386,14 +381,6 @@ export class HomeModelImpl extends Disposable implements HomeModel, ViewSettings
       await this._app.api.updateOrg('current', {userOrgPrefs: org.userOrgPrefs});
     }
   }
-
-  private async _updateCurrentOrgUsage() {
-    const currentOrg = this.app.currentOrg;
-    if (!roles.isOwner(currentOrg)) { return; }
-
-    const api = this.app.api;
-    this.currentOrgUsage.set(await api.getOrgUsageSummary(currentOrg.id));
-  }
 }
 
 // Check if active product allows just a single workspace.

app/client/ui/AppUI.ts

@@ -105,7 +105,7 @@ function pagePanelsHome(owner: IDisposableOwner, appModel: AppModel, app: App) {
     },
     headerMain: createTopBarHome(appModel),
     contentMain: createDocMenu(pageModel),
-    contentTop: dom.create(SiteUsageBanner, pageModel),
+    contentTop: dom.create(SiteUsageBanner, appModel),
   });
 }
 

app/client/ui/BillingPage.ts

@@ -1,3 +1,4 @@
+import {SiteUsageBanner} from 'app/client/components/SiteUsageBanner';
 import {beaconOpenMessage} from 'app/client/lib/helpScout';
 import {AppModel, reportError} from 'app/client/models/AppModel';
 import {BillingModel, BillingModelImpl, ISubscriptionModel} from 'app/client/models/BillingModel';
@@ -51,6 +52,8 @@ export class BillingPage extends Disposable {
 
   constructor(private _appModel: AppModel) {
     super();
+
+    this._appModel.refreshOrgUsage().catch(reportError);
   }
 
   public buildDom() {
@@ -70,7 +73,8 @@ export class BillingPage extends Disposable {
             content: leftPanelBasic(this._appModel, panelOpen),
           },
           headerMain: this._createTopBarBilling(),
-          contentMain: this.buildCurrentPageDom()
+          contentMain: this.buildCurrentPageDom(),
+          contentTop: dom.create(SiteUsageBanner, this._appModel),
         });
       }
     });

app/client/ui/setupPage.ts

@@ -1,10 +1,14 @@
+import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {AppModel, TopAppModelImpl} from 'app/client/models/AppModel';
 import {setUpErrorHandling} from 'app/client/models/errors';
 import {buildSnackbarDom} from 'app/client/ui/NotifyUI';
 import {addViewportTag} from 'app/client/ui/viewport';
 import {attachCssRootVars} from 'app/client/ui2018/cssVars';
+import {BaseAPI} from 'app/common/BaseAPI';
 import {dom, DomContents} from 'grainjs';
 
+const G = getBrowserGlobals('document', 'window');
+
 /**
  * Sets up error handling and global styles, and replaces the DOM body with
  * the result of calling `buildPage`.
@@ -14,6 +18,12 @@ export function setupPage(buildPage: (appModel: AppModel) => DomContents) {
   const topAppModel = TopAppModelImpl.create(null, {});
   attachCssRootVars(topAppModel.productFlavor);
   addViewportTag();
+
+  // Add globals needed by test utils.
+  G.window.gristApp = {
+    testNumPendingApiRequests: () => BaseAPI.numPendingRequests(),
+  };
+
   dom.update(document.body, dom.maybe(topAppModel.appObs, (appModel) => [
     buildPage(appModel),
     buildSnackbarDom(appModel.notifier, appModel),

app/client/ui/tooltips.ts

@@ -6,9 +6,10 @@
  */
 
 import {prepareForTransition} from 'app/client/ui/transitions';
-import {testId} from 'app/client/ui2018/cssVars';
+import {colors, testId} from 'app/client/ui2018/cssVars';
+import {IconName} from 'app/client/ui2018/IconList';
 import {icon} from 'app/client/ui2018/icons';
-import {dom, DomContents, DomElementMethod, styled} from 'grainjs';
+import {dom, DomContents, DomElementArg, DomElementMethod, styled} from 'grainjs';
 import Popper from 'popper.js';
 
 export interface ITipOptions {
@@ -193,6 +194,34 @@ export function tooltipCloseButton(ctl: ITooltipControl): HTMLElement {
   );
 }
 
+/**
+ * Renders an icon that shows a tooltip with the specified `tipContent` on hover.
+ */
+export function iconTooltip(
+  iconName: IconName,
+  tipContent: ITooltipContentFunc,
+  ...domArgs: DomElementArg[]
+) {
+  return cssIconTooltip(iconName,
+    hoverTooltip(tipContent, {
+      openDelay: 0,
+      closeDelay: 0,
+      openOnClick: true,
+    }),
+    ...domArgs,
+  );
+}
+
+/**
+ * Renders an info icon that shows a tooltip with the specified `tipContent` on hover.
+ */
+export function infoTooltip(tipContent: DomContents, ...domArgs: DomElementArg[]) {
+  return iconTooltip('Info',
+    () => cssInfoTooltipBody(tipContent),
+    ...domArgs,
+  );
+}
+
 const cssTooltip = styled('div', `
   position: absolute;
   z-index: 5000;      /* should be higher than a modal */
@@ -225,3 +254,15 @@ const cssTooltipCloseButton = styled('div', `
     --icon-color: black;
   }
 `);
+
+const cssIconTooltip = styled(icon, `
+  height: 12px;
+  width: 12px;
+  background-color: ${colors.slate};
+  flex-shrink: 0;
+`);
+
+const cssInfoTooltipBody = styled('div', `
+  text-align: left;
+  max-width: 200px;
+`);

app/common/DocLimits.ts

@@ -62,7 +62,7 @@ export function getDataLimitRatio(
  * Maps `dataLimitStatus` status to an integer and returns it; larger integer
  * values indicate a more "severe" status.
  *
- * Useful for relatively comparing the severity of two statuses.
+ * Useful for relatively comparing the severity of two DataLimitStatus values.
  */
 export function getSeverity(dataLimitStatus: DataLimitStatus): number {
   switch (dataLimitStatus) {

app/server/lib/ActiveDoc.ts

@@ -50,7 +50,7 @@ import {
   getUsageRatio,
 } from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
-import {Features} from 'app/common/Features';
+import {Product} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
 import {parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
@@ -135,9 +135,19 @@ const REMOVE_UNUSED_ATTACHMENTS_INTERVAL_MS = 60 * 60 * 1000;
 // Apply the UpdateCurrentTime user action every hour
 const UPDATE_CURRENT_TIME_INTERVAL_MS = 60 * 60 * 1000;
 
+// Measure and broadcast data size every 5 minutes
+const UPDATE_DATA_SIZE_INTERVAL_MS = 5 * 60 * 1000;
+
 // A hook for dependency injection.
 export const Deps = {ACTIVEDOC_TIMEOUT};
 
+interface UpdateUsageOptions {
+  // Whether usage should be synced to the home database. Defaults to true.
+  syncUsageToDatabase?: boolean;
+  // Whether usage should be broadcast to all doc clients. Defaults to true.
+  broadcastUsageToClients?: boolean;
+}
+
 /**
  * Represents an active document with the given name. The document isn't actually open until
  * either .loadDoc() or .createEmptyDoc() is called.
@@ -184,11 +194,9 @@ export class ActiveDoc extends EventEmitter {
                                     // initialized.  True on success.
   private _fullyLoaded: boolean = false;  // Becomes true once all columns are loaded/computed.
   private _lastMemoryMeasurement: number = 0;    // Timestamp when memory was last measured.
-  private _lastDataSizeMeasurement: number = 0;  // Timestamp when dbstat data size was last measured.
-  private _lastDataLimitStatus?: DataLimitStatus;
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
   private _docUsage: DocumentUsage|null = null;
-  private _productFeatures?: Features;
+  private _product?: Product;
   private _gracePeriodStart: Date|null = null;
   private _isForkOrSnapshot: boolean = false;
 
@@ -208,6 +216,11 @@ export class ActiveDoc extends EventEmitter {
       () => this._applyUserActions(makeExceptionalDocSession('system'), [["UpdateCurrentTime"]]),
       UPDATE_CURRENT_TIME_INTERVAL_MS,
     ),
+    // Measure and broadcast data size every 5 minutes
+    setInterval(
+      () => this._checkDataSizeLimitRatio(makeExceptionalDocSession('system')),
+      UPDATE_DATA_SIZE_INTERVAL_MS,
+    ),
   ];
 
   constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
@@ -217,7 +230,7 @@ export class ActiveDoc extends EventEmitter {
     if (_options?.safeMode) { this._recoveryMode = true; }
     if (_options?.doc) {
       const {gracePeriodStart, workspace, usage} = _options.doc;
-      this._productFeatures = workspace.org.billingAccount?.product.features;
+      this._product = workspace.org.billingAccount?.product;
       this._gracePeriodStart = gracePeriodStart;
 
       if (!this._isForkOrSnapshot) {
@@ -234,7 +247,6 @@ export class ActiveDoc extends EventEmitter {
          *
          * TODO: Revisit this later and patch up the loophole. */
         this._docUsage = usage;
-        this._lastDataLimitStatus = this.dataLimitStatus;
       }
     }
     this._docManager = docManager;
@@ -282,25 +294,25 @@ export class ActiveDoc extends EventEmitter {
   public get rowLimitRatio(): number {
     return getUsageRatio(
       this._docUsage?.rowCount,
-      this._productFeatures?.baseMaxRowsPerDocument
+      this._product?.features.baseMaxRowsPerDocument
     );
   }
 
   public get dataSizeLimitRatio(): number {
     return getUsageRatio(
       this._docUsage?.dataSizeBytes,
-      this._productFeatures?.baseMaxDataSizePerDocument
+      this._product?.features.baseMaxDataSizePerDocument
     );
   }
 
   public get dataLimitRatio(): number {
-    return getDataLimitRatio(this._docUsage, this._productFeatures);
+    return getDataLimitRatio(this._docUsage, this._product?.features);
   }
 
   public get dataLimitStatus(): DataLimitStatus {
     return getDataLimitStatus({
       docUsage: this._docUsage,
-      productFeatures: this._productFeatures,
+      productFeatures: this._product?.features,
       gracePeriodStart: this._gracePeriodStart,
     });
   }
@@ -446,15 +458,16 @@ export class ActiveDoc extends EventEmitter {
       for (const interval of this._intervals) {
         clearInterval(interval);
       }
+      // We'll defer syncing usage until everything is calculated.
+      const usageOptions = {syncUsageToDatabase: false, broadcastUsageToClients: false};
 
       // Remove expired attachments, i.e. attachments that were soft deleted a while ago. This
       // needs to happen periodically, and doing it here means we can guarantee that it happens
       // even if the doc is only ever opened briefly, without having to slow down startup.
-      const removeAttachmentsPromise = this.removeUnusedAttachments(true, {syncUsageToDatabase: false});
+      const removeAttachmentsPromise = this.removeUnusedAttachments(true, usageOptions);
 
-      // Update data size as well. We'll schedule a sync to the database once both this and the
-      // above promise settle.
-      const updateDataSizePromise = this._updateDataSize({syncUsageToDatabase: false});
+      // Update data size; we'll be syncing both it and attachments size to the database soon.
+      const updateDataSizePromise = this._updateDataSize(usageOptions);
 
       try {
         await removeAttachmentsPromise;
@@ -1440,11 +1453,14 @@ export class ActiveDoc extends EventEmitter {
    * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago.
    * @param options.syncUsageToDatabase: if true, schedule an update to the usage column of the docs table, if
    * any unused attachments were soft-deleted. defaults to true.
+   * @param options.broadcastUsageToClients: if true, broadcast updated doc usage to all clients, if
+   * any unused attachments were soft-deleted. defaults to true.
    */
-  public async removeUnusedAttachments(expiredOnly: boolean, options: {syncUsageToDatabase?: boolean} = {}) {
-    const {syncUsageToDatabase = true} = options;
+  public async removeUnusedAttachments(expiredOnly: boolean, options?: UpdateUsageOptions) {
     const hadChanges = await this.updateUsedAttachmentsIfNeeded();
-    if (hadChanges) { await this._updateAttachmentsSize({syncUsageToDatabase}); }
+    if (hadChanges) {
+      await this._updateAttachmentsSize(options);
+    }
     const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly);
     if (rowIds.length) {
       const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
@@ -1508,35 +1524,20 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public async updateRowCount(rowCount: number, docSession: OptDocSession | null) {
-    this._updateDocUsage({rowCount});
+    // Up-to-date row counts are included in every DocUserAction, so we can skip broadcasting here.
+    await this._updateDocUsage({rowCount}, {broadcastUsageToClients: false});
     log.rawInfo('Sandbox row count', {...this.getLogMeta(docSession), rowCount});
     await this._checkDataLimitRatio();
 
-    // Calculating data size is potentially expensive, so by default measure it at most once every 5 minutes.
-    // Measure it after every change if the user is currently being warned specifically about
-    // approaching or exceeding the data size limit but not the row count limit,
-    // because we don't need to warn about both limits at the same time.
-    let checkDataSizePeriod = 5 * 60;
+    // Calculating data size is potentially expensive, so skip calculating it unless the
+    // user is currently being warned specifically about approaching or exceeding the data
+    // size limit, but not the row count limit; we don't need to warn about both limits at
+    // the same time.
     if (
       this.dataSizeLimitRatio > APPROACHING_LIMIT_RATIO && this.rowLimitRatio <= APPROACHING_LIMIT_RATIO ||
       this.dataSizeLimitRatio > 1.0 && this.rowLimitRatio <= 1.0
     ) {
-      checkDataSizePeriod = 0;
-    }
-
-    const now = Date.now();
-    if (now - this._lastDataSizeMeasurement > checkDataSizePeriod * 1000) {
-      this._lastDataSizeMeasurement = now;
-
-      // When the data size isn't critically high so we're only measuring it infrequently,
-      // do it in the background so we don't delay responding to the client.
-      // When it's being measured after every change, wait for it to finish to avoid race conditions
-      // from multiple measurements and updates happening concurrently.
-      if (checkDataSizePeriod === 0) {
       await this._checkDataSizeLimitRatio(docSession);
-      } else {
-        this._checkDataSizeLimitRatio(docSession).catch(e => console.error(e));
-      }
     }
   }
 
@@ -1692,43 +1693,43 @@ export class ActiveDoc extends EventEmitter {
 
   /**
    * Applies all metrics from `usage` to the current document usage state.
-   * Syncs updated usage to the home database by default, unless
-   * `options.syncUsageToDatabase` is set to false.
+   *
+   * Allows specifying `options` for toggling whether usage is synced to
+   * the home database and/or broadcast to clients.
    */
-  private _updateDocUsage(
-    usage: Partial<DocumentUsage>,
-    options: {
-      syncUsageToDatabase?: boolean
-    } = {}
-  ) {
-    const {syncUsageToDatabase = true} = options;
+  private async _updateDocUsage(usage: Partial<DocumentUsage>, options: UpdateUsageOptions = {}) {
+    const {syncUsageToDatabase = true, broadcastUsageToClients = true} = options;
+    const oldStatus = this.dataLimitStatus;
     this._docUsage = {...(this._docUsage || {}), ...usage};
-    if (this._lastDataLimitStatus === this.dataLimitStatus) {
-      // If status is unchanged, there's no need to sync usage to the database, as it currently
-      // won't result in any noticeable difference to site usage banners. On shutdown, we'll
-      // still schedule a sync so that the latest usage is persisted.
-      return;
-    }
-
-    const lastStatus = this._lastDataLimitStatus;
-    this._lastDataLimitStatus = this.dataLimitStatus;
-    if (!syncUsageToDatabase) { return; }
-
-    // If status decreased, we'll want to update usage in the DB with minimal delay, so that site
-    // usage banners show up-to-date statistics. If status increased or stayed the same, we'll
-    // schedule a delayed update, since it's less critical for such banners to update quickly
-    // when usage grows.
-    const didStatusDecrease = (
-      lastStatus !== undefined &&
-      getSeverity(this.dataLimitStatus) < getSeverity(lastStatus)
-    );
+    if (syncUsageToDatabase) {
+      /* If status decreased, we'll update usage in the database with minimal delay, so site usage
+       * banners show up-to-date statistics. If status increased or stayed the same, we'll schedule
+       * a delayed update, since it's less critical for banners to update immediately. */
+      const didStatusDecrease = getSeverity(this.dataLimitStatus) < getSeverity(oldStatus);
       this._syncDocUsageToDatabase(didStatusDecrease);
     }
+    if (broadcastUsageToClients) {
+      await this._broadcastDocUsageToClients();
+    }
+  }
 
   private _syncDocUsageToDatabase(minimizeDelay = false) {
     this._docManager.storageManager.scheduleUsageUpdate(this._docName, this._docUsage, minimizeDelay);
   }
 
+  private async _broadcastDocUsageToClients() {
+    if (this.muted || this.docClients.clientCount() === 0) { return; }
+
+    await this.docClients.broadcastDocMessage(
+      null,
+      'docUsage',
+      {docUsage: this.getDocUsageSummary(), product: this._product},
+      async (session, data) => {
+        return {...data, docUsage: await this.getFilteredDocUsageSummary(session)};
+      },
+    );
+  }
+
   private async _updateGracePeriodStart(gracePeriodStart: Date | null) {
     this._gracePeriodStart = gracePeriodStart;
     if (!this._isForkOrSnapshot) {
@@ -1758,15 +1759,14 @@ export class ActiveDoc extends EventEmitter {
   }
 
   /**
-   * Calculates the total data size in bytes and sets it in _docUsage. Schedules
-   * a sync to the database, unless `options.syncUsageToDatabase` is set to false.
+   * Calculates the total data size in bytes, sets it in _docUsage, and returns it.
    *
-   * Returns the calculated data size.
+   * Allows specifying `options` for toggling whether usage is synced to
+   * the home database and/or broadcast to clients.
    */
-  private async _updateDataSize(options: {syncUsageToDatabase?: boolean} = {}): Promise<number> {
-    const {syncUsageToDatabase = true} = options;
+  private async _updateDataSize(options?: UpdateUsageOptions): Promise<number> {
     const dataSizeBytes = await this.docStorage.getDataSize();
-    this._updateDocUsage({dataSizeBytes}, {syncUsageToDatabase});
+    await this._updateDocUsage({dataSizeBytes}, options);
     return dataSizeBytes;
   }
 
@@ -1930,7 +1930,7 @@ export class ActiveDoc extends EventEmitter {
       const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT;
       this._inactivityTimer.setDelay(closeTimeout);
       this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`);
-      this._initializeDocUsageIfNeeded(docSession);
+      void this._initializeDocUsage(docSession);
     } catch (err) {
       this._fullyLoaded = true;
       if (!this._shuttingDown) {
@@ -1971,18 +1971,24 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
-  private _initializeDocUsageIfNeeded(docSession: OptDocSession) {
-    // TODO: Broadcast a message to clients after usage is fully calculated.
+  private async _initializeDocUsage(docSession: OptDocSession) {
+    const promises: Promise<unknown>[] = [];
+    // We'll defer syncing/broadcasting usage until everything is calculated.
+    const options = {syncUsageToDatabase: false, broadcastUsageToClients: false};
     if (this._docUsage?.dataSizeBytes === undefined) {
-      this._updateDataSize().catch(e => {
-        this._log.warn(docSession, 'failed to update data size', e);
-      });
+      promises.push(this._updateDataSize(options));
     }
-
     if (this._docUsage?.attachmentsSizeBytes === undefined) {
-      this._updateAttachmentsSize().catch(e => {
-        this._log.warn(docSession, 'failed to update attachments size', e);
-      });
+      promises.push(this._updateAttachmentsSize(options));
+    }
+    if (promises.length === 0) { return; }
+
+    try {
+      await Promise.all(promises);
+      this._syncDocUsageToDatabase();
+      await this._broadcastDocUsageToClients();
+    } catch (e) {
+      this._log.warn(docSession, 'failed to initialize doc usage', e);
     }
   }
 
@@ -2089,7 +2095,7 @@ export class ActiveDoc extends EventEmitter {
    * limits to be exceeded.
    */
   private async _isUploadSizeBelowLimit(uploadSizeBytes: number): Promise<boolean> {
-    const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
+    const maxSize = this._product?.features.baseMaxAttachmentsBytesPerDocument;
     if (!maxSize) { return true; }
 
     let currentSize = this._docUsage?.attachmentsSizeBytes;
@@ -2098,15 +2104,14 @@ export class ActiveDoc extends EventEmitter {
   }
 
   /**
-   * Calculates the total attachments size in bytes and sets it in _docUsage. Schedules
-   * a sync to the database, unless `options.syncUsageToDatabase` is set to false.
+   * Calculates the total attachments size in bytes, sets it in _docUsage, and returns it.
    *
-   * Returns the calculated attachments size.
+   * Allows specifying `options` for toggling whether usage is synced to
+   * the home database and/or broadcast to clients.
    */
-  private async _updateAttachmentsSize(options: {syncUsageToDatabase?: boolean} = {}): Promise<number> {
-    const {syncUsageToDatabase = true} = options;
+  private async _updateAttachmentsSize(options?: UpdateUsageOptions): Promise<number> {
     const attachmentsSizeBytes = await this.docStorage.getTotalAttachmentFileSizes();
-    this._updateDocUsage({attachmentsSizeBytes}, {syncUsageToDatabase});
+    await this._updateDocUsage({attachmentsSizeBytes}, options);
     return attachmentsSizeBytes;
   }
 }

test/nbrowser/gristUtils.ts

@@ -806,11 +806,10 @@ export async function waitAppFocus(yesNo: boolean = true): Promise<void> {
  * has been processed.
  * @param optTimeout: Timeout in ms, defaults to 2000.
  */
- // TODO: waits also for api requests (both to home server or doc worker) to be resolved (maybe
- // requires to track requests in app/common/UserAPI)
 export async function waitForServer(optTimeout: number = 2000) {
   await driver.wait(() => driver.executeScript(
-    "return !window.gristApp.comm.hasActiveRequests() && window.gristApp.testNumPendingApiRequests() === 0",
+    "return (!window.gristApp.comm || !window.gristApp.comm.hasActiveRequests())"
+    + " && window.gristApp.testNumPendingApiRequests() === 0",
     optTimeout,
     "Timed out waiting for server requests to complete"
   ));