(core) Add attachment and data size usage

Summary:
Adds attachment and data size to the usage section of
the raw data page. Also makes in-document usage banners
update as user actions are applied, causing them to be
hidden/shown or updated based on the current state of
the document.

Test Plan: Browser tests.

Reviewers: jarek

Reviewed By: jarek

Subscribers: alexmojaki

Differential Revision: https://phab.getgrist.com/D3395

app/client/components/DocComm.ts

@@ -5,6 +5,7 @@ import {ActionGroup} from 'app/common/ActionGroup';
 import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI';
 import {DocAction, UserAction} from 'app/common/DocActions';
 import {OpenLocalDocResult} from 'app/common/DocListAPI';
+import {DocUsage} from 'app/common/DocUsage';
 import {docUrl} from 'app/common/urlUtils';
 import {Events as BackboneEvents} from 'backbone';
 import {Disposable, Emitter} from 'grainjs';
@@ -17,6 +18,7 @@ export interface DocUserAction extends CommMessage {
   data: {
     docActions: DocAction[];
     actionGroup: ActionGroup;
+    docUsage: DocUsage;
     error?: string;
   };
 }

app/client/components/DocUsageBanner.ts

@@ -1,4 +1,4 @@
-import {buildUpgradeMessage, getLimitStatusMessage} from 'app/client/components/DocumentUsage';
+import {buildLimitStatusMessage, buildUpgradeMessage} from 'app/client/components/DocumentUsage';
 import {sessionStorageBoolObs} from 'app/client/lib/localStorageObs';
 import {DocPageModel} from 'app/client/models/DocPageModel';
 import {colors, isNarrowScreenObs} from 'app/client/ui2018/cssVars';
@@ -66,7 +66,7 @@ export class DocUsageBanner extends Disposable {
           cssBannerMessage(
             cssWhiteIcon('Idea'),
             cssLightlyBoldedText(
-              getLimitStatusMessage('approachingLimit', features),
+              buildLimitStatusMessage('approachingLimit', features),
               ' ',
               buildUpgradeMessage(org.access === 'owners'),
               testId('text'),
@@ -99,7 +99,7 @@ export class DocUsageBanner extends Disposable {
                 }
 
                 return [
-                  getLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features),
+                  buildLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features),
                   ' ',
                   buildUpgradeMessage(isOwner),
                 ];

app/client/components/DocumentUsage.ts

@@ -1,13 +1,14 @@
 import {DocPageModel} from 'app/client/models/DocPageModel';
+import {urlState} from 'app/client/models/gristUrlState';
 import {docListHeader} from 'app/client/ui/DocMenuCss';
 import {colors, mediaXSmall} from 'app/client/ui2018/cssVars';
 import {icon} from 'app/client/ui2018/icons';
 import {cssLink} from 'app/client/ui2018/links';
 import {loadingSpinner} from 'app/client/ui2018/loaders';
+import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/DocUsage';
 import {Features} from 'app/common/Features';
 import {commonUrls} from 'app/common/gristUrls';
 import {capitalizeFirstWord} from 'app/common/gutil';
-import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/Usage';
 import {Computed, Disposable, dom, DomContents, DomElementArg, makeTestId, styled} from 'grainjs';
 
 const testId = makeTestId('test-doc-usage-');
@@ -15,6 +16,12 @@ const testId = makeTestId('test-doc-usage-');
 // Default used by the progress bar to visually indicate row usage.
 const DEFAULT_MAX_ROWS = 20000;
 
+// Default used by the progress bar to visually indicate data size usage.
+const DEFAULT_MAX_DATA_SIZE = DEFAULT_MAX_ROWS * 2 * 1024; // 40MB (2KiB per row)
+
+// Default used by the progress bar to visually indicate attachments size usage.
+const DEFAULT_MAX_ATTACHMENTS_SIZE = 1 * 1024 * 1024 * 1024; // 1GiB
+
 const ACCESS_DENIED_MESSAGE = 'Usage statistics are only available to users with '
   + 'full access to the document data.';
 
@@ -25,6 +32,8 @@ export class DocumentUsage extends Disposable {
   private readonly _currentDoc = this._docPageModel.currentDoc;
   private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus;
   private readonly _rowCount = this._docPageModel.rowCount;
+  private readonly _dataSizeBytes = this._docPageModel.dataSizeBytes;
+  private readonly _attachmentsSizeBytes = this._docPageModel.attachmentsSizeBytes;
 
   private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => {
     return doc?.workspace.org ?? null;
@@ -47,20 +56,65 @@ export class DocumentUsage extends Disposable {
       };
     });
 
-  private readonly _isLoading: Computed<boolean> =
+  private readonly _dataSizeMetrics: Computed<MetricOptions | null> =
-    Computed.create(this, this._currentDoc, this._rowCount, (_use, doc, rowCount) => {
+    Computed.create(this, this._currentOrg, this._dataSizeBytes, (_use, org, dataSize) => {
-      return doc === null || rowCount === 'pending';
+      const features = org?.billingAccount?.product.features;
+      if (!features || typeof dataSize !== 'number') { return null; }
+
+      const {baseMaxDataSizePerDocument: maxSize} = features;
+      // Invalid data size limits are currently treated as if they are undefined.
+      const maxValue = maxSize && maxSize > 0 ? maxSize : undefined;
+      return {
+        name: 'Data Size',
+        currentValue: dataSize,
+        maximumValue: maxValue ?? DEFAULT_MAX_DATA_SIZE,
+        unit: 'MB',
+        shouldHideLimits: maxValue === undefined,
+        formatValue: (val) => {
+          // To display a nice, round number for `maximumValue`, we first convert
+          // to KiBs (base-2), and then convert to MBs (base-10). Normally, we wouldn't
+          // mix conversions like this, but to display something that matches our
+          // marketing limits (e.g. 40MB for Pro plan), we need to bend conversions a bit.
+          return ((val / 1024) / 1000).toFixed(2);
+        },
+      };
+    });
+
+  private readonly _attachmentsSizeMetrics: Computed<MetricOptions | null> =
+    Computed.create(this, this._currentOrg, this._attachmentsSizeBytes, (_use, org, attachmentsSize) => {
+      const features = org?.billingAccount?.product.features;
+      if (!features || typeof attachmentsSize !== 'number') { return null; }
+
+      const {baseMaxAttachmentsBytesPerDocument: maxSize} = features;
+      // Invalid attachments size limits are currently treated as if they are undefined.
+      const maxValue = maxSize && maxSize > 0 ? maxSize : undefined;
+      return {
+        name: 'Attachments Size',
+        currentValue: attachmentsSize,
+        maximumValue: maxValue ?? DEFAULT_MAX_ATTACHMENTS_SIZE,
+        unit: 'GB',
+        shouldHideLimits: maxValue === undefined,
+        formatValue: (val) => (val / (1024 * 1024 * 1024)).toFixed(2),
+      };
     });
 
+  private readonly _isLoading: Computed<boolean> =
+    Computed.create(
+      this, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes,
+      (_use, doc, rowCount, dataSize, attachmentsSize) => {
+        return !doc || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'pending');
+      }
+    );
+
   private readonly _isAccessDenied: Computed<boolean | null> =
     Computed.create(
-      this, this._isLoading, this._currentDoc, this._rowCount,
+      this, this._isLoading, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes,
-      (_use, isLoading, doc, rowCount) => {
+      (_use, isLoading, doc, rowCount, dataSize, attachmentsSize) => {
         if (isLoading) { return null; }
 
         const {access} = doc!.workspace.org;
         const isPublicUser = access === 'guests' || access === null;
-        return isPublicUser || rowCount === 'hidden';
+        return isPublicUser || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'hidden');
       }
     );
 
@@ -91,7 +145,9 @@ export class DocumentUsage extends Disposable {
       if (!org || !status) { return null; }
 
       return buildMessage([
-        getLimitStatusMessage(status, org.billingAccount?.product.features),
+        buildLimitStatusMessage(status, org.billingAccount?.product.features, {
+          disableRawDataLink: true
+        }),
         ' ',
         buildUpgradeMessage(org.access === 'owners')
       ]);
@@ -104,18 +160,79 @@ export class DocumentUsage extends Disposable {
         dom.maybe(this._rowMetrics, (metrics) =>
           buildUsageMetric(metrics, testId('rows')),
         ),
+        dom.maybe(this._dataSizeMetrics, (metrics) =>
+          buildUsageMetric(metrics, testId('data-size')),
+        ),
+        dom.maybe(this._attachmentsSizeMetrics, (metrics) =>
+          buildUsageMetric(metrics, testId('attachments-size')),
+        ),
         testId('metrics'),
       ),
     );
   }
 }
 
-function buildMessage(message: DomContents) {
+export function buildLimitStatusMessage(
-  return cssWarningMessage(
+  status: NonNullable<DataLimitStatus>,
-    cssIcon('Idea'),
+  features?: Features,
-    cssLightlyBoldedText(message, testId('message-text')),
+  options: {
-    testId('message'),
+    disableRawDataLink?: boolean;
-  );
+  } = {}
+) {
+  const {disableRawDataLink = false} = options;
+  switch (status) {
+    case 'approachingLimit': {
+      return [
+        'This document is ',
+        disableRawDataLink ? 'approaching' : buildRawDataPageLink('approaching'),
+        ' free plan limits.'
+      ];
+    }
+    case 'gracePeriod': {
+      const gracePeriodDays = features?.gracePeriodDays;
+      if (!gracePeriodDays) {
+        return [
+          'Document limits ',
+          disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
+          '.'
+        ];
+      }
+
+      return [
+        'Document limits ',
+        disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
+        `. In ${gracePeriodDays} days, this document will be read-only.`
+      ];
+    }
+    case 'deleteOnly': {
+      return [
+        'This document ',
+        disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
+        ' free plan limits and is now read-only, but you can delete rows.'
+      ];
+    }
+  }
+}
+
+export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') {
+  if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; }
+
+  const upgradeLinkText = 'start your 30-day free trial of the Pro plan.';
+  return [
+    variant === 'short' ? null : 'For higher limits, ',
+    buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText),
+  ];
+}
+
+function buildUpgradeLink(linkText: string) {
+  return cssUnderlinedLink(linkText, {
+    href: commonUrls.plans,
+    target: '_blank',
+  });
+}
+
+function buildRawDataPageLink(linkText: string) {
+  return cssUnderlinedLink(linkText, urlState().setLinkUrl({docPage: 'data'}));
 }
 
 interface MetricOptions {
@@ -126,6 +243,7 @@ interface MetricOptions {
   unit?: string;
   // If true, limits will always be hidden, even if `maximumValue` is a positive number.
   shouldHideLimits?: boolean;
+  formatValue?(value: number): string;
 }
 
 /**
@@ -134,7 +252,14 @@ interface MetricOptions {
  * close `currentValue` is to hitting `maximumValue`.
  */
 function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
-  const {name, currentValue, maximumValue, unit, shouldHideLimits} = options;
+  const {
+    name,
+    currentValue,
+    maximumValue,
+    unit,
+    shouldHideLimits,
+    formatValue = (val) => val.toString(),
+  } = options;
   const ratioUsed = currentValue / (maximumValue || Infinity);
   const percentUsed = Math.min(100, Math.floor(ratioUsed * 100));
   return cssUsageMetric(
@@ -150,8 +275,8 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
       ),
     ),
     dom('div',
-      currentValue
+      formatValue(currentValue)
-        + (shouldHideLimits || !maximumValue ? '' : ' of ' + maximumValue)
+        + (shouldHideLimits || !maximumValue ? '' : ' of ' + formatValue(maximumValue))
         + (unit ? ` ${unit}` : ''),
       testId('value'),
     ),
@@ -159,38 +284,12 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
   );
 }
 
-export function getLimitStatusMessage(status: NonNullable<DataLimitStatus>, features?: Features): string {
+function buildMessage(message: DomContents) {
-  switch (status) {
+  return cssWarningMessage(
-    case 'approachingLimit': {
+    cssIcon('Idea'),
-      return 'This document is approaching free plan limits.';
+    cssLightlyBoldedText(message, testId('message-text')),
-    }
+    testId('message'),
-    case 'gracePeriod': {
+  );
-      const gracePeriodDays = features?.gracePeriodDays;
-      if (!gracePeriodDays) { return 'Document limits exceeded.'; }
-
-      return `Document limits exceeded. In ${gracePeriodDays} days, this document will be read-only.`;
-    }
-    case 'deleteOnly': {
-      return 'This document exceeded free plan limits and is now read-only, but you can delete rows.';
-    }
-  }
-}
-
-export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') {
-  if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; }
-
-  const upgradeLinkText = 'start your 30-day free trial of the Pro plan.';
-  return [
-    variant === 'short' ? null : 'For higher limits, ',
-    buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText),
-  ];
-}
-
-export function buildUpgradeLink(linkText: string) {
-  return cssUnderlinedLink(linkText, {
-    href: commonUrls.plans,
-    target: '_blank',
-  });
 }
 
 const cssLightlyBoldedText = styled('div', `
@@ -233,13 +332,8 @@ const cssUsageMetrics = styled('div', `
   display: flex;
   flex-wrap: wrap;
   margin-top: 24px;
-  gap: 56px;
+  row-gap: 24px;
-
+  column-gap: 54px;
-  @media ${mediaXSmall} {
-    & {
-      gap: 24px;
-    }
-  }
 `);
 
 const cssUsageMetric = styled('div', `

app/client/components/GristDoc.ts

@@ -478,9 +478,7 @@ export class GristDoc extends DisposableWithEvents {
       if (schemaUpdated) {
         this.trigger('schemaUpdateAction', docActions);
       }
-      if (typeof actionGroup.rowCount === "number") {
+      this.docPageModel.updateDocUsage(message.data.docUsage);
-        this.docPageModel.rowCount.set(actionGroup.rowCount);
-      }
     }
   }
 

app/client/models/DocPageModel.ts

@@ -17,10 +17,10 @@ import {confirmModal} from 'app/client/ui2018/modals';
 import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow';
 import {delay} from 'app/common/delay';
 import {OpenDocMode, UserOverride} from 'app/common/DocListAPI';
+import {AttachmentsSize, DataLimitStatus, DataSize, DocUsage, RowCount} from 'app/common/DocUsage';
 import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls';
 import {getReconnectTimeout} from 'app/common/gutil';
 import {canEdit} from 'app/common/roles';
-import {DataLimitStatus, RowCount} from 'app/common/Usage';
 import {Document, NEW_DOCUMENT_CODE, Organization, UserAPI, Workspace} from 'app/common/UserAPI';
 import {Holder, Observable, subscribe} from 'grainjs';
 import {Computed, Disposable, dom, DomArg, DomElementArg} from 'grainjs';
@@ -66,13 +66,16 @@ export interface DocPageModel {
 
   gristDoc: Observable<GristDoc|null>;             // Instance of GristDoc once it exists.
 
+  dataLimitStatus: Observable<DataLimitStatus>;
   rowCount: Observable<RowCount>;
-  dataLimitStatus: Observable<DataLimitStatus|undefined>;
+  dataSizeBytes: Observable<DataSize>;
+  attachmentsSizeBytes: Observable<AttachmentsSize>;
 
   createLeftPane(leftPanelOpen: Observable<boolean>): DomArg;
   renameDoc(value: string): Promise<void>;
   updateCurrentDoc(urlId: string, openMode: OpenDocMode): Promise<Document>;
   refreshCurrentDoc(doc: DocInfo): Promise<Document>;
+  updateDocUsage(docUsage: DocUsage): void;
 }
 
 export interface ImportSource {
@@ -109,8 +112,10 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
   // Observable set to the instance of GristDoc once it's created.
   public readonly gristDoc = Observable.create<GristDoc|null>(this, null);
 
+  public readonly dataLimitStatus = Observable.create<DataLimitStatus>(this, null);
   public readonly rowCount = Observable.create<RowCount>(this, 'pending');
-  public readonly dataLimitStatus = Observable.create<DataLimitStatus|undefined>(this, null);
+  public readonly dataSizeBytes = Observable.create<DataSize>(this, 'pending');
+  public readonly attachmentsSizeBytes = Observable.create<AttachmentsSize>(this, 'pending');
 
   // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the
   // URL, and when it changes, we need to re-open.
@@ -203,6 +208,13 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
     return urlState().pushUrl(nextState, {avoidReload: true, ...options});
   }
 
+  public updateDocUsage(docUsage: DocUsage) {
+    this.rowCount.set(docUsage.rowCount);
+    this.dataLimitStatus.set(docUsage.dataLimitStatus);
+    this.dataSizeBytes.set(docUsage.dataSizeBytes);
+    this.attachmentsSizeBytes.set(docUsage.attachmentsSizeBytes);
+  }
+
   private _onOpenError(err: Error) {
     if (err instanceof CancelledError) {
       // This means that we started loading a new doc before the previous one finished loading.
@@ -260,8 +272,9 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
       doc.userOverride = openDocResponse.userOverride || null;
       this.currentDoc.set({...doc});
     }
-    this.rowCount.set(openDocResponse.rowCount);
+    if (openDocResponse.docUsage) {
-    this.dataLimitStatus.set(openDocResponse.dataLimitStatus);
+      this.updateDocUsage(openDocResponse.docUsage);
+    }
     const gdModule = await gristDocModulePromise;
     const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier);
     flow.checkIfCancelled();

app/client/ui2018/menus.ts

@@ -537,6 +537,10 @@ const cssCmdKey = styled('span', `
   margin-left: 16px;
   color: ${colors.slate};
   margin-right: -12px;
+
+  .${weasel.cssMenuItem.className}-sel > & {
+    color: ${colors.lightGrey};
+  }
 `);
 
 const cssAnnotateMenuItem = styled('span', `

app/common/ActionGroup.ts

@@ -25,5 +25,4 @@ export interface ActionGroup extends MinimalActionGroup {
   user: string;
   primaryAction: string;  // The name of the first user action in the ActionGroup.
   internal: boolean;      // True if it is inappropriate to log/undo the action.
-  rowCount?: number;
 }

app/common/DocListAPI.ts

@@ -1,8 +1,8 @@
 import {MinimalActionGroup} from 'app/common/ActionGroup';
 import {TableDataAction} from 'app/common/DocActions';
+import {DocUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {StringUnion} from 'app/common/StringUnion';
-import {DataLimitStatus, RowCount} from 'app/common/Usage';
 import {FullUser} from 'app/common/UserAPI';
 
 // Possible flavors of items in a list of documents.
@@ -43,10 +43,9 @@ export interface OpenLocalDocResult {
   clientId: string;  // the docFD is meaningful only in the context of this session
   doc: {[tableId: string]: TableDataAction};
   log: MinimalActionGroup[];
-  rowCount: RowCount;
   recoveryMode?: boolean;
   userOverride?: UserOverride;
-  dataLimitStatus?: DataLimitStatus;
+  docUsage?: DocUsage;
 }
 
 export interface UserOverride {

app/common/DocUsage.ts

@@ -0,0 +1,29 @@
+import {ApiError} from 'app/common/ApiError';
+
+export interface DocUsage {
+  dataLimitStatus: DataLimitStatus;
+  rowCount: RowCount;
+  dataSizeBytes: DataSize;
+  attachmentsSizeBytes: AttachmentsSize;
+}
+
+type NumberOrStatus = number | 'hidden' | 'pending';
+
+export type RowCount = NumberOrStatus;
+
+export type DataSize = NumberOrStatus;
+
+export type AttachmentsSize = NumberOrStatus;
+
+export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null;
+
+export type NonHidden<T> = Exclude<T, 'hidden'>;
+
+// Ratio of usage at which we start telling users that they're approaching limits.
+export const APPROACHING_LIMIT_RATIO = 0.9;
+
+export class LimitExceededError extends ApiError {
+  constructor(message: string) {
+    super(message, 413);
+  }
+}

app/common/Usage.ts

@@ -1,6 +0,0 @@
-export type RowCount = number | 'hidden' | 'pending';
-
-export type DataLimitStatus = null | 'approachingLimit' | 'gracePeriod' | 'deleteOnly';
-
-// Ratio of the row/data size limit where we tell users that they're approaching the limit.
-export const APPROACHING_LIMIT_RATIO = 0.9;

app/common/gutil.ts

@@ -935,3 +935,16 @@ export function assertIsDefined<T>(name: string, value: T): asserts value is Non
     throw new Error(`Expected '${name}' to be defined, but received ${value}`);
   }
 }
+
+/**
+ * Calls function `fn`, passes any thrown errors to function `recover`, and finally calls `fn`
+ * once more if `recover` doesn't throw.
+ */
+ export async function retryOnce<T>(fn: () => Promise<T>, recover: (e: unknown) => Promise<void>): Promise<T> {
+  try {
+    return await fn();
+  } catch (e) {
+    await recover(e);
+    return await fn();
+  }
+}

app/server/lib/ActiveDoc.ts

@@ -35,16 +35,24 @@ import {
 import {DocData} from 'app/common/DocData';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentSettings} from 'app/common/DocumentSettings';
+import {
+  APPROACHING_LIMIT_RATIO,
+  AttachmentsSize,
+  DataLimitStatus,
+  DataSize,
+  DocUsage,
+  LimitExceededError,
+  NonHidden,
+  RowCount
+} from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {Features} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
-import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
+import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
-import {canEdit} from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord} from 'app/common/TableData';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
-import {APPROACHING_LIMIT_RATIO, DataLimitStatus, RowCount} from 'app/common/Usage';
 import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
 import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
@@ -173,8 +181,9 @@ export class ActiveDoc extends EventEmitter {
   private _lastMemoryMeasurement: number = 0;    // Timestamp when memory was last measured.
   private _lastDataSizeMeasurement: number = 0;  // Timestamp when dbstat data size was last measured.
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
-  private _rowCount: RowCount = 'pending';
+  private _rowCount: NonHidden<RowCount> = 'pending';
-  private _dataSize?: number;
+  private _dataSize: NonHidden<DataSize> = 'pending';
+  private _attachmentsSize: NonHidden<AttachmentsSize> = 'pending';
   private _productFeatures?: Features;
   private _gracePeriodStart: Date|null = null;
 
@@ -245,8 +254,8 @@ export class ActiveDoc extends EventEmitter {
   public get isShuttingDown(): boolean { return this._shuttingDown; }
 
   public get rowLimitRatio() {
-    if (!this._rowLimit || this._rowLimit <= 0 || typeof this._rowCount !== 'number') {
+    if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') {
-      // Invalid row limits are currently treated as if they are undefined.
+      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
       return 0;
     }
 
@@ -254,8 +263,8 @@ export class ActiveDoc extends EventEmitter {
   }
 
   public get dataSizeLimitRatio() {
-    if (!this._dataSizeLimit || this._dataSizeLimit <= 0 || !this._dataSize) {
+    if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') {
-      // Invalid data size limits are currently treated as if they are undefined.
+      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
       return 0;
     }
 
@@ -282,15 +291,17 @@ export class ActiveDoc extends EventEmitter {
     return null;
   }
 
-  public async getRowCount(docSession: OptDocSession): Promise<RowCount> {
+  public get docUsage(): DocUsage {
-    const hasFullReadAccess = await this._granularAccess.canReadEverything(docSession);
+    return {
-    const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
+      dataLimitStatus: this.dataLimitStatus,
-    return hasFullReadAccess && hasEditRole ? this._rowCount : 'hidden';
+      rowCount: this._rowCount,
+      dataSizeBytes: this._dataSize,
+      attachmentsSizeBytes: this._attachmentsSize,
+    };
   }
 
-  public async getDataLimitStatus(docSession: OptDocSession): Promise<DataLimitStatus> {
+  public getFilteredDocUsage(docSession: OptDocSession): Promise<DocUsage> {
-    const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
+    return this._granularAccess.filterDocUsage(docSession, this.docUsage);
-    return hasEditRole ? this.dataLimitStatus : null;
   }
 
   public async getUserOverride(docSession: OptDocSession) {
@@ -692,10 +703,31 @@ export class ActiveDoc extends EventEmitter {
     const userId = getDocSessionUserId(docSession);
     const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId));
     try {
-      await this._checkDocAttachmentsLimit(upload);
+      // We'll assert that the upload won't cause limits to be exceeded, retrying once after
+      // soft-deleting any unused attachments.
+      await retryOnce(
+        () => this._assertUploadSizeBelowLimit(upload),
+        async (e) => {
+          if (!(e instanceof LimitExceededError)) { throw e; }
+
+          // Check if any attachments are unused and can be soft-deleted to reduce the existing
+          // total size. We could do this from the beginning, but updateUsedAttachmentsIfNeeded
+          // is potentially expensive, so this optimises for the common case of not exceeding the limit.
+          const hadChanges = await this.updateUsedAttachmentsIfNeeded();
+          if (hadChanges) {
+            await this._updateAttachmentsSize();
+          } else {
+            // No point in retrying if nothing changed.
+            throw new LimitExceededError("Exceeded attachments limit for document");
+          }
+        }
+      );
       const userActions: UserAction[] = await Promise.all(
         upload.files.map(file => this._prepAttachment(docSession, file)));
       const result = await this.applyUserActions(docSession, userActions);
+      this._updateAttachmentsSize().catch(e => {
+        this._log.warn(docSession, 'failed to update attachments size', e);
+      });
       return result.retValues;
     } finally {
       await globalUploadSet.cleanup(uploadId);
@@ -1352,7 +1384,7 @@ export class ActiveDoc extends EventEmitter {
    * so that undo can 'undelete' attachments.
    * Returns true if any changes were made, i.e. some row(s) of _grist_Attachments were updated.
    */
-  public async updateUsedAttachments() {
+  public async updateUsedAttachmentsIfNeeded() {
     const changes = await this.docStorage.scanAttachmentsForUsageChanges();
     if (!changes.length) {
       return false;
@@ -1371,7 +1403,8 @@ export class ActiveDoc extends EventEmitter {
    * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago.
    */
   public async removeUnusedAttachments(expiredOnly: boolean) {
-    await this.updateUsedAttachments();
+    const hadChanges = await this.updateUsedAttachmentsIfNeeded();
+    if (hadChanges) { await this._updateAttachmentsSize(); }
     const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly);
     if (rowIds.length) {
       const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
@@ -1807,6 +1840,10 @@ export class ActiveDoc extends EventEmitter {
       const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT;
       this._inactivityTimer.setDelay(closeTimeout);
       this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`);
+      // TODO: Initialize data and attachments size from Document.usage once it's available.
+      this._updateAttachmentsSize().catch(e => {
+        this._log.warn(docSession, 'failed to update attachments size', e);
+      });
     } catch (err) {
       this._fullyLoaded = true;
       if (!this._shuttingDown) {
@@ -1935,35 +1972,32 @@ export class ActiveDoc extends EventEmitter {
   /**
    * Throw an error if the provided upload would exceed the total attachment filesize limit for this document.
    */
-  private async _checkDocAttachmentsLimit(upload: UploadInfo) {
+  private async _assertUploadSizeBelowLimit(upload: UploadInfo) {
-    const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
-    if (!maxSize) {
-      // This document has no limit, nothing to check.
-      return;
-    }
-
     // Minor flaw: while we don't double-count existing duplicate files in the total size,
     // we don't check here if any of the uploaded files already exist and could be left out of the calculation.
-    const totalAddedSize = sum(upload.files.map(f => f.size));
+    const uploadSizeBytes = sum(upload.files.map(f => f.size));
+    if (await this._isUploadSizeBelowLimit(uploadSizeBytes)) { return; }
 
-    // Returns true if this upload won't bring the total over the limit.
+    // TODO probably want a nicer error message here.
-    const isOK = async () => (await this.docStorage.getTotalAttachmentFileSizes()) + totalAddedSize <= maxSize;
+    throw new LimitExceededError("Exceeded attachments limit for document");
+  }
 
-    if (await isOK()) {
+  /**
-      return;
+   * Returns true if an upload with size `uploadSizeBytes` won't cause attachment size
-    }
+   * limits to be exceeded.
+   */
+  private async _isUploadSizeBelowLimit(uploadSizeBytes: number): Promise<boolean> {
+    const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
+    if (!maxSize) { return true; }
 
-    // Looks like the limit is being exceeded.
+    const currentSize = this._attachmentsSize !== 'pending'
-    // Check if any attachments are unused and can be soft-deleted to reduce the existing total size.
+      ? this._attachmentsSize
-    // We could do this from the beginning, but updateUsedAttachments is potentially expensive,
+      : await this.docStorage.getTotalAttachmentFileSizes();
-    // so this optimises the common case of not exceeding the limit.
+    return currentSize + uploadSizeBytes <= maxSize;
-    // updateUsedAttachments returns true if there were any changes. Otherwise there's no point checking isOK again.
+  }
-    if (await this.updateUsedAttachments() && await isOK()) {
-      return;
-    }
 
-    // TODO probably want a nicer error message here.
+  private async _updateAttachmentsSize() {
-    throw new Error("Exceeded attachments limit for document");
+    this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes();
   }
 }
 
@@ -1989,3 +2023,8 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table
   }
   return tableRefs[tableRowIndex];
 }
+
+// Helper that returns true if `limit` is set to a valid, positive number.
+function isEnforceableLimit(limit: number | undefined): limit is number {
+  return limit !== undefined && limit > 0;
+}

app/server/lib/DocApi.ts

@@ -248,7 +248,7 @@ export class DocWorkerApi {
 
     // Mostly for testing
     this._app.post('/api/docs/:docId/attachments/updateUsed', canEdit, withDoc(async (activeDoc, req, res) => {
-      await activeDoc.updateUsedAttachments();
+      await activeDoc.updateUsedAttachmentsIfNeeded();
       res.json(null);
     }));
     this._app.post('/api/docs/:docId/attachments/removeUnused', isOwner, withDoc(async (activeDoc, req, res) => {

app/server/lib/DocManager.ts

@@ -9,6 +9,7 @@ import {ApiError} from 'app/common/ApiError';
 import {mapSetOrClear} from 'app/common/AsyncCreate';
 import {BrowserSettings} from 'app/common/BrowserSettings';
 import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
+import {DocUsage} from 'app/common/DocUsage';
 import {Invite} from 'app/common/sharing';
 import {tbind} from 'app/common/tbind';
 import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
@@ -313,24 +314,28 @@ export class DocManager extends EventEmitter {
         }
       }
 
-      const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([
+      const [metaTables, recentActions, userOverride] = await Promise.all([
         activeDoc.fetchMetaTables(docSession),
         activeDoc.getRecentMinimalActions(docSession),
         activeDoc.getUserOverride(docSession),
-        activeDoc.getRowCount(docSession),
-        activeDoc.getDataLimitStatus(docSession),
       ]);
 
-      const result = {
+      let docUsage: DocUsage | undefined;
+      try {
+        docUsage = await activeDoc.getFilteredDocUsage(docSession);
+      } catch (e) {
+        log.warn("DocManager.openDoc failed to get doc usage", e);
+      }
+
+      const result: OpenLocalDocResult = {
         docFD: docSession.fd,
         clientId: docSession.client.clientId,
         doc: metaTables,
         log: recentActions,
         recoveryMode: activeDoc.recoveryMode,
         userOverride,
-        rowCount,
+        docUsage,
-        dataLimitStatus,
+      };
-      } as OpenLocalDocResult;
 
       if (!activeDoc.muted) {
         this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));

app/server/lib/DocStorage.ts

@@ -1238,9 +1238,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
 
   /**
    * Returns the total number of bytes used for storing attachments that haven't been soft-deleted.
-   * May be stale if ActiveDoc.updateUsedAttachments isn't called first.
+   * May be stale if ActiveDoc.updateUsedAttachmentsIfNeeded isn't called first.
    */
-  public async getTotalAttachmentFileSizes() {
+  public async getTotalAttachmentFileSizes(): Promise<number> {
     const result = await this.get(`
       SELECT SUM(len) AS total
       FROM (
@@ -1258,7 +1258,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
         GROUP BY meta.fileIdent
       )
     `);
-    return result!.total as number;
+    return result!.total ?? 0;
   }
 
   /**
@@ -1307,7 +1307,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
 
   /**
    * Return row IDs of unused attachments in _grist_Attachments.
-   * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachments.
+   * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachmentsIfNeeded.
    * @param expiredOnly: if true, only return attachments where timeDeleted is at least
    *                     ATTACHMENTS_EXPIRY_DAYS days ago.
    */

app/server/lib/GranularAccess.ts

@@ -10,6 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app
 import { TableDataAction, UserAction } from 'app/common/DocActions';
 import { DocData } from 'app/common/DocData';
 import { UserOverride } from 'app/common/DocListAPI';
+import { DocUsage } from 'app/common/DocUsage';
 import { normalizeEmail } from 'app/common/emails';
 import { ErrorWithCode } from 'app/common/ErrorWithCode';
 import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause';
@@ -101,6 +102,12 @@ const SURPRISING_ACTIONS = new Set([
 // Actions we'll allow unconditionally for now.
 const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);
 
+interface DocUpdateMessage {
+  actionGroup: ActionGroup;
+  docActions: DocAction[];
+  docUsage: DocUsage;
+}
+
 /**
  * Granular access for a single bundle, in different phases.
  */
@@ -108,7 +115,7 @@ export interface GranularAccessForBundle {
   canApplyBundle(): Promise<void>;
   appliedBundle(): Promise<void>;
   finishedBundle(): Promise<void>;
-  sendDocUpdateForBundle(actionGroup: ActionGroup): Promise<void>;
+  sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise<void>;
 }
 
 /**
@@ -362,13 +369,16 @@ export class GranularAccess implements GranularAccessForBundle {
   /**
    * Filter an ActionGroup to be sent to a client.
    */
-  public async filterActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<ActionGroup> {
+  public async filterActionGroup(
-    if (await this.allowActionGroup(docSession, actionGroup)) { return actionGroup; }
+    docSession: OptDocSession,
+    actionGroup: ActionGroup,
+    options: {role?: Role | null} = {}
+  ): Promise<ActionGroup> {
+    if (await this.allowActionGroup(docSession, actionGroup, options)) { return actionGroup; }
     // For now, if there's any nuance at all, suppress the summary and description.
     const result: ActionGroup = { ...actionGroup };
     result.actionSummary = createEmptyActionSummary();
     result.desc = '';
-    result.rowCount = undefined;
     return result;
   }
 
@@ -376,8 +386,33 @@ export class GranularAccess implements GranularAccessForBundle {
    * Check whether an ActionGroup can be sent to the client.  TODO: in future, we'll want
    * to filter acceptable parts of ActionGroup, rather than denying entirely.
    */
-  public async allowActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<boolean> {
+  public async allowActionGroup(
-    return this.canReadEverything(docSession);
+    docSession: OptDocSession,
+    _actionGroup: ActionGroup,
+    options: {role?: Role | null} = {}
+  ): Promise<boolean> {
+    return this.canReadEverything(docSession, options);
+  }
+
+  /**
+   * Filter DocUsage to be sent to a client.
+   */
+  public async filterDocUsage(
+    docSession: OptDocSession,
+    docUsage: DocUsage,
+    options: {role?: Role | null} = {}
+  ): Promise<DocUsage> {
+    const result: DocUsage = { ...docUsage };
+    const role = options.role ?? await this.getNominalAccess(docSession);
+    const hasEditRole = canEdit(role);
+    if (!hasEditRole) { result.dataLimitStatus = null; }
+    const hasFullReadAccess = await this.canReadEverything(docSession);
+    if (!hasEditRole || !hasFullReadAccess) {
+      result.rowCount = 'hidden';
+      result.dataSizeBytes = 'hidden';
+      result.attachmentsSizeBytes = 'hidden';
+    }
+    return result;
   }
 
   /**
@@ -577,8 +612,11 @@ export class GranularAccess implements GranularAccessForBundle {
    * Check whether user can read everything in document.  Checks both home-level and doc-level
    * permissions.
    */
-  public async canReadEverything(docSession: OptDocSession): Promise<boolean> {
+  public async canReadEverything(
-    const access = await this.getNominalAccess(docSession);
+    docSession: OptDocSession,
+    options: {role?: Role | null} = {}
+  ): Promise<boolean> {
+    const access = options.role ?? await this.getNominalAccess(docSession);
     if (!canView(access)) { return false; }
     const permInfo = await this._getAccess(docSession);
     return this.getReadPermission(permInfo.getFullAccess()) === 'allow';
@@ -709,11 +747,11 @@ export class GranularAccess implements GranularAccessForBundle {
   /**
    * Broadcast document changes to all clients, with appropriate filtering.
    */
-  public async sendDocUpdateForBundle(actionGroup: ActionGroup) {
+  public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) {
     if (!this._activeBundle) { throw new Error('no active bundle'); }
     const { docActions, docSession } = this._activeBundle;
     const client = docSession && docSession.client || null;
-    const message = { actionGroup, docActions };
+    const message: DocUpdateMessage = { actionGroup, docActions, docUsage };
     await this._docClients.broadcastDocMessage(client, 'docUserAction',
                                                message,
                                                (_docSession) => this._filterDocUpdate(_docSession, message));
@@ -866,18 +904,18 @@ export class GranularAccess implements GranularAccessForBundle {
    * This filters a message being broadcast to all clients to be appropriate for one
    * particular client, if that client may need some material filtered out.
    */
-  private async _filterDocUpdate(docSession: OptDocSession, message: {
+  private async _filterDocUpdate(docSession: OptDocSession, message: DocUpdateMessage) {
-    actionGroup: ActionGroup,
-    docActions: DocAction[]
-  }) {
     if (!this._activeBundle) { throw new Error('no active bundle'); }
-    if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
+    const role = await this.getNominalAccess(docSession);
-      return message;
-    }
     const result = {
-      actionGroup: await this.filterActionGroup(docSession, message.actionGroup),
+      ...message,
-      docActions: await this.filterOutgoingDocActions(docSession, message.docActions),
+      docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}),
     };
+    if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
+      return result;
+    }
+    result.actionGroup = await this.filterActionGroup(docSession, message.actionGroup, {role});
+    result.docActions = await this.filterOutgoingDocActions(docSession, message.docActions);
     if (result.docActions.length === 0) { return null; }
     return result;
   }

app/server/lib/Sharing.ts

@@ -313,9 +313,8 @@ export class Sharing {
         internal,
       });
       actionGroup.actionSummary = actionSummary;
-      actionGroup.rowCount = sandboxActionBundle.rowCount;
       await accessControl.appliedBundle();
-      await accessControl.sendDocUpdateForBundle(actionGroup);
+      await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage);
       if (docSession) {
         docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0;
       }

test/server/lib/DocApi.ts

@@ -1725,7 +1725,7 @@ function testDocApi() {
       assert.equal(resp.status, 200);
 
       // Remove the not expired attachments (2 and 3).
-      // We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachments.
+      // We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded.
       resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
       assert.equal(resp.status, 200);
       await checkAttachmentIds([]);