(core) Add attachment and data size usage

Summary: Adds attachment and data size to the usage section of the raw data page. Also makes in-document usage banners update as user actions are applied, causing them to be hidden/shown or updated based on the current state of the document. Test Plan: Browser tests. Reviewers: jarek Reviewed By: jarek Subscribers: alexmojaki Differential Revision: https://phab.getgrist.com/D3395
2026-03-02 04:09:24 +00:00 · 2022-05-02 22:20:31 -07:00
parent f194d6861b
commit 1e42871cc9
18 changed files with 381 additions and 155 deletions
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -35,16 +35,24 @@ import {
 import {DocData} from 'app/common/DocData';
 import {DocSnapshots} from 'app/common/DocSnapshot';
 import {DocumentSettings} from 'app/common/DocumentSettings';
+import {
+  APPROACHING_LIMIT_RATIO,
+  AttachmentsSize,
+  DataLimitStatus,
+  DataSize,
+  DocUsage,
+  LimitExceededError,
+  NonHidden,
+  RowCount
+} from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {Features} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
-import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
+import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
-import {canEdit} from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord} from 'app/common/TableData';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
-import {APPROACHING_LIMIT_RATIO, DataLimitStatus, RowCount} from 'app/common/Usage';
 import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
 import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
@@ -173,8 +181,9 @@ export class ActiveDoc extends EventEmitter {
  private _lastMemoryMeasurement: number = 0;    // Timestamp when memory was last measured.
  private _lastDataSizeMeasurement: number = 0;  // Timestamp when dbstat data size was last measured.
  private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
-  private _rowCount: RowCount = 'pending';
-  private _dataSize?: number;
+  private _rowCount: NonHidden<RowCount> = 'pending';
+  private _dataSize: NonHidden<DataSize> = 'pending';
+  private _attachmentsSize: NonHidden<AttachmentsSize> = 'pending';
  private _productFeatures?: Features;
  private _gracePeriodStart: Date|null = null;

@@ -245,8 +254,8 @@ export class ActiveDoc extends EventEmitter {
  public get isShuttingDown(): boolean { return this._shuttingDown; }

  public get rowLimitRatio() {
-    if (!this._rowLimit || this._rowLimit <= 0 || typeof this._rowCount !== 'number') {
-      // Invalid row limits are currently treated as if they are undefined.
+    if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') {
+      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
      return 0;
    }

@@ -254,8 +263,8 @@ export class ActiveDoc extends EventEmitter {
  }

  public get dataSizeLimitRatio() {
-    if (!this._dataSizeLimit || this._dataSizeLimit <= 0 || !this._dataSize) {
-      // Invalid data size limits are currently treated as if they are undefined.
+    if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') {
+      // If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
      return 0;
    }

@@ -282,15 +291,17 @@ export class ActiveDoc extends EventEmitter {
    return null;
  }

-  public async getRowCount(docSession: OptDocSession): Promise<RowCount> {
-    const hasFullReadAccess = await this._granularAccess.canReadEverything(docSession);
-    const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
-    return hasFullReadAccess && hasEditRole ? this._rowCount : 'hidden';
+  public get docUsage(): DocUsage {
+    return {
+      dataLimitStatus: this.dataLimitStatus,
+      rowCount: this._rowCount,
+      dataSizeBytes: this._dataSize,
+      attachmentsSizeBytes: this._attachmentsSize,
+    };
  }

-  public async getDataLimitStatus(docSession: OptDocSession): Promise<DataLimitStatus> {
-    const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
-    return hasEditRole ? this.dataLimitStatus : null;
+  public getFilteredDocUsage(docSession: OptDocSession): Promise<DocUsage> {
+    return this._granularAccess.filterDocUsage(docSession, this.docUsage);
  }

  public async getUserOverride(docSession: OptDocSession) {
@@ -692,10 +703,31 @@ export class ActiveDoc extends EventEmitter {
    const userId = getDocSessionUserId(docSession);
    const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId));
    try {
-      await this._checkDocAttachmentsLimit(upload);
+      // We'll assert that the upload won't cause limits to be exceeded, retrying once after
+      // soft-deleting any unused attachments.
+      await retryOnce(
+        () => this._assertUploadSizeBelowLimit(upload),
+        async (e) => {
+          if (!(e instanceof LimitExceededError)) { throw e; }
+
+          // Check if any attachments are unused and can be soft-deleted to reduce the existing
+          // total size. We could do this from the beginning, but updateUsedAttachmentsIfNeeded
+          // is potentially expensive, so this optimises for the common case of not exceeding the limit.
+          const hadChanges = await this.updateUsedAttachmentsIfNeeded();
+          if (hadChanges) {
+            await this._updateAttachmentsSize();
+          } else {
+            // No point in retrying if nothing changed.
+            throw new LimitExceededError("Exceeded attachments limit for document");
+          }
+        }
+      );
      const userActions: UserAction[] = await Promise.all(
        upload.files.map(file => this._prepAttachment(docSession, file)));
      const result = await this.applyUserActions(docSession, userActions);
+      this._updateAttachmentsSize().catch(e => {
+        this._log.warn(docSession, 'failed to update attachments size', e);
+      });
      return result.retValues;
    } finally {
      await globalUploadSet.cleanup(uploadId);
@@ -1352,7 +1384,7 @@ export class ActiveDoc extends EventEmitter {
   * so that undo can 'undelete' attachments.
   * Returns true if any changes were made, i.e. some row(s) of _grist_Attachments were updated.
   */
-  public async updateUsedAttachments() {
+  public async updateUsedAttachmentsIfNeeded() {
    const changes = await this.docStorage.scanAttachmentsForUsageChanges();
    if (!changes.length) {
      return false;
@@ -1371,7 +1403,8 @@ export class ActiveDoc extends EventEmitter {
   * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago.
   */
  public async removeUnusedAttachments(expiredOnly: boolean) {
-    await this.updateUsedAttachments();
+    const hadChanges = await this.updateUsedAttachmentsIfNeeded();
+    if (hadChanges) { await this._updateAttachmentsSize(); }
    const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly);
    if (rowIds.length) {
      const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
@@ -1807,6 +1840,10 @@ export class ActiveDoc extends EventEmitter {
      const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT;
      this._inactivityTimer.setDelay(closeTimeout);
      this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`);
+      // TODO: Initialize data and attachments size from Document.usage once it's available.
+      this._updateAttachmentsSize().catch(e => {
+        this._log.warn(docSession, 'failed to update attachments size', e);
+      });
    } catch (err) {
      this._fullyLoaded = true;
      if (!this._shuttingDown) {
@@ -1935,35 +1972,32 @@ export class ActiveDoc extends EventEmitter {
  /**
   * Throw an error if the provided upload would exceed the total attachment filesize limit for this document.
   */
-  private async _checkDocAttachmentsLimit(upload: UploadInfo) {
-    const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
-    if (!maxSize) {
-      // This document has no limit, nothing to check.
-      return;
-    }
-
+  private async _assertUploadSizeBelowLimit(upload: UploadInfo) {
    // Minor flaw: while we don't double-count existing duplicate files in the total size,
    // we don't check here if any of the uploaded files already exist and could be left out of the calculation.
-    const totalAddedSize = sum(upload.files.map(f => f.size));
-
-    // Returns true if this upload won't bring the total over the limit.
-    const isOK = async () => (await this.docStorage.getTotalAttachmentFileSizes()) + totalAddedSize <= maxSize;
-
-    if (await isOK()) {
-      return;
-    }
-
-    // Looks like the limit is being exceeded.
-    // Check if any attachments are unused and can be soft-deleted to reduce the existing total size.
-    // We could do this from the beginning, but updateUsedAttachments is potentially expensive,
-    // so this optimises the common case of not exceeding the limit.
-    // updateUsedAttachments returns true if there were any changes. Otherwise there's no point checking isOK again.
-    if (await this.updateUsedAttachments() && await isOK()) {
-      return;
-    }
+    const uploadSizeBytes = sum(upload.files.map(f => f.size));
+    if (await this._isUploadSizeBelowLimit(uploadSizeBytes)) { return; }

    // TODO probably want a nicer error message here.
-    throw new Error("Exceeded attachments limit for document");
+    throw new LimitExceededError("Exceeded attachments limit for document");
+  }
+
+  /**
+   * Returns true if an upload with size `uploadSizeBytes` won't cause attachment size
+   * limits to be exceeded.
+   */
+  private async _isUploadSizeBelowLimit(uploadSizeBytes: number): Promise<boolean> {
+    const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
+    if (!maxSize) { return true; }
+
+    const currentSize = this._attachmentsSize !== 'pending'
+      ? this._attachmentsSize
+      : await this.docStorage.getTotalAttachmentFileSizes();
+    return currentSize + uploadSizeBytes <= maxSize;
+  }
+
+  private async _updateAttachmentsSize() {
+    this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes();
  }
 }

@@ -1989,3 +2023,8 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table
  }
  return tableRefs[tableRowIndex];
 }
+
+// Helper that returns true if `limit` is set to a valid, positive number.
+function isEnforceableLimit(limit: number | undefined): limit is number {
+  return limit !== undefined && limit > 0;
+}
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -248,7 +248,7 @@ export class DocWorkerApi {

    // Mostly for testing
    this._app.post('/api/docs/:docId/attachments/updateUsed', canEdit, withDoc(async (activeDoc, req, res) => {
-      await activeDoc.updateUsedAttachments();
+      await activeDoc.updateUsedAttachmentsIfNeeded();
      res.json(null);
    }));
    this._app.post('/api/docs/:docId/attachments/removeUnused', isOwner, withDoc(async (activeDoc, req, res) => {
--- a/app/server/lib/DocManager.ts
+++ b/app/server/lib/DocManager.ts
@@ -9,6 +9,7 @@ import {ApiError} from 'app/common/ApiError';
 import {mapSetOrClear} from 'app/common/AsyncCreate';
 import {BrowserSettings} from 'app/common/BrowserSettings';
 import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
+import {DocUsage} from 'app/common/DocUsage';
 import {Invite} from 'app/common/sharing';
 import {tbind} from 'app/common/tbind';
 import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
@@ -313,24 +314,28 @@ export class DocManager extends EventEmitter {
        }
      }

-      const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([
+      const [metaTables, recentActions, userOverride] = await Promise.all([
        activeDoc.fetchMetaTables(docSession),
        activeDoc.getRecentMinimalActions(docSession),
        activeDoc.getUserOverride(docSession),
-        activeDoc.getRowCount(docSession),
-        activeDoc.getDataLimitStatus(docSession),
      ]);

-      const result = {
+      let docUsage: DocUsage | undefined;
+      try {
+        docUsage = await activeDoc.getFilteredDocUsage(docSession);
+      } catch (e) {
+        log.warn("DocManager.openDoc failed to get doc usage", e);
+      }
+
+      const result: OpenLocalDocResult = {
        docFD: docSession.fd,
        clientId: docSession.client.clientId,
        doc: metaTables,
        log: recentActions,
        recoveryMode: activeDoc.recoveryMode,
        userOverride,
-        rowCount,
-        dataLimitStatus,
-      } as OpenLocalDocResult;
+        docUsage,
+      };

      if (!activeDoc.muted) {
        this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));
--- a/app/server/lib/DocStorage.ts
+++ b/app/server/lib/DocStorage.ts
@@ -1238,9 +1238,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {

  /**
   * Returns the total number of bytes used for storing attachments that haven't been soft-deleted.
-   * May be stale if ActiveDoc.updateUsedAttachments isn't called first.
+   * May be stale if ActiveDoc.updateUsedAttachmentsIfNeeded isn't called first.
   */
-  public async getTotalAttachmentFileSizes() {
+  public async getTotalAttachmentFileSizes(): Promise<number> {
    const result = await this.get(`
      SELECT SUM(len) AS total
      FROM (
@@ -1258,7 +1258,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
        GROUP BY meta.fileIdent
      )
    `);
-    return result!.total as number;
+    return result!.total ?? 0;
  }

  /**
@@ -1307,7 +1307,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {

  /**
   * Return row IDs of unused attachments in _grist_Attachments.
-   * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachments.
+   * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachmentsIfNeeded.
   * @param expiredOnly: if true, only return attachments where timeDeleted is at least
   *                     ATTACHMENTS_EXPIRY_DAYS days ago.
   */
--- a/app/server/lib/GranularAccess.ts
+++ b/app/server/lib/GranularAccess.ts
@@ -10,6 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app
 import { TableDataAction, UserAction } from 'app/common/DocActions';
 import { DocData } from 'app/common/DocData';
 import { UserOverride } from 'app/common/DocListAPI';
+import { DocUsage } from 'app/common/DocUsage';
 import { normalizeEmail } from 'app/common/emails';
 import { ErrorWithCode } from 'app/common/ErrorWithCode';
 import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause';
@@ -101,6 +102,12 @@ const SURPRISING_ACTIONS = new Set([
 // Actions we'll allow unconditionally for now.
 const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);

+interface DocUpdateMessage {
+  actionGroup: ActionGroup;
+  docActions: DocAction[];
+  docUsage: DocUsage;
+}
+
 /**
 * Granular access for a single bundle, in different phases.
 */
@@ -108,7 +115,7 @@ export interface GranularAccessForBundle {
  canApplyBundle(): Promise<void>;
  appliedBundle(): Promise<void>;
  finishedBundle(): Promise<void>;
-  sendDocUpdateForBundle(actionGroup: ActionGroup): Promise<void>;
+  sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise<void>;
 }

 /**
@@ -362,13 +369,16 @@ export class GranularAccess implements GranularAccessForBundle {
  /**
   * Filter an ActionGroup to be sent to a client.
   */
-  public async filterActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<ActionGroup> {
-    if (await this.allowActionGroup(docSession, actionGroup)) { return actionGroup; }
+  public async filterActionGroup(
+    docSession: OptDocSession,
+    actionGroup: ActionGroup,
+    options: {role?: Role | null} = {}
+  ): Promise<ActionGroup> {
+    if (await this.allowActionGroup(docSession, actionGroup, options)) { return actionGroup; }
    // For now, if there's any nuance at all, suppress the summary and description.
    const result: ActionGroup = { ...actionGroup };
    result.actionSummary = createEmptyActionSummary();
    result.desc = '';
-    result.rowCount = undefined;
    return result;
  }

@@ -376,8 +386,33 @@ export class GranularAccess implements GranularAccessForBundle {
   * Check whether an ActionGroup can be sent to the client.  TODO: in future, we'll want
   * to filter acceptable parts of ActionGroup, rather than denying entirely.
   */
-  public async allowActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<boolean> {
-    return this.canReadEverything(docSession);
+  public async allowActionGroup(
+    docSession: OptDocSession,
+    _actionGroup: ActionGroup,
+    options: {role?: Role | null} = {}
+  ): Promise<boolean> {
+    return this.canReadEverything(docSession, options);
+  }
+
+  /**
+   * Filter DocUsage to be sent to a client.
+   */
+  public async filterDocUsage(
+    docSession: OptDocSession,
+    docUsage: DocUsage,
+    options: {role?: Role | null} = {}
+  ): Promise<DocUsage> {
+    const result: DocUsage = { ...docUsage };
+    const role = options.role ?? await this.getNominalAccess(docSession);
+    const hasEditRole = canEdit(role);
+    if (!hasEditRole) { result.dataLimitStatus = null; }
+    const hasFullReadAccess = await this.canReadEverything(docSession);
+    if (!hasEditRole || !hasFullReadAccess) {
+      result.rowCount = 'hidden';
+      result.dataSizeBytes = 'hidden';
+      result.attachmentsSizeBytes = 'hidden';
+    }
+    return result;
  }

  /**
@@ -577,8 +612,11 @@ export class GranularAccess implements GranularAccessForBundle {
   * Check whether user can read everything in document.  Checks both home-level and doc-level
   * permissions.
   */
-  public async canReadEverything(docSession: OptDocSession): Promise<boolean> {
-    const access = await this.getNominalAccess(docSession);
+  public async canReadEverything(
+    docSession: OptDocSession,
+    options: {role?: Role | null} = {}
+  ): Promise<boolean> {
+    const access = options.role ?? await this.getNominalAccess(docSession);
    if (!canView(access)) { return false; }
    const permInfo = await this._getAccess(docSession);
    return this.getReadPermission(permInfo.getFullAccess()) === 'allow';
@@ -709,11 +747,11 @@ export class GranularAccess implements GranularAccessForBundle {
  /**
   * Broadcast document changes to all clients, with appropriate filtering.
   */
-  public async sendDocUpdateForBundle(actionGroup: ActionGroup) {
+  public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) {
    if (!this._activeBundle) { throw new Error('no active bundle'); }
    const { docActions, docSession } = this._activeBundle;
    const client = docSession && docSession.client || null;
-    const message = { actionGroup, docActions };
+    const message: DocUpdateMessage = { actionGroup, docActions, docUsage };
    await this._docClients.broadcastDocMessage(client, 'docUserAction',
                                               message,
                                               (_docSession) => this._filterDocUpdate(_docSession, message));
@@ -866,18 +904,18 @@ export class GranularAccess implements GranularAccessForBundle {
   * This filters a message being broadcast to all clients to be appropriate for one
   * particular client, if that client may need some material filtered out.
   */
-  private async _filterDocUpdate(docSession: OptDocSession, message: {
-    actionGroup: ActionGroup,
-    docActions: DocAction[]
-  }) {
+  private async _filterDocUpdate(docSession: OptDocSession, message: DocUpdateMessage) {
    if (!this._activeBundle) { throw new Error('no active bundle'); }
-    if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
-      return message;
-    }
+    const role = await this.getNominalAccess(docSession);
    const result = {
-      actionGroup: await this.filterActionGroup(docSession, message.actionGroup),
-      docActions: await this.filterOutgoingDocActions(docSession, message.docActions),
+      ...message,
+      docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}),
    };
+    if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
+      return result;
+    }
+    result.actionGroup = await this.filterActionGroup(docSession, message.actionGroup, {role});
+    result.docActions = await this.filterOutgoingDocActions(docSession, message.docActions);
    if (result.docActions.length === 0) { return null; }
    return result;
  }
--- a/app/server/lib/Sharing.ts
+++ b/app/server/lib/Sharing.ts
@@ -313,9 +313,8 @@ export class Sharing {
        internal,
      });
      actionGroup.actionSummary = actionSummary;
-      actionGroup.rowCount = sandboxActionBundle.rowCount;
      await accessControl.appliedBundle();
-      await accessControl.sendDocUpdateForBundle(actionGroup);
+      await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage);
      if (docSession) {
        docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0;
      }