diff --git a/app/client/components/DocComm.ts b/app/client/components/DocComm.ts index 9fb5bbc1..f8430068 100644 --- a/app/client/components/DocComm.ts +++ b/app/client/components/DocComm.ts @@ -5,6 +5,7 @@ import {ActionGroup} from 'app/common/ActionGroup'; import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI'; import {DocAction, UserAction} from 'app/common/DocActions'; import {OpenLocalDocResult} from 'app/common/DocListAPI'; +import {DocUsage} from 'app/common/DocUsage'; import {docUrl} from 'app/common/urlUtils'; import {Events as BackboneEvents} from 'backbone'; import {Disposable, Emitter} from 'grainjs'; @@ -17,6 +18,7 @@ export interface DocUserAction extends CommMessage { data: { docActions: DocAction[]; actionGroup: ActionGroup; + docUsage: DocUsage; error?: string; }; } diff --git a/app/client/components/DocUsageBanner.ts b/app/client/components/DocUsageBanner.ts index ce225040..a0ee5e60 100644 --- a/app/client/components/DocUsageBanner.ts +++ b/app/client/components/DocUsageBanner.ts @@ -1,4 +1,4 @@ -import {buildUpgradeMessage, getLimitStatusMessage} from 'app/client/components/DocumentUsage'; +import {buildLimitStatusMessage, buildUpgradeMessage} from 'app/client/components/DocumentUsage'; import {sessionStorageBoolObs} from 'app/client/lib/localStorageObs'; import {DocPageModel} from 'app/client/models/DocPageModel'; import {colors, isNarrowScreenObs} from 'app/client/ui2018/cssVars'; @@ -66,7 +66,7 @@ export class DocUsageBanner extends Disposable { cssBannerMessage( cssWhiteIcon('Idea'), cssLightlyBoldedText( - getLimitStatusMessage('approachingLimit', features), + buildLimitStatusMessage('approachingLimit', features), ' ', buildUpgradeMessage(org.access === 'owners'), testId('text'), @@ -99,7 +99,7 @@ export class DocUsageBanner extends Disposable { } return [ - getLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features), + buildLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features), ' ', buildUpgradeMessage(isOwner), ]; diff --git a/app/client/components/DocumentUsage.ts b/app/client/components/DocumentUsage.ts index 72d42398..80534807 100644 --- a/app/client/components/DocumentUsage.ts +++ b/app/client/components/DocumentUsage.ts @@ -1,13 +1,14 @@ import {DocPageModel} from 'app/client/models/DocPageModel'; +import {urlState} from 'app/client/models/gristUrlState'; import {docListHeader} from 'app/client/ui/DocMenuCss'; import {colors, mediaXSmall} from 'app/client/ui2018/cssVars'; import {icon} from 'app/client/ui2018/icons'; import {cssLink} from 'app/client/ui2018/links'; import {loadingSpinner} from 'app/client/ui2018/loaders'; +import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/DocUsage'; import {Features} from 'app/common/Features'; import {commonUrls} from 'app/common/gristUrls'; import {capitalizeFirstWord} from 'app/common/gutil'; -import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/Usage'; import {Computed, Disposable, dom, DomContents, DomElementArg, makeTestId, styled} from 'grainjs'; const testId = makeTestId('test-doc-usage-'); @@ -15,6 +16,12 @@ const testId = makeTestId('test-doc-usage-'); // Default used by the progress bar to visually indicate row usage. const DEFAULT_MAX_ROWS = 20000; +// Default used by the progress bar to visually indicate data size usage. +const DEFAULT_MAX_DATA_SIZE = DEFAULT_MAX_ROWS * 2 * 1024; // 40MB (2KiB per row) + +// Default used by the progress bar to visually indicate attachments size usage. +const DEFAULT_MAX_ATTACHMENTS_SIZE = 1 * 1024 * 1024 * 1024; // 1GiB + const ACCESS_DENIED_MESSAGE = 'Usage statistics are only available to users with ' + 'full access to the document data.'; @@ -25,6 +32,8 @@ export class DocumentUsage extends Disposable { private readonly _currentDoc = this._docPageModel.currentDoc; private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus; private readonly _rowCount = this._docPageModel.rowCount; + private readonly _dataSizeBytes = this._docPageModel.dataSizeBytes; + private readonly _attachmentsSizeBytes = this._docPageModel.attachmentsSizeBytes; private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => { return doc?.workspace.org ?? null; @@ -47,20 +56,65 @@ export class DocumentUsage extends Disposable { }; }); - private readonly _isLoading: Computed = - Computed.create(this, this._currentDoc, this._rowCount, (_use, doc, rowCount) => { - return doc === null || rowCount === 'pending'; + private readonly _dataSizeMetrics: Computed = + Computed.create(this, this._currentOrg, this._dataSizeBytes, (_use, org, dataSize) => { + const features = org?.billingAccount?.product.features; + if (!features || typeof dataSize !== 'number') { return null; } + + const {baseMaxDataSizePerDocument: maxSize} = features; + // Invalid data size limits are currently treated as if they are undefined. + const maxValue = maxSize && maxSize > 0 ? maxSize : undefined; + return { + name: 'Data Size', + currentValue: dataSize, + maximumValue: maxValue ?? DEFAULT_MAX_DATA_SIZE, + unit: 'MB', + shouldHideLimits: maxValue === undefined, + formatValue: (val) => { + // To display a nice, round number for `maximumValue`, we first convert + // to KiBs (base-2), and then convert to MBs (base-10). Normally, we wouldn't + // mix conversions like this, but to display something that matches our + // marketing limits (e.g. 40MB for Pro plan), we need to bend conversions a bit. + return ((val / 1024) / 1000).toFixed(2); + }, + }; + }); + + private readonly _attachmentsSizeMetrics: Computed = + Computed.create(this, this._currentOrg, this._attachmentsSizeBytes, (_use, org, attachmentsSize) => { + const features = org?.billingAccount?.product.features; + if (!features || typeof attachmentsSize !== 'number') { return null; } + + const {baseMaxAttachmentsBytesPerDocument: maxSize} = features; + // Invalid attachments size limits are currently treated as if they are undefined. + const maxValue = maxSize && maxSize > 0 ? maxSize : undefined; + return { + name: 'Attachments Size', + currentValue: attachmentsSize, + maximumValue: maxValue ?? DEFAULT_MAX_ATTACHMENTS_SIZE, + unit: 'GB', + shouldHideLimits: maxValue === undefined, + formatValue: (val) => (val / (1024 * 1024 * 1024)).toFixed(2), + }; }); + private readonly _isLoading: Computed = + Computed.create( + this, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes, + (_use, doc, rowCount, dataSize, attachmentsSize) => { + return !doc || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'pending'); + } + ); + private readonly _isAccessDenied: Computed = Computed.create( - this, this._isLoading, this._currentDoc, this._rowCount, - (_use, isLoading, doc, rowCount) => { + this, this._isLoading, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes, + (_use, isLoading, doc, rowCount, dataSize, attachmentsSize) => { if (isLoading) { return null; } const {access} = doc!.workspace.org; const isPublicUser = access === 'guests' || access === null; - return isPublicUser || rowCount === 'hidden'; + return isPublicUser || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'hidden'); } ); @@ -91,7 +145,9 @@ export class DocumentUsage extends Disposable { if (!org || !status) { return null; } return buildMessage([ - getLimitStatusMessage(status, org.billingAccount?.product.features), + buildLimitStatusMessage(status, org.billingAccount?.product.features, { + disableRawDataLink: true + }), ' ', buildUpgradeMessage(org.access === 'owners') ]); @@ -104,18 +160,79 @@ export class DocumentUsage extends Disposable { dom.maybe(this._rowMetrics, (metrics) => buildUsageMetric(metrics, testId('rows')), ), + dom.maybe(this._dataSizeMetrics, (metrics) => + buildUsageMetric(metrics, testId('data-size')), + ), + dom.maybe(this._attachmentsSizeMetrics, (metrics) => + buildUsageMetric(metrics, testId('attachments-size')), + ), testId('metrics'), ), ); } } -function buildMessage(message: DomContents) { - return cssWarningMessage( - cssIcon('Idea'), - cssLightlyBoldedText(message, testId('message-text')), - testId('message'), - ); +export function buildLimitStatusMessage( + status: NonNullable, + features?: Features, + options: { + disableRawDataLink?: boolean; + } = {} +) { + const {disableRawDataLink = false} = options; + switch (status) { + case 'approachingLimit': { + return [ + 'This document is ', + disableRawDataLink ? 'approaching' : buildRawDataPageLink('approaching'), + ' free plan limits.' + ]; + } + case 'gracePeriod': { + const gracePeriodDays = features?.gracePeriodDays; + if (!gracePeriodDays) { + return [ + 'Document limits ', + disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'), + '.' + ]; + } + + return [ + 'Document limits ', + disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'), + `. In ${gracePeriodDays} days, this document will be read-only.` + ]; + } + case 'deleteOnly': { + return [ + 'This document ', + disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'), + ' free plan limits and is now read-only, but you can delete rows.' + ]; + } + } +} + +export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') { + if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; } + + const upgradeLinkText = 'start your 30-day free trial of the Pro plan.'; + return [ + variant === 'short' ? null : 'For higher limits, ', + buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText), + ]; +} + +function buildUpgradeLink(linkText: string) { + return cssUnderlinedLink(linkText, { + href: commonUrls.plans, + target: '_blank', + }); +} + +function buildRawDataPageLink(linkText: string) { + return cssUnderlinedLink(linkText, urlState().setLinkUrl({docPage: 'data'})); } interface MetricOptions { @@ -126,6 +243,7 @@ interface MetricOptions { unit?: string; // If true, limits will always be hidden, even if `maximumValue` is a positive number. shouldHideLimits?: boolean; + formatValue?(value: number): string; } /** @@ -134,7 +252,14 @@ interface MetricOptions { * close `currentValue` is to hitting `maximumValue`. */ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) { - const {name, currentValue, maximumValue, unit, shouldHideLimits} = options; + const { + name, + currentValue, + maximumValue, + unit, + shouldHideLimits, + formatValue = (val) => val.toString(), + } = options; const ratioUsed = currentValue / (maximumValue || Infinity); const percentUsed = Math.min(100, Math.floor(ratioUsed * 100)); return cssUsageMetric( @@ -150,8 +275,8 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) { ), ), dom('div', - currentValue - + (shouldHideLimits || !maximumValue ? '' : ' of ' + maximumValue) + formatValue(currentValue) + + (shouldHideLimits || !maximumValue ? '' : ' of ' + formatValue(maximumValue)) + (unit ? ` ${unit}` : ''), testId('value'), ), @@ -159,38 +284,12 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) { ); } -export function getLimitStatusMessage(status: NonNullable, features?: Features): string { - switch (status) { - case 'approachingLimit': { - return 'This document is approaching free plan limits.'; - } - case 'gracePeriod': { - const gracePeriodDays = features?.gracePeriodDays; - if (!gracePeriodDays) { return 'Document limits exceeded.'; } - - return `Document limits exceeded. In ${gracePeriodDays} days, this document will be read-only.`; - } - case 'deleteOnly': { - return 'This document exceeded free plan limits and is now read-only, but you can delete rows.'; - } - } -} - -export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') { - if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; } - - const upgradeLinkText = 'start your 30-day free trial of the Pro plan.'; - return [ - variant === 'short' ? null : 'For higher limits, ', - buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText), - ]; -} - -export function buildUpgradeLink(linkText: string) { - return cssUnderlinedLink(linkText, { - href: commonUrls.plans, - target: '_blank', - }); +function buildMessage(message: DomContents) { + return cssWarningMessage( + cssIcon('Idea'), + cssLightlyBoldedText(message, testId('message-text')), + testId('message'), + ); } const cssLightlyBoldedText = styled('div', ` @@ -233,13 +332,8 @@ const cssUsageMetrics = styled('div', ` display: flex; flex-wrap: wrap; margin-top: 24px; - gap: 56px; - - @media ${mediaXSmall} { - & { - gap: 24px; - } - } + row-gap: 24px; + column-gap: 54px; `); const cssUsageMetric = styled('div', ` diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 17cabe63..62460f41 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -478,9 +478,7 @@ export class GristDoc extends DisposableWithEvents { if (schemaUpdated) { this.trigger('schemaUpdateAction', docActions); } - if (typeof actionGroup.rowCount === "number") { - this.docPageModel.rowCount.set(actionGroup.rowCount); - } + this.docPageModel.updateDocUsage(message.data.docUsage); } } diff --git a/app/client/models/DocPageModel.ts b/app/client/models/DocPageModel.ts index 7f8bd61b..1fa70795 100644 --- a/app/client/models/DocPageModel.ts +++ b/app/client/models/DocPageModel.ts @@ -17,10 +17,10 @@ import {confirmModal} from 'app/client/ui2018/modals'; import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow'; import {delay} from 'app/common/delay'; import {OpenDocMode, UserOverride} from 'app/common/DocListAPI'; +import {AttachmentsSize, DataLimitStatus, DataSize, DocUsage, RowCount} from 'app/common/DocUsage'; import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls'; import {getReconnectTimeout} from 'app/common/gutil'; import {canEdit} from 'app/common/roles'; -import {DataLimitStatus, RowCount} from 'app/common/Usage'; import {Document, NEW_DOCUMENT_CODE, Organization, UserAPI, Workspace} from 'app/common/UserAPI'; import {Holder, Observable, subscribe} from 'grainjs'; import {Computed, Disposable, dom, DomArg, DomElementArg} from 'grainjs'; @@ -66,13 +66,16 @@ export interface DocPageModel { gristDoc: Observable; // Instance of GristDoc once it exists. + dataLimitStatus: Observable; rowCount: Observable; - dataLimitStatus: Observable; + dataSizeBytes: Observable; + attachmentsSizeBytes: Observable; createLeftPane(leftPanelOpen: Observable): DomArg; renameDoc(value: string): Promise; updateCurrentDoc(urlId: string, openMode: OpenDocMode): Promise; refreshCurrentDoc(doc: DocInfo): Promise; + updateDocUsage(docUsage: DocUsage): void; } export interface ImportSource { @@ -109,8 +112,10 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { // Observable set to the instance of GristDoc once it's created. public readonly gristDoc = Observable.create(this, null); + public readonly dataLimitStatus = Observable.create(this, null); public readonly rowCount = Observable.create(this, 'pending'); - public readonly dataLimitStatus = Observable.create(this, null); + public readonly dataSizeBytes = Observable.create(this, 'pending'); + public readonly attachmentsSizeBytes = Observable.create(this, 'pending'); // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the // URL, and when it changes, we need to re-open. @@ -203,6 +208,13 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { return urlState().pushUrl(nextState, {avoidReload: true, ...options}); } + public updateDocUsage(docUsage: DocUsage) { + this.rowCount.set(docUsage.rowCount); + this.dataLimitStatus.set(docUsage.dataLimitStatus); + this.dataSizeBytes.set(docUsage.dataSizeBytes); + this.attachmentsSizeBytes.set(docUsage.attachmentsSizeBytes); + } + private _onOpenError(err: Error) { if (err instanceof CancelledError) { // This means that we started loading a new doc before the previous one finished loading. @@ -260,8 +272,9 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { doc.userOverride = openDocResponse.userOverride || null; this.currentDoc.set({...doc}); } - this.rowCount.set(openDocResponse.rowCount); - this.dataLimitStatus.set(openDocResponse.dataLimitStatus); + if (openDocResponse.docUsage) { + this.updateDocUsage(openDocResponse.docUsage); + } const gdModule = await gristDocModulePromise; const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier); flow.checkIfCancelled(); diff --git a/app/client/ui2018/menus.ts b/app/client/ui2018/menus.ts index 5e262e29..091be7a0 100644 --- a/app/client/ui2018/menus.ts +++ b/app/client/ui2018/menus.ts @@ -537,6 +537,10 @@ const cssCmdKey = styled('span', ` margin-left: 16px; color: ${colors.slate}; margin-right: -12px; + + .${weasel.cssMenuItem.className}-sel > & { + color: ${colors.lightGrey}; + } `); const cssAnnotateMenuItem = styled('span', ` diff --git a/app/common/ActionGroup.ts b/app/common/ActionGroup.ts index 05c42c2c..4b9d6f15 100644 --- a/app/common/ActionGroup.ts +++ b/app/common/ActionGroup.ts @@ -25,5 +25,4 @@ export interface ActionGroup extends MinimalActionGroup { user: string; primaryAction: string; // The name of the first user action in the ActionGroup. internal: boolean; // True if it is inappropriate to log/undo the action. - rowCount?: number; } diff --git a/app/common/DocListAPI.ts b/app/common/DocListAPI.ts index 3c09de45..9645ed97 100644 --- a/app/common/DocListAPI.ts +++ b/app/common/DocListAPI.ts @@ -1,8 +1,8 @@ import {MinimalActionGroup} from 'app/common/ActionGroup'; import {TableDataAction} from 'app/common/DocActions'; +import {DocUsage} from 'app/common/DocUsage'; import {Role} from 'app/common/roles'; import {StringUnion} from 'app/common/StringUnion'; -import {DataLimitStatus, RowCount} from 'app/common/Usage'; import {FullUser} from 'app/common/UserAPI'; // Possible flavors of items in a list of documents. @@ -43,10 +43,9 @@ export interface OpenLocalDocResult { clientId: string; // the docFD is meaningful only in the context of this session doc: {[tableId: string]: TableDataAction}; log: MinimalActionGroup[]; - rowCount: RowCount; recoveryMode?: boolean; userOverride?: UserOverride; - dataLimitStatus?: DataLimitStatus; + docUsage?: DocUsage; } export interface UserOverride { diff --git a/app/common/DocUsage.ts b/app/common/DocUsage.ts new file mode 100644 index 00000000..02099d22 --- /dev/null +++ b/app/common/DocUsage.ts @@ -0,0 +1,29 @@ +import {ApiError} from 'app/common/ApiError'; + +export interface DocUsage { + dataLimitStatus: DataLimitStatus; + rowCount: RowCount; + dataSizeBytes: DataSize; + attachmentsSizeBytes: AttachmentsSize; +} + +type NumberOrStatus = number | 'hidden' | 'pending'; + +export type RowCount = NumberOrStatus; + +export type DataSize = NumberOrStatus; + +export type AttachmentsSize = NumberOrStatus; + +export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null; + +export type NonHidden = Exclude; + +// Ratio of usage at which we start telling users that they're approaching limits. +export const APPROACHING_LIMIT_RATIO = 0.9; + +export class LimitExceededError extends ApiError { + constructor(message: string) { + super(message, 413); + } +} diff --git a/app/common/Usage.ts b/app/common/Usage.ts deleted file mode 100644 index e9042dfe..00000000 --- a/app/common/Usage.ts +++ /dev/null @@ -1,6 +0,0 @@ -export type RowCount = number | 'hidden' | 'pending'; - -export type DataLimitStatus = null | 'approachingLimit' | 'gracePeriod' | 'deleteOnly'; - -// Ratio of the row/data size limit where we tell users that they're approaching the limit. -export const APPROACHING_LIMIT_RATIO = 0.9; diff --git a/app/common/gutil.ts b/app/common/gutil.ts index dc5796e6..1887eb50 100644 --- a/app/common/gutil.ts +++ b/app/common/gutil.ts @@ -935,3 +935,16 @@ export function assertIsDefined(name: string, value: T): asserts value is Non throw new Error(`Expected '${name}' to be defined, but received ${value}`); } } + +/** + * Calls function `fn`, passes any thrown errors to function `recover`, and finally calls `fn` + * once more if `recover` doesn't throw. + */ + export async function retryOnce(fn: () => Promise, recover: (e: unknown) => Promise): Promise { + try { + return await fn(); + } catch (e) { + await recover(e); + return await fn(); + } +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 84cff6a1..1d6315ef 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -35,16 +35,24 @@ import { import {DocData} from 'app/common/DocData'; import {DocSnapshots} from 'app/common/DocSnapshot'; import {DocumentSettings} from 'app/common/DocumentSettings'; +import { + APPROACHING_LIMIT_RATIO, + AttachmentsSize, + DataLimitStatus, + DataSize, + DocUsage, + LimitExceededError, + NonHidden, + RowCount +} from 'app/common/DocUsage'; import {normalizeEmail} from 'app/common/emails'; import {Features} from 'app/common/Features'; import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause'; -import {byteString, countIf, safeJsonParse} from 'app/common/gutil'; +import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; -import {canEdit} from 'app/common/roles'; import {schema, SCHEMA_VERSION} from 'app/common/schema'; import {MetaRowRecord} from 'app/common/TableData'; import {FetchUrlOptions, UploadResult} from 'app/common/uploads'; -import {APPROACHING_LIMIT_RATIO, DataLimitStatus, RowCount} from 'app/common/Usage'; import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI'; import {convertFromColumn} from 'app/common/ValueConverter'; import {guessColInfoWithDocData} from 'app/common/ValueGuesser'; @@ -173,8 +181,9 @@ export class ActiveDoc extends EventEmitter { private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured. private _lastDataSizeMeasurement: number = 0; // Timestamp when dbstat data size was last measured. private _fetchCache = new MapWithTTL>(DEFAULT_CACHE_TTL); - private _rowCount: RowCount = 'pending'; - private _dataSize?: number; + private _rowCount: NonHidden = 'pending'; + private _dataSize: NonHidden = 'pending'; + private _attachmentsSize: NonHidden = 'pending'; private _productFeatures?: Features; private _gracePeriodStart: Date|null = null; @@ -245,8 +254,8 @@ export class ActiveDoc extends EventEmitter { public get isShuttingDown(): boolean { return this._shuttingDown; } public get rowLimitRatio() { - if (!this._rowLimit || this._rowLimit <= 0 || typeof this._rowCount !== 'number') { - // Invalid row limits are currently treated as if they are undefined. + if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') { + // If limit can't be enforced (e.g. undefined, non-positive), assume no limit. return 0; } @@ -254,8 +263,8 @@ export class ActiveDoc extends EventEmitter { } public get dataSizeLimitRatio() { - if (!this._dataSizeLimit || this._dataSizeLimit <= 0 || !this._dataSize) { - // Invalid data size limits are currently treated as if they are undefined. + if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') { + // If limit can't be enforced (e.g. undefined, non-positive), assume no limit. return 0; } @@ -282,15 +291,17 @@ export class ActiveDoc extends EventEmitter { return null; } - public async getRowCount(docSession: OptDocSession): Promise { - const hasFullReadAccess = await this._granularAccess.canReadEverything(docSession); - const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession)); - return hasFullReadAccess && hasEditRole ? this._rowCount : 'hidden'; + public get docUsage(): DocUsage { + return { + dataLimitStatus: this.dataLimitStatus, + rowCount: this._rowCount, + dataSizeBytes: this._dataSize, + attachmentsSizeBytes: this._attachmentsSize, + }; } - public async getDataLimitStatus(docSession: OptDocSession): Promise { - const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession)); - return hasEditRole ? this.dataLimitStatus : null; + public getFilteredDocUsage(docSession: OptDocSession): Promise { + return this._granularAccess.filterDocUsage(docSession, this.docUsage); } public async getUserOverride(docSession: OptDocSession) { @@ -692,10 +703,31 @@ export class ActiveDoc extends EventEmitter { const userId = getDocSessionUserId(docSession); const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)); try { - await this._checkDocAttachmentsLimit(upload); + // We'll assert that the upload won't cause limits to be exceeded, retrying once after + // soft-deleting any unused attachments. + await retryOnce( + () => this._assertUploadSizeBelowLimit(upload), + async (e) => { + if (!(e instanceof LimitExceededError)) { throw e; } + + // Check if any attachments are unused and can be soft-deleted to reduce the existing + // total size. We could do this from the beginning, but updateUsedAttachmentsIfNeeded + // is potentially expensive, so this optimises for the common case of not exceeding the limit. + const hadChanges = await this.updateUsedAttachmentsIfNeeded(); + if (hadChanges) { + await this._updateAttachmentsSize(); + } else { + // No point in retrying if nothing changed. + throw new LimitExceededError("Exceeded attachments limit for document"); + } + } + ); const userActions: UserAction[] = await Promise.all( upload.files.map(file => this._prepAttachment(docSession, file))); const result = await this.applyUserActions(docSession, userActions); + this._updateAttachmentsSize().catch(e => { + this._log.warn(docSession, 'failed to update attachments size', e); + }); return result.retValues; } finally { await globalUploadSet.cleanup(uploadId); @@ -1352,7 +1384,7 @@ export class ActiveDoc extends EventEmitter { * so that undo can 'undelete' attachments. * Returns true if any changes were made, i.e. some row(s) of _grist_Attachments were updated. */ - public async updateUsedAttachments() { + public async updateUsedAttachmentsIfNeeded() { const changes = await this.docStorage.scanAttachmentsForUsageChanges(); if (!changes.length) { return false; @@ -1371,7 +1403,8 @@ export class ActiveDoc extends EventEmitter { * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago. */ public async removeUnusedAttachments(expiredOnly: boolean) { - await this.updateUsedAttachments(); + const hadChanges = await this.updateUsedAttachmentsIfNeeded(); + if (hadChanges) { await this._updateAttachmentsSize(); } const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly); if (rowIds.length) { const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds]; @@ -1807,6 +1840,10 @@ export class ActiveDoc extends EventEmitter { const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT; this._inactivityTimer.setDelay(closeTimeout); this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`); + // TODO: Initialize data and attachments size from Document.usage once it's available. + this._updateAttachmentsSize().catch(e => { + this._log.warn(docSession, 'failed to update attachments size', e); + }); } catch (err) { this._fullyLoaded = true; if (!this._shuttingDown) { @@ -1935,35 +1972,32 @@ export class ActiveDoc extends EventEmitter { /** * Throw an error if the provided upload would exceed the total attachment filesize limit for this document. */ - private async _checkDocAttachmentsLimit(upload: UploadInfo) { - const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument; - if (!maxSize) { - // This document has no limit, nothing to check. - return; - } - + private async _assertUploadSizeBelowLimit(upload: UploadInfo) { // Minor flaw: while we don't double-count existing duplicate files in the total size, // we don't check here if any of the uploaded files already exist and could be left out of the calculation. - const totalAddedSize = sum(upload.files.map(f => f.size)); + const uploadSizeBytes = sum(upload.files.map(f => f.size)); + if (await this._isUploadSizeBelowLimit(uploadSizeBytes)) { return; } - // Returns true if this upload won't bring the total over the limit. - const isOK = async () => (await this.docStorage.getTotalAttachmentFileSizes()) + totalAddedSize <= maxSize; + // TODO probably want a nicer error message here. + throw new LimitExceededError("Exceeded attachments limit for document"); + } - if (await isOK()) { - return; - } + /** + * Returns true if an upload with size `uploadSizeBytes` won't cause attachment size + * limits to be exceeded. + */ + private async _isUploadSizeBelowLimit(uploadSizeBytes: number): Promise { + const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument; + if (!maxSize) { return true; } - // Looks like the limit is being exceeded. - // Check if any attachments are unused and can be soft-deleted to reduce the existing total size. - // We could do this from the beginning, but updateUsedAttachments is potentially expensive, - // so this optimises the common case of not exceeding the limit. - // updateUsedAttachments returns true if there were any changes. Otherwise there's no point checking isOK again. - if (await this.updateUsedAttachments() && await isOK()) { - return; - } + const currentSize = this._attachmentsSize !== 'pending' + ? this._attachmentsSize + : await this.docStorage.getTotalAttachmentFileSizes(); + return currentSize + uploadSizeBytes <= maxSize; + } - // TODO probably want a nicer error message here. - throw new Error("Exceeded attachments limit for document"); + private async _updateAttachmentsSize() { + this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes(); } } @@ -1989,3 +2023,8 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table } return tableRefs[tableRowIndex]; } + +// Helper that returns true if `limit` is set to a valid, positive number. +function isEnforceableLimit(limit: number | undefined): limit is number { + return limit !== undefined && limit > 0; +} diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index aba4b693..c7d6b44f 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -248,7 +248,7 @@ export class DocWorkerApi { // Mostly for testing this._app.post('/api/docs/:docId/attachments/updateUsed', canEdit, withDoc(async (activeDoc, req, res) => { - await activeDoc.updateUsedAttachments(); + await activeDoc.updateUsedAttachmentsIfNeeded(); res.json(null); })); this._app.post('/api/docs/:docId/attachments/removeUnused', isOwner, withDoc(async (activeDoc, req, res) => { diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index a99750c0..3601c1d0 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -9,6 +9,7 @@ import {ApiError} from 'app/common/ApiError'; import {mapSetOrClear} from 'app/common/AsyncCreate'; import {BrowserSettings} from 'app/common/BrowserSettings'; import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI'; +import {DocUsage} from 'app/common/DocUsage'; import {Invite} from 'app/common/sharing'; import {tbind} from 'app/common/tbind'; import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; @@ -313,24 +314,28 @@ export class DocManager extends EventEmitter { } } - const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([ + const [metaTables, recentActions, userOverride] = await Promise.all([ activeDoc.fetchMetaTables(docSession), activeDoc.getRecentMinimalActions(docSession), activeDoc.getUserOverride(docSession), - activeDoc.getRowCount(docSession), - activeDoc.getDataLimitStatus(docSession), ]); - const result = { + let docUsage: DocUsage | undefined; + try { + docUsage = await activeDoc.getFilteredDocUsage(docSession); + } catch (e) { + log.warn("DocManager.openDoc failed to get doc usage", e); + } + + const result: OpenLocalDocResult = { docFD: docSession.fd, clientId: docSession.client.clientId, doc: metaTables, log: recentActions, recoveryMode: activeDoc.recoveryMode, userOverride, - rowCount, - dataLimitStatus, - } as OpenLocalDocResult; + docUsage, + }; if (!activeDoc.muted) { this.emit('open-doc', this.storageManager.getPath(activeDoc.docName)); diff --git a/app/server/lib/DocStorage.ts b/app/server/lib/DocStorage.ts index bfd03945..af500a76 100644 --- a/app/server/lib/DocStorage.ts +++ b/app/server/lib/DocStorage.ts @@ -1238,9 +1238,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { /** * Returns the total number of bytes used for storing attachments that haven't been soft-deleted. - * May be stale if ActiveDoc.updateUsedAttachments isn't called first. + * May be stale if ActiveDoc.updateUsedAttachmentsIfNeeded isn't called first. */ - public async getTotalAttachmentFileSizes() { + public async getTotalAttachmentFileSizes(): Promise { const result = await this.get(` SELECT SUM(len) AS total FROM ( @@ -1258,7 +1258,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { GROUP BY meta.fileIdent ) `); - return result!.total as number; + return result!.total ?? 0; } /** @@ -1307,7 +1307,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage { /** * Return row IDs of unused attachments in _grist_Attachments. - * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachments. + * Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachmentsIfNeeded. * @param expiredOnly: if true, only return attachments where timeDeleted is at least * ATTACHMENTS_EXPIRY_DAYS days ago. */ diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 58782972..c966815d 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -10,6 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app import { TableDataAction, UserAction } from 'app/common/DocActions'; import { DocData } from 'app/common/DocData'; import { UserOverride } from 'app/common/DocListAPI'; +import { DocUsage } from 'app/common/DocUsage'; import { normalizeEmail } from 'app/common/emails'; import { ErrorWithCode } from 'app/common/ErrorWithCode'; import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause'; @@ -101,6 +102,12 @@ const SURPRISING_ACTIONS = new Set([ // Actions we'll allow unconditionally for now. const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']); +interface DocUpdateMessage { + actionGroup: ActionGroup; + docActions: DocAction[]; + docUsage: DocUsage; +} + /** * Granular access for a single bundle, in different phases. */ @@ -108,7 +115,7 @@ export interface GranularAccessForBundle { canApplyBundle(): Promise; appliedBundle(): Promise; finishedBundle(): Promise; - sendDocUpdateForBundle(actionGroup: ActionGroup): Promise; + sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise; } /** @@ -362,13 +369,16 @@ export class GranularAccess implements GranularAccessForBundle { /** * Filter an ActionGroup to be sent to a client. */ - public async filterActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise { - if (await this.allowActionGroup(docSession, actionGroup)) { return actionGroup; } + public async filterActionGroup( + docSession: OptDocSession, + actionGroup: ActionGroup, + options: {role?: Role | null} = {} + ): Promise { + if (await this.allowActionGroup(docSession, actionGroup, options)) { return actionGroup; } // For now, if there's any nuance at all, suppress the summary and description. const result: ActionGroup = { ...actionGroup }; result.actionSummary = createEmptyActionSummary(); result.desc = ''; - result.rowCount = undefined; return result; } @@ -376,8 +386,33 @@ export class GranularAccess implements GranularAccessForBundle { * Check whether an ActionGroup can be sent to the client. TODO: in future, we'll want * to filter acceptable parts of ActionGroup, rather than denying entirely. */ - public async allowActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise { - return this.canReadEverything(docSession); + public async allowActionGroup( + docSession: OptDocSession, + _actionGroup: ActionGroup, + options: {role?: Role | null} = {} + ): Promise { + return this.canReadEverything(docSession, options); + } + + /** + * Filter DocUsage to be sent to a client. + */ + public async filterDocUsage( + docSession: OptDocSession, + docUsage: DocUsage, + options: {role?: Role | null} = {} + ): Promise { + const result: DocUsage = { ...docUsage }; + const role = options.role ?? await this.getNominalAccess(docSession); + const hasEditRole = canEdit(role); + if (!hasEditRole) { result.dataLimitStatus = null; } + const hasFullReadAccess = await this.canReadEverything(docSession); + if (!hasEditRole || !hasFullReadAccess) { + result.rowCount = 'hidden'; + result.dataSizeBytes = 'hidden'; + result.attachmentsSizeBytes = 'hidden'; + } + return result; } /** @@ -577,8 +612,11 @@ export class GranularAccess implements GranularAccessForBundle { * Check whether user can read everything in document. Checks both home-level and doc-level * permissions. */ - public async canReadEverything(docSession: OptDocSession): Promise { - const access = await this.getNominalAccess(docSession); + public async canReadEverything( + docSession: OptDocSession, + options: {role?: Role | null} = {} + ): Promise { + const access = options.role ?? await this.getNominalAccess(docSession); if (!canView(access)) { return false; } const permInfo = await this._getAccess(docSession); return this.getReadPermission(permInfo.getFullAccess()) === 'allow'; @@ -709,11 +747,11 @@ export class GranularAccess implements GranularAccessForBundle { /** * Broadcast document changes to all clients, with appropriate filtering. */ - public async sendDocUpdateForBundle(actionGroup: ActionGroup) { + public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) { if (!this._activeBundle) { throw new Error('no active bundle'); } const { docActions, docSession } = this._activeBundle; const client = docSession && docSession.client || null; - const message = { actionGroup, docActions }; + const message: DocUpdateMessage = { actionGroup, docActions, docUsage }; await this._docClients.broadcastDocMessage(client, 'docUserAction', message, (_docSession) => this._filterDocUpdate(_docSession, message)); @@ -866,18 +904,18 @@ export class GranularAccess implements GranularAccessForBundle { * This filters a message being broadcast to all clients to be appropriate for one * particular client, if that client may need some material filtered out. */ - private async _filterDocUpdate(docSession: OptDocSession, message: { - actionGroup: ActionGroup, - docActions: DocAction[] - }) { + private async _filterDocUpdate(docSession: OptDocSession, message: DocUpdateMessage) { if (!this._activeBundle) { throw new Error('no active bundle'); } - if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) { - return message; - } + const role = await this.getNominalAccess(docSession); const result = { - actionGroup: await this.filterActionGroup(docSession, message.actionGroup), - docActions: await this.filterOutgoingDocActions(docSession, message.docActions), + ...message, + docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}), }; + if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) { + return result; + } + result.actionGroup = await this.filterActionGroup(docSession, message.actionGroup, {role}); + result.docActions = await this.filterOutgoingDocActions(docSession, message.docActions); if (result.docActions.length === 0) { return null; } return result; } diff --git a/app/server/lib/Sharing.ts b/app/server/lib/Sharing.ts index 17626cd6..a9cfa47c 100644 --- a/app/server/lib/Sharing.ts +++ b/app/server/lib/Sharing.ts @@ -313,9 +313,8 @@ export class Sharing { internal, }); actionGroup.actionSummary = actionSummary; - actionGroup.rowCount = sandboxActionBundle.rowCount; await accessControl.appliedBundle(); - await accessControl.sendDocUpdateForBundle(actionGroup); + await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage); if (docSession) { docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0; } diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts index d817b2c1..a341a791 100644 --- a/test/server/lib/DocApi.ts +++ b/test/server/lib/DocApi.ts @@ -1725,7 +1725,7 @@ function testDocApi() { assert.equal(resp.status, 200); // Remove the not expired attachments (2 and 3). - // We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachments. + // We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded. resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy); assert.equal(resp.status, 200); await checkAttachmentIds([]);