(core) Add attachment and data size usage

Summary:
Adds attachment and data size to the usage section of
the raw data page. Also makes in-document usage banners
update as user actions are applied, causing them to be
hidden/shown or updated based on the current state of
the document.

Test Plan: Browser tests.

Reviewers: jarek

Reviewed By: jarek

Subscribers: alexmojaki

Differential Revision: https://phab.getgrist.com/D3395
pull/203/head
George Gevoian 2 years ago
parent f194d6861b
commit 1e42871cc9

@ -5,6 +5,7 @@ import {ActionGroup} from 'app/common/ActionGroup';
import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI';
import {DocAction, UserAction} from 'app/common/DocActions';
import {OpenLocalDocResult} from 'app/common/DocListAPI';
import {DocUsage} from 'app/common/DocUsage';
import {docUrl} from 'app/common/urlUtils';
import {Events as BackboneEvents} from 'backbone';
import {Disposable, Emitter} from 'grainjs';
@ -17,6 +18,7 @@ export interface DocUserAction extends CommMessage {
data: {
docActions: DocAction[];
actionGroup: ActionGroup;
docUsage: DocUsage;
error?: string;
};
}

@ -1,4 +1,4 @@
import {buildUpgradeMessage, getLimitStatusMessage} from 'app/client/components/DocumentUsage';
import {buildLimitStatusMessage, buildUpgradeMessage} from 'app/client/components/DocumentUsage';
import {sessionStorageBoolObs} from 'app/client/lib/localStorageObs';
import {DocPageModel} from 'app/client/models/DocPageModel';
import {colors, isNarrowScreenObs} from 'app/client/ui2018/cssVars';
@ -66,7 +66,7 @@ export class DocUsageBanner extends Disposable {
cssBannerMessage(
cssWhiteIcon('Idea'),
cssLightlyBoldedText(
getLimitStatusMessage('approachingLimit', features),
buildLimitStatusMessage('approachingLimit', features),
' ',
buildUpgradeMessage(org.access === 'owners'),
testId('text'),
@ -99,7 +99,7 @@ export class DocUsageBanner extends Disposable {
}
return [
getLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features),
buildLimitStatusMessage(isDeleteOnly ? 'deleteOnly' : 'gracePeriod', features),
' ',
buildUpgradeMessage(isOwner),
];

@ -1,13 +1,14 @@
import {DocPageModel} from 'app/client/models/DocPageModel';
import {urlState} from 'app/client/models/gristUrlState';
import {docListHeader} from 'app/client/ui/DocMenuCss';
import {colors, mediaXSmall} from 'app/client/ui2018/cssVars';
import {icon} from 'app/client/ui2018/icons';
import {cssLink} from 'app/client/ui2018/links';
import {loadingSpinner} from 'app/client/ui2018/loaders';
import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/DocUsage';
import {Features} from 'app/common/Features';
import {commonUrls} from 'app/common/gristUrls';
import {capitalizeFirstWord} from 'app/common/gutil';
import {APPROACHING_LIMIT_RATIO, DataLimitStatus} from 'app/common/Usage';
import {Computed, Disposable, dom, DomContents, DomElementArg, makeTestId, styled} from 'grainjs';
const testId = makeTestId('test-doc-usage-');
@ -15,6 +16,12 @@ const testId = makeTestId('test-doc-usage-');
// Default used by the progress bar to visually indicate row usage.
const DEFAULT_MAX_ROWS = 20000;
// Default used by the progress bar to visually indicate data size usage.
const DEFAULT_MAX_DATA_SIZE = DEFAULT_MAX_ROWS * 2 * 1024; // 40MB (2KiB per row)
// Default used by the progress bar to visually indicate attachments size usage.
const DEFAULT_MAX_ATTACHMENTS_SIZE = 1 * 1024 * 1024 * 1024; // 1GiB
const ACCESS_DENIED_MESSAGE = 'Usage statistics are only available to users with '
+ 'full access to the document data.';
@ -25,6 +32,8 @@ export class DocumentUsage extends Disposable {
private readonly _currentDoc = this._docPageModel.currentDoc;
private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus;
private readonly _rowCount = this._docPageModel.rowCount;
private readonly _dataSizeBytes = this._docPageModel.dataSizeBytes;
private readonly _attachmentsSizeBytes = this._docPageModel.attachmentsSizeBytes;
private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => {
return doc?.workspace.org ?? null;
@ -47,20 +56,65 @@ export class DocumentUsage extends Disposable {
};
});
private readonly _isLoading: Computed<boolean> =
Computed.create(this, this._currentDoc, this._rowCount, (_use, doc, rowCount) => {
return doc === null || rowCount === 'pending';
private readonly _dataSizeMetrics: Computed<MetricOptions | null> =
Computed.create(this, this._currentOrg, this._dataSizeBytes, (_use, org, dataSize) => {
const features = org?.billingAccount?.product.features;
if (!features || typeof dataSize !== 'number') { return null; }
const {baseMaxDataSizePerDocument: maxSize} = features;
// Invalid data size limits are currently treated as if they are undefined.
const maxValue = maxSize && maxSize > 0 ? maxSize : undefined;
return {
name: 'Data Size',
currentValue: dataSize,
maximumValue: maxValue ?? DEFAULT_MAX_DATA_SIZE,
unit: 'MB',
shouldHideLimits: maxValue === undefined,
formatValue: (val) => {
// To display a nice, round number for `maximumValue`, we first convert
// to KiBs (base-2), and then convert to MBs (base-10). Normally, we wouldn't
// mix conversions like this, but to display something that matches our
// marketing limits (e.g. 40MB for Pro plan), we need to bend conversions a bit.
return ((val / 1024) / 1000).toFixed(2);
},
};
});
private readonly _attachmentsSizeMetrics: Computed<MetricOptions | null> =
Computed.create(this, this._currentOrg, this._attachmentsSizeBytes, (_use, org, attachmentsSize) => {
const features = org?.billingAccount?.product.features;
if (!features || typeof attachmentsSize !== 'number') { return null; }
const {baseMaxAttachmentsBytesPerDocument: maxSize} = features;
// Invalid attachments size limits are currently treated as if they are undefined.
const maxValue = maxSize && maxSize > 0 ? maxSize : undefined;
return {
name: 'Attachments Size',
currentValue: attachmentsSize,
maximumValue: maxValue ?? DEFAULT_MAX_ATTACHMENTS_SIZE,
unit: 'GB',
shouldHideLimits: maxValue === undefined,
formatValue: (val) => (val / (1024 * 1024 * 1024)).toFixed(2),
};
});
private readonly _isLoading: Computed<boolean> =
Computed.create(
this, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes,
(_use, doc, rowCount, dataSize, attachmentsSize) => {
return !doc || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'pending');
}
);
private readonly _isAccessDenied: Computed<boolean | null> =
Computed.create(
this, this._isLoading, this._currentDoc, this._rowCount,
(_use, isLoading, doc, rowCount) => {
this, this._isLoading, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes,
(_use, isLoading, doc, rowCount, dataSize, attachmentsSize) => {
if (isLoading) { return null; }
const {access} = doc!.workspace.org;
const isPublicUser = access === 'guests' || access === null;
return isPublicUser || rowCount === 'hidden';
return isPublicUser || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'hidden');
}
);
@ -91,7 +145,9 @@ export class DocumentUsage extends Disposable {
if (!org || !status) { return null; }
return buildMessage([
getLimitStatusMessage(status, org.billingAccount?.product.features),
buildLimitStatusMessage(status, org.billingAccount?.product.features, {
disableRawDataLink: true
}),
' ',
buildUpgradeMessage(org.access === 'owners')
]);
@ -104,18 +160,79 @@ export class DocumentUsage extends Disposable {
dom.maybe(this._rowMetrics, (metrics) =>
buildUsageMetric(metrics, testId('rows')),
),
dom.maybe(this._dataSizeMetrics, (metrics) =>
buildUsageMetric(metrics, testId('data-size')),
),
dom.maybe(this._attachmentsSizeMetrics, (metrics) =>
buildUsageMetric(metrics, testId('attachments-size')),
),
testId('metrics'),
),
);
}
}
function buildMessage(message: DomContents) {
return cssWarningMessage(
cssIcon('Idea'),
cssLightlyBoldedText(message, testId('message-text')),
testId('message'),
);
export function buildLimitStatusMessage(
status: NonNullable<DataLimitStatus>,
features?: Features,
options: {
disableRawDataLink?: boolean;
} = {}
) {
const {disableRawDataLink = false} = options;
switch (status) {
case 'approachingLimit': {
return [
'This document is ',
disableRawDataLink ? 'approaching' : buildRawDataPageLink('approaching'),
' free plan limits.'
];
}
case 'gracePeriod': {
const gracePeriodDays = features?.gracePeriodDays;
if (!gracePeriodDays) {
return [
'Document limits ',
disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
'.'
];
}
return [
'Document limits ',
disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
`. In ${gracePeriodDays} days, this document will be read-only.`
];
}
case 'deleteOnly': {
return [
'This document ',
disableRawDataLink ? 'exceeded' : buildRawDataPageLink('exceeded'),
' free plan limits and is now read-only, but you can delete rows.'
];
}
}
}
export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') {
if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; }
const upgradeLinkText = 'start your 30-day free trial of the Pro plan.';
return [
variant === 'short' ? null : 'For higher limits, ',
buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText),
];
}
function buildUpgradeLink(linkText: string) {
return cssUnderlinedLink(linkText, {
href: commonUrls.plans,
target: '_blank',
});
}
function buildRawDataPageLink(linkText: string) {
return cssUnderlinedLink(linkText, urlState().setLinkUrl({docPage: 'data'}));
}
interface MetricOptions {
@ -126,6 +243,7 @@ interface MetricOptions {
unit?: string;
// If true, limits will always be hidden, even if `maximumValue` is a positive number.
shouldHideLimits?: boolean;
formatValue?(value: number): string;
}
/**
@ -134,7 +252,14 @@ interface MetricOptions {
* close `currentValue` is to hitting `maximumValue`.
*/
function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
const {name, currentValue, maximumValue, unit, shouldHideLimits} = options;
const {
name,
currentValue,
maximumValue,
unit,
shouldHideLimits,
formatValue = (val) => val.toString(),
} = options;
const ratioUsed = currentValue / (maximumValue || Infinity);
const percentUsed = Math.min(100, Math.floor(ratioUsed * 100));
return cssUsageMetric(
@ -150,8 +275,8 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
),
),
dom('div',
currentValue
+ (shouldHideLimits || !maximumValue ? '' : ' of ' + maximumValue)
formatValue(currentValue)
+ (shouldHideLimits || !maximumValue ? '' : ' of ' + formatValue(maximumValue))
+ (unit ? ` ${unit}` : ''),
testId('value'),
),
@ -159,38 +284,12 @@ function buildUsageMetric(options: MetricOptions, ...domArgs: DomElementArg[]) {
);
}
export function getLimitStatusMessage(status: NonNullable<DataLimitStatus>, features?: Features): string {
switch (status) {
case 'approachingLimit': {
return 'This document is approaching free plan limits.';
}
case 'gracePeriod': {
const gracePeriodDays = features?.gracePeriodDays;
if (!gracePeriodDays) { return 'Document limits exceeded.'; }
return `Document limits exceeded. In ${gracePeriodDays} days, this document will be read-only.`;
}
case 'deleteOnly': {
return 'This document exceeded free plan limits and is now read-only, but you can delete rows.';
}
}
}
export function buildUpgradeMessage(isOwner: boolean, variant: 'short' | 'long' = 'long') {
if (!isOwner) { return 'Contact the site owner to upgrade the plan to raise limits.'; }
const upgradeLinkText = 'start your 30-day free trial of the Pro plan.';
return [
variant === 'short' ? null : 'For higher limits, ',
buildUpgradeLink(variant === 'short' ? capitalizeFirstWord(upgradeLinkText) : upgradeLinkText),
];
}
export function buildUpgradeLink(linkText: string) {
return cssUnderlinedLink(linkText, {
href: commonUrls.plans,
target: '_blank',
});
function buildMessage(message: DomContents) {
return cssWarningMessage(
cssIcon('Idea'),
cssLightlyBoldedText(message, testId('message-text')),
testId('message'),
);
}
const cssLightlyBoldedText = styled('div', `
@ -233,13 +332,8 @@ const cssUsageMetrics = styled('div', `
display: flex;
flex-wrap: wrap;
margin-top: 24px;
gap: 56px;
@media ${mediaXSmall} {
& {
gap: 24px;
}
}
row-gap: 24px;
column-gap: 54px;
`);
const cssUsageMetric = styled('div', `

@ -478,9 +478,7 @@ export class GristDoc extends DisposableWithEvents {
if (schemaUpdated) {
this.trigger('schemaUpdateAction', docActions);
}
if (typeof actionGroup.rowCount === "number") {
this.docPageModel.rowCount.set(actionGroup.rowCount);
}
this.docPageModel.updateDocUsage(message.data.docUsage);
}
}

@ -17,10 +17,10 @@ import {confirmModal} from 'app/client/ui2018/modals';
import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow';
import {delay} from 'app/common/delay';
import {OpenDocMode, UserOverride} from 'app/common/DocListAPI';
import {AttachmentsSize, DataLimitStatus, DataSize, DocUsage, RowCount} from 'app/common/DocUsage';
import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls';
import {getReconnectTimeout} from 'app/common/gutil';
import {canEdit} from 'app/common/roles';
import {DataLimitStatus, RowCount} from 'app/common/Usage';
import {Document, NEW_DOCUMENT_CODE, Organization, UserAPI, Workspace} from 'app/common/UserAPI';
import {Holder, Observable, subscribe} from 'grainjs';
import {Computed, Disposable, dom, DomArg, DomElementArg} from 'grainjs';
@ -66,13 +66,16 @@ export interface DocPageModel {
gristDoc: Observable<GristDoc|null>; // Instance of GristDoc once it exists.
dataLimitStatus: Observable<DataLimitStatus>;
rowCount: Observable<RowCount>;
dataLimitStatus: Observable<DataLimitStatus|undefined>;
dataSizeBytes: Observable<DataSize>;
attachmentsSizeBytes: Observable<AttachmentsSize>;
createLeftPane(leftPanelOpen: Observable<boolean>): DomArg;
renameDoc(value: string): Promise<void>;
updateCurrentDoc(urlId: string, openMode: OpenDocMode): Promise<Document>;
refreshCurrentDoc(doc: DocInfo): Promise<Document>;
updateDocUsage(docUsage: DocUsage): void;
}
export interface ImportSource {
@ -109,8 +112,10 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
// Observable set to the instance of GristDoc once it's created.
public readonly gristDoc = Observable.create<GristDoc|null>(this, null);
public readonly dataLimitStatus = Observable.create<DataLimitStatus>(this, null);
public readonly rowCount = Observable.create<RowCount>(this, 'pending');
public readonly dataLimitStatus = Observable.create<DataLimitStatus|undefined>(this, null);
public readonly dataSizeBytes = Observable.create<DataSize>(this, 'pending');
public readonly attachmentsSizeBytes = Observable.create<AttachmentsSize>(this, 'pending');
// Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the
// URL, and when it changes, we need to re-open.
@ -203,6 +208,13 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
return urlState().pushUrl(nextState, {avoidReload: true, ...options});
}
public updateDocUsage(docUsage: DocUsage) {
this.rowCount.set(docUsage.rowCount);
this.dataLimitStatus.set(docUsage.dataLimitStatus);
this.dataSizeBytes.set(docUsage.dataSizeBytes);
this.attachmentsSizeBytes.set(docUsage.attachmentsSizeBytes);
}
private _onOpenError(err: Error) {
if (err instanceof CancelledError) {
// This means that we started loading a new doc before the previous one finished loading.
@ -260,8 +272,9 @@ export class DocPageModelImpl extends Disposable implements DocPageModel {
doc.userOverride = openDocResponse.userOverride || null;
this.currentDoc.set({...doc});
}
this.rowCount.set(openDocResponse.rowCount);
this.dataLimitStatus.set(openDocResponse.dataLimitStatus);
if (openDocResponse.docUsage) {
this.updateDocUsage(openDocResponse.docUsage);
}
const gdModule = await gristDocModulePromise;
const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier);
flow.checkIfCancelled();

@ -537,6 +537,10 @@ const cssCmdKey = styled('span', `
margin-left: 16px;
color: ${colors.slate};
margin-right: -12px;
.${weasel.cssMenuItem.className}-sel > & {
color: ${colors.lightGrey};
}
`);
const cssAnnotateMenuItem = styled('span', `

@ -25,5 +25,4 @@ export interface ActionGroup extends MinimalActionGroup {
user: string;
primaryAction: string; // The name of the first user action in the ActionGroup.
internal: boolean; // True if it is inappropriate to log/undo the action.
rowCount?: number;
}

@ -1,8 +1,8 @@
import {MinimalActionGroup} from 'app/common/ActionGroup';
import {TableDataAction} from 'app/common/DocActions';
import {DocUsage} from 'app/common/DocUsage';
import {Role} from 'app/common/roles';
import {StringUnion} from 'app/common/StringUnion';
import {DataLimitStatus, RowCount} from 'app/common/Usage';
import {FullUser} from 'app/common/UserAPI';
// Possible flavors of items in a list of documents.
@ -43,10 +43,9 @@ export interface OpenLocalDocResult {
clientId: string; // the docFD is meaningful only in the context of this session
doc: {[tableId: string]: TableDataAction};
log: MinimalActionGroup[];
rowCount: RowCount;
recoveryMode?: boolean;
userOverride?: UserOverride;
dataLimitStatus?: DataLimitStatus;
docUsage?: DocUsage;
}
export interface UserOverride {

@ -0,0 +1,29 @@
import {ApiError} from 'app/common/ApiError';
export interface DocUsage {
dataLimitStatus: DataLimitStatus;
rowCount: RowCount;
dataSizeBytes: DataSize;
attachmentsSizeBytes: AttachmentsSize;
}
type NumberOrStatus = number | 'hidden' | 'pending';
export type RowCount = NumberOrStatus;
export type DataSize = NumberOrStatus;
export type AttachmentsSize = NumberOrStatus;
export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null;
export type NonHidden<T> = Exclude<T, 'hidden'>;
// Ratio of usage at which we start telling users that they're approaching limits.
export const APPROACHING_LIMIT_RATIO = 0.9;
export class LimitExceededError extends ApiError {
constructor(message: string) {
super(message, 413);
}
}

@ -1,6 +0,0 @@
export type RowCount = number | 'hidden' | 'pending';
export type DataLimitStatus = null | 'approachingLimit' | 'gracePeriod' | 'deleteOnly';
// Ratio of the row/data size limit where we tell users that they're approaching the limit.
export const APPROACHING_LIMIT_RATIO = 0.9;

@ -935,3 +935,16 @@ export function assertIsDefined<T>(name: string, value: T): asserts value is Non
throw new Error(`Expected '${name}' to be defined, but received ${value}`);
}
}
/**
* Calls function `fn`, passes any thrown errors to function `recover`, and finally calls `fn`
* once more if `recover` doesn't throw.
*/
export async function retryOnce<T>(fn: () => Promise<T>, recover: (e: unknown) => Promise<void>): Promise<T> {
try {
return await fn();
} catch (e) {
await recover(e);
return await fn();
}
}

@ -35,16 +35,24 @@ import {
import {DocData} from 'app/common/DocData';
import {DocSnapshots} from 'app/common/DocSnapshot';
import {DocumentSettings} from 'app/common/DocumentSettings';
import {
APPROACHING_LIMIT_RATIO,
AttachmentsSize,
DataLimitStatus,
DataSize,
DocUsage,
LimitExceededError,
NonHidden,
RowCount
} from 'app/common/DocUsage';
import {normalizeEmail} from 'app/common/emails';
import {Features} from 'app/common/Features';
import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
import {byteString, countIf, safeJsonParse} from 'app/common/gutil';
import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
import {InactivityTimer} from 'app/common/InactivityTimer';
import {canEdit} from 'app/common/roles';
import {schema, SCHEMA_VERSION} from 'app/common/schema';
import {MetaRowRecord} from 'app/common/TableData';
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {APPROACHING_LIMIT_RATIO, DataLimitStatus, RowCount} from 'app/common/Usage';
import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
import {convertFromColumn} from 'app/common/ValueConverter';
import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
@ -173,8 +181,9 @@ export class ActiveDoc extends EventEmitter {
private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured.
private _lastDataSizeMeasurement: number = 0; // Timestamp when dbstat data size was last measured.
private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
private _rowCount: RowCount = 'pending';
private _dataSize?: number;
private _rowCount: NonHidden<RowCount> = 'pending';
private _dataSize: NonHidden<DataSize> = 'pending';
private _attachmentsSize: NonHidden<AttachmentsSize> = 'pending';
private _productFeatures?: Features;
private _gracePeriodStart: Date|null = null;
@ -245,8 +254,8 @@ export class ActiveDoc extends EventEmitter {
public get isShuttingDown(): boolean { return this._shuttingDown; }
public get rowLimitRatio() {
if (!this._rowLimit || this._rowLimit <= 0 || typeof this._rowCount !== 'number') {
// Invalid row limits are currently treated as if they are undefined.
if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') {
// If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
return 0;
}
@ -254,8 +263,8 @@ export class ActiveDoc extends EventEmitter {
}
public get dataSizeLimitRatio() {
if (!this._dataSizeLimit || this._dataSizeLimit <= 0 || !this._dataSize) {
// Invalid data size limits are currently treated as if they are undefined.
if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') {
// If limit can't be enforced (e.g. undefined, non-positive), assume no limit.
return 0;
}
@ -282,15 +291,17 @@ export class ActiveDoc extends EventEmitter {
return null;
}
public async getRowCount(docSession: OptDocSession): Promise<RowCount> {
const hasFullReadAccess = await this._granularAccess.canReadEverything(docSession);
const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
return hasFullReadAccess && hasEditRole ? this._rowCount : 'hidden';
public get docUsage(): DocUsage {
return {
dataLimitStatus: this.dataLimitStatus,
rowCount: this._rowCount,
dataSizeBytes: this._dataSize,
attachmentsSizeBytes: this._attachmentsSize,
};
}
public async getDataLimitStatus(docSession: OptDocSession): Promise<DataLimitStatus> {
const hasEditRole = canEdit(await this._granularAccess.getNominalAccess(docSession));
return hasEditRole ? this.dataLimitStatus : null;
public getFilteredDocUsage(docSession: OptDocSession): Promise<DocUsage> {
return this._granularAccess.filterDocUsage(docSession, this.docUsage);
}
public async getUserOverride(docSession: OptDocSession) {
@ -692,10 +703,31 @@ export class ActiveDoc extends EventEmitter {
const userId = getDocSessionUserId(docSession);
const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId));
try {
await this._checkDocAttachmentsLimit(upload);
// We'll assert that the upload won't cause limits to be exceeded, retrying once after
// soft-deleting any unused attachments.
await retryOnce(
() => this._assertUploadSizeBelowLimit(upload),
async (e) => {
if (!(e instanceof LimitExceededError)) { throw e; }
// Check if any attachments are unused and can be soft-deleted to reduce the existing
// total size. We could do this from the beginning, but updateUsedAttachmentsIfNeeded
// is potentially expensive, so this optimises for the common case of not exceeding the limit.
const hadChanges = await this.updateUsedAttachmentsIfNeeded();
if (hadChanges) {
await this._updateAttachmentsSize();
} else {
// No point in retrying if nothing changed.
throw new LimitExceededError("Exceeded attachments limit for document");
}
}
);
const userActions: UserAction[] = await Promise.all(
upload.files.map(file => this._prepAttachment(docSession, file)));
const result = await this.applyUserActions(docSession, userActions);
this._updateAttachmentsSize().catch(e => {
this._log.warn(docSession, 'failed to update attachments size', e);
});
return result.retValues;
} finally {
await globalUploadSet.cleanup(uploadId);
@ -1352,7 +1384,7 @@ export class ActiveDoc extends EventEmitter {
* so that undo can 'undelete' attachments.
* Returns true if any changes were made, i.e. some row(s) of _grist_Attachments were updated.
*/
public async updateUsedAttachments() {
public async updateUsedAttachmentsIfNeeded() {
const changes = await this.docStorage.scanAttachmentsForUsageChanges();
if (!changes.length) {
return false;
@ -1371,7 +1403,8 @@ export class ActiveDoc extends EventEmitter {
* @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago.
*/
public async removeUnusedAttachments(expiredOnly: boolean) {
await this.updateUsedAttachments();
const hadChanges = await this.updateUsedAttachmentsIfNeeded();
if (hadChanges) { await this._updateAttachmentsSize(); }
const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly);
if (rowIds.length) {
const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
@ -1807,6 +1840,10 @@ export class ActiveDoc extends EventEmitter {
const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT;
this._inactivityTimer.setDelay(closeTimeout);
this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`);
// TODO: Initialize data and attachments size from Document.usage once it's available.
this._updateAttachmentsSize().catch(e => {
this._log.warn(docSession, 'failed to update attachments size', e);
});
} catch (err) {
this._fullyLoaded = true;
if (!this._shuttingDown) {
@ -1935,35 +1972,32 @@ export class ActiveDoc extends EventEmitter {
/**
* Throw an error if the provided upload would exceed the total attachment filesize limit for this document.
*/
private async _checkDocAttachmentsLimit(upload: UploadInfo) {
const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
if (!maxSize) {
// This document has no limit, nothing to check.
return;
}
private async _assertUploadSizeBelowLimit(upload: UploadInfo) {
// Minor flaw: while we don't double-count existing duplicate files in the total size,
// we don't check here if any of the uploaded files already exist and could be left out of the calculation.
const totalAddedSize = sum(upload.files.map(f => f.size));
const uploadSizeBytes = sum(upload.files.map(f => f.size));
if (await this._isUploadSizeBelowLimit(uploadSizeBytes)) { return; }
// Returns true if this upload won't bring the total over the limit.
const isOK = async () => (await this.docStorage.getTotalAttachmentFileSizes()) + totalAddedSize <= maxSize;
// TODO probably want a nicer error message here.
throw new LimitExceededError("Exceeded attachments limit for document");
}
if (await isOK()) {
return;
}
/**
* Returns true if an upload with size `uploadSizeBytes` won't cause attachment size
* limits to be exceeded.
*/
private async _isUploadSizeBelowLimit(uploadSizeBytes: number): Promise<boolean> {
const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument;
if (!maxSize) { return true; }
// Looks like the limit is being exceeded.
// Check if any attachments are unused and can be soft-deleted to reduce the existing total size.
// We could do this from the beginning, but updateUsedAttachments is potentially expensive,
// so this optimises the common case of not exceeding the limit.
// updateUsedAttachments returns true if there were any changes. Otherwise there's no point checking isOK again.
if (await this.updateUsedAttachments() && await isOK()) {
return;
}
const currentSize = this._attachmentsSize !== 'pending'
? this._attachmentsSize
: await this.docStorage.getTotalAttachmentFileSizes();
return currentSize + uploadSizeBytes <= maxSize;
}
// TODO probably want a nicer error message here.
throw new Error("Exceeded attachments limit for document");
private async _updateAttachmentsSize() {
this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes();
}
}
@ -1989,3 +2023,8 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table
}
return tableRefs[tableRowIndex];
}
// Helper that returns true if `limit` is set to a valid, positive number.
function isEnforceableLimit(limit: number | undefined): limit is number {
return limit !== undefined && limit > 0;
}

@ -248,7 +248,7 @@ export class DocWorkerApi {
// Mostly for testing
this._app.post('/api/docs/:docId/attachments/updateUsed', canEdit, withDoc(async (activeDoc, req, res) => {
await activeDoc.updateUsedAttachments();
await activeDoc.updateUsedAttachmentsIfNeeded();
res.json(null);
}));
this._app.post('/api/docs/:docId/attachments/removeUnused', isOwner, withDoc(async (activeDoc, req, res) => {

@ -9,6 +9,7 @@ import {ApiError} from 'app/common/ApiError';
import {mapSetOrClear} from 'app/common/AsyncCreate';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
import {DocUsage} from 'app/common/DocUsage';
import {Invite} from 'app/common/sharing';
import {tbind} from 'app/common/tbind';
import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
@ -313,24 +314,28 @@ export class DocManager extends EventEmitter {
}
}
const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([
const [metaTables, recentActions, userOverride] = await Promise.all([
activeDoc.fetchMetaTables(docSession),
activeDoc.getRecentMinimalActions(docSession),
activeDoc.getUserOverride(docSession),
activeDoc.getRowCount(docSession),
activeDoc.getDataLimitStatus(docSession),
]);
const result = {
let docUsage: DocUsage | undefined;
try {
docUsage = await activeDoc.getFilteredDocUsage(docSession);
} catch (e) {
log.warn("DocManager.openDoc failed to get doc usage", e);
}
const result: OpenLocalDocResult = {
docFD: docSession.fd,
clientId: docSession.client.clientId,
doc: metaTables,
log: recentActions,
recoveryMode: activeDoc.recoveryMode,
userOverride,
rowCount,
dataLimitStatus,
} as OpenLocalDocResult;
docUsage,
};
if (!activeDoc.muted) {
this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));

@ -1238,9 +1238,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
/**
* Returns the total number of bytes used for storing attachments that haven't been soft-deleted.
* May be stale if ActiveDoc.updateUsedAttachments isn't called first.
* May be stale if ActiveDoc.updateUsedAttachmentsIfNeeded isn't called first.
*/
public async getTotalAttachmentFileSizes() {
public async getTotalAttachmentFileSizes(): Promise<number> {
const result = await this.get(`
SELECT SUM(len) AS total
FROM (
@ -1258,7 +1258,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
GROUP BY meta.fileIdent
)
`);
return result!.total as number;
return result!.total ?? 0;
}
/**
@ -1307,7 +1307,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
/**
* Return row IDs of unused attachments in _grist_Attachments.
* Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachments.
* Uses the timeDeleted column which is updated in ActiveDoc.updateUsedAttachmentsIfNeeded.
* @param expiredOnly: if true, only return attachments where timeDeleted is at least
* ATTACHMENTS_EXPIRY_DAYS days ago.
*/

@ -10,6 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app
import { TableDataAction, UserAction } from 'app/common/DocActions';
import { DocData } from 'app/common/DocData';
import { UserOverride } from 'app/common/DocListAPI';
import { DocUsage } from 'app/common/DocUsage';
import { normalizeEmail } from 'app/common/emails';
import { ErrorWithCode } from 'app/common/ErrorWithCode';
import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause';
@ -101,6 +102,12 @@ const SURPRISING_ACTIONS = new Set([
// Actions we'll allow unconditionally for now.
const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']);
interface DocUpdateMessage {
actionGroup: ActionGroup;
docActions: DocAction[];
docUsage: DocUsage;
}
/**
* Granular access for a single bundle, in different phases.
*/
@ -108,7 +115,7 @@ export interface GranularAccessForBundle {
canApplyBundle(): Promise<void>;
appliedBundle(): Promise<void>;
finishedBundle(): Promise<void>;
sendDocUpdateForBundle(actionGroup: ActionGroup): Promise<void>;
sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise<void>;
}
/**
@ -362,13 +369,16 @@ export class GranularAccess implements GranularAccessForBundle {
/**
* Filter an ActionGroup to be sent to a client.
*/
public async filterActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<ActionGroup> {
if (await this.allowActionGroup(docSession, actionGroup)) { return actionGroup; }
public async filterActionGroup(
docSession: OptDocSession,
actionGroup: ActionGroup,
options: {role?: Role | null} = {}
): Promise<ActionGroup> {
if (await this.allowActionGroup(docSession, actionGroup, options)) { return actionGroup; }
// For now, if there's any nuance at all, suppress the summary and description.
const result: ActionGroup = { ...actionGroup };
result.actionSummary = createEmptyActionSummary();
result.desc = '';
result.rowCount = undefined;
return result;
}
@ -376,8 +386,33 @@ export class GranularAccess implements GranularAccessForBundle {
* Check whether an ActionGroup can be sent to the client. TODO: in future, we'll want
* to filter acceptable parts of ActionGroup, rather than denying entirely.
*/
public async allowActionGroup(docSession: OptDocSession, actionGroup: ActionGroup): Promise<boolean> {
return this.canReadEverything(docSession);
public async allowActionGroup(
docSession: OptDocSession,
_actionGroup: ActionGroup,
options: {role?: Role | null} = {}
): Promise<boolean> {
return this.canReadEverything(docSession, options);
}
/**
* Filter DocUsage to be sent to a client.
*/
public async filterDocUsage(
docSession: OptDocSession,
docUsage: DocUsage,
options: {role?: Role | null} = {}
): Promise<DocUsage> {
const result: DocUsage = { ...docUsage };
const role = options.role ?? await this.getNominalAccess(docSession);
const hasEditRole = canEdit(role);
if (!hasEditRole) { result.dataLimitStatus = null; }
const hasFullReadAccess = await this.canReadEverything(docSession);
if (!hasEditRole || !hasFullReadAccess) {
result.rowCount = 'hidden';
result.dataSizeBytes = 'hidden';
result.attachmentsSizeBytes = 'hidden';
}
return result;
}
/**
@ -577,8 +612,11 @@ export class GranularAccess implements GranularAccessForBundle {
* Check whether user can read everything in document. Checks both home-level and doc-level
* permissions.
*/
public async canReadEverything(docSession: OptDocSession): Promise<boolean> {
const access = await this.getNominalAccess(docSession);
public async canReadEverything(
docSession: OptDocSession,
options: {role?: Role | null} = {}
): Promise<boolean> {
const access = options.role ?? await this.getNominalAccess(docSession);
if (!canView(access)) { return false; }
const permInfo = await this._getAccess(docSession);
return this.getReadPermission(permInfo.getFullAccess()) === 'allow';
@ -709,11 +747,11 @@ export class GranularAccess implements GranularAccessForBundle {
/**
* Broadcast document changes to all clients, with appropriate filtering.
*/
public async sendDocUpdateForBundle(actionGroup: ActionGroup) {
public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) {
if (!this._activeBundle) { throw new Error('no active bundle'); }
const { docActions, docSession } = this._activeBundle;
const client = docSession && docSession.client || null;
const message = { actionGroup, docActions };
const message: DocUpdateMessage = { actionGroup, docActions, docUsage };
await this._docClients.broadcastDocMessage(client, 'docUserAction',
message,
(_docSession) => this._filterDocUpdate(_docSession, message));
@ -866,18 +904,18 @@ export class GranularAccess implements GranularAccessForBundle {
* This filters a message being broadcast to all clients to be appropriate for one
* particular client, if that client may need some material filtered out.
*/
private async _filterDocUpdate(docSession: OptDocSession, message: {
actionGroup: ActionGroup,
docActions: DocAction[]
}) {
private async _filterDocUpdate(docSession: OptDocSession, message: DocUpdateMessage) {
if (!this._activeBundle) { throw new Error('no active bundle'); }
if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
return message;
}
const role = await this.getNominalAccess(docSession);
const result = {
actionGroup: await this.filterActionGroup(docSession, message.actionGroup),
docActions: await this.filterOutgoingDocActions(docSession, message.docActions),
...message,
docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}),
};
if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) {
return result;
}
result.actionGroup = await this.filterActionGroup(docSession, message.actionGroup, {role});
result.docActions = await this.filterOutgoingDocActions(docSession, message.docActions);
if (result.docActions.length === 0) { return null; }
return result;
}

@ -313,9 +313,8 @@ export class Sharing {
internal,
});
actionGroup.actionSummary = actionSummary;
actionGroup.rowCount = sandboxActionBundle.rowCount;
await accessControl.appliedBundle();
await accessControl.sendDocUpdateForBundle(actionGroup);
await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage);
if (docSession) {
docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0;
}

@ -1725,7 +1725,7 @@ function testDocApi() {
assert.equal(resp.status, 200);
// Remove the not expired attachments (2 and 3).
// We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachments.
// We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded.
resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([]);

Loading…
Cancel
Save