diff --git a/app/client/components/DocComm.ts b/app/client/components/DocComm.ts index f8430068..b57584dc 100644 --- a/app/client/components/DocComm.ts +++ b/app/client/components/DocComm.ts @@ -5,7 +5,7 @@ import {ActionGroup} from 'app/common/ActionGroup'; import {ActiveDocAPI, ApplyUAOptions, ApplyUAResult} from 'app/common/ActiveDocAPI'; import {DocAction, UserAction} from 'app/common/DocActions'; import {OpenLocalDocResult} from 'app/common/DocListAPI'; -import {DocUsage} from 'app/common/DocUsage'; +import {FilteredDocUsageSummary} from 'app/common/DocUsage'; import {docUrl} from 'app/common/urlUtils'; import {Events as BackboneEvents} from 'backbone'; import {Disposable, Emitter} from 'grainjs'; @@ -18,7 +18,7 @@ export interface DocUserAction extends CommMessage { data: { docActions: DocAction[]; actionGroup: ActionGroup; - docUsage: DocUsage; + docUsage: FilteredDocUsageSummary; error?: string; }; } diff --git a/app/client/components/DocUsageBanner.ts b/app/client/components/DocUsageBanner.ts index a0ee5e60..0762f7b4 100644 --- a/app/client/components/DocUsageBanner.ts +++ b/app/client/components/DocUsageBanner.ts @@ -11,12 +11,12 @@ export class DocUsageBanner extends Disposable { // Whether the banner is vertically expanded on narrow screens. private readonly _isExpanded = Observable.create(this, true); - private readonly _currentDoc = this._docPageModel.currentDoc; private readonly _currentDocId = this._docPageModel.currentDocId; - private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus; + private readonly _currentDocUsage = this._docPageModel.currentDocUsage; + private readonly _currentOrg = this._docPageModel.currentOrg; - private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => { - return doc?.workspace.org ?? null; + private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => { + return usage?.dataLimitStatus ?? null; }); private readonly _shouldShowBanner: Computed = diff --git a/app/client/components/DocumentUsage.ts b/app/client/components/DocumentUsage.ts index 80534807..bfd4f241 100644 --- a/app/client/components/DocumentUsage.ts +++ b/app/client/components/DocumentUsage.ts @@ -30,13 +30,23 @@ const ACCESS_DENIED_MESSAGE = 'Usage statistics are only available to users with */ export class DocumentUsage extends Disposable { private readonly _currentDoc = this._docPageModel.currentDoc; - private readonly _dataLimitStatus = this._docPageModel.dataLimitStatus; - private readonly _rowCount = this._docPageModel.rowCount; - private readonly _dataSizeBytes = this._docPageModel.dataSizeBytes; - private readonly _attachmentsSizeBytes = this._docPageModel.attachmentsSizeBytes; + private readonly _currentDocUsage = this._docPageModel.currentDocUsage; + private readonly _currentOrg = this._docPageModel.currentOrg; - private readonly _currentOrg = Computed.create(this, this._currentDoc, (_use, doc) => { - return doc?.workspace.org ?? null; + private readonly _dataLimitStatus = Computed.create(this, this._currentDocUsage, (_use, usage) => { + return usage?.dataLimitStatus ?? null; + }); + + private readonly _rowCount = Computed.create(this, this._currentDocUsage, (_use, usage) => { + return usage?.rowCount; + }); + + private readonly _dataSizeBytes = Computed.create(this, this._currentDocUsage, (_use, usage) => { + return usage?.dataSizeBytes; + }); + + private readonly _attachmentsSizeBytes = Computed.create(this, this._currentDocUsage, (_use, usage) => { + return usage?.attachmentsSizeBytes; }); private readonly _rowMetrics: Computed = @@ -102,7 +112,9 @@ export class DocumentUsage extends Disposable { Computed.create( this, this._currentDoc, this._rowCount, this._dataSizeBytes, this._attachmentsSizeBytes, (_use, doc, rowCount, dataSize, attachmentsSize) => { - return !doc || [rowCount, dataSize, attachmentsSize].some(metric => metric === 'pending'); + return !doc || [rowCount, dataSize, attachmentsSize].some(metric => { + return metric === 'pending' || metric === undefined; + }); } ); diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 96d40de5..692442cd 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -478,7 +478,7 @@ export class GristDoc extends DisposableWithEvents { if (schemaUpdated) { this.trigger('schemaUpdateAction', docActions); } - this.docPageModel.updateDocUsage(message.data.docUsage); + this.docPageModel.updateCurrentDocUsage(message.data.docUsage); } } diff --git a/app/client/models/DocPageModel.ts b/app/client/models/DocPageModel.ts index 1fa70795..63f0f333 100644 --- a/app/client/models/DocPageModel.ts +++ b/app/client/models/DocPageModel.ts @@ -17,7 +17,7 @@ import {confirmModal} from 'app/client/ui2018/modals'; import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow'; import {delay} from 'app/common/delay'; import {OpenDocMode, UserOverride} from 'app/common/DocListAPI'; -import {AttachmentsSize, DataLimitStatus, DataSize, DocUsage, RowCount} from 'app/common/DocUsage'; +import {FilteredDocUsageSummary} from 'app/common/DocUsage'; import {IGristUrlState, parseUrlId, UrlIdParts} from 'app/common/gristUrls'; import {getReconnectTimeout} from 'app/common/gutil'; import {canEdit} from 'app/common/roles'; @@ -44,6 +44,7 @@ export interface DocPageModel { appModel: AppModel; currentDoc: Observable; + currentDocUsage: Observable; // This block is to satisfy previous interface, but usable as this.currentDoc.get().id, etc. currentDocId: Observable; @@ -66,16 +67,11 @@ export interface DocPageModel { gristDoc: Observable; // Instance of GristDoc once it exists. - dataLimitStatus: Observable; - rowCount: Observable; - dataSizeBytes: Observable; - attachmentsSizeBytes: Observable; - createLeftPane(leftPanelOpen: Observable): DomArg; renameDoc(value: string): Promise; updateCurrentDoc(urlId: string, openMode: OpenDocMode): Promise; refreshCurrentDoc(doc: DocInfo): Promise; - updateDocUsage(docUsage: DocUsage): void; + updateCurrentDocUsage(docUsage: FilteredDocUsageSummary): void; } export interface ImportSource { @@ -88,6 +84,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { public readonly pageType = "doc"; public readonly currentDoc = Observable.create(this, null); + public readonly currentDocUsage = Observable.create(this, null); public readonly currentUrlId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.urlId : undefined); public readonly currentDocId = Computed.create(this, this.currentDoc, (use, doc) => doc ? doc.id : undefined); @@ -112,11 +109,6 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { // Observable set to the instance of GristDoc once it's created. public readonly gristDoc = Observable.create(this, null); - public readonly dataLimitStatus = Observable.create(this, null); - public readonly rowCount = Observable.create(this, 'pending'); - public readonly dataSizeBytes = Observable.create(this, 'pending'); - public readonly attachmentsSizeBytes = Observable.create(this, 'pending'); - // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the // URL, and when it changes, we need to re-open. // If making a comparison, the id of the document we are comparing with is also included @@ -199,6 +191,10 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { return this.updateCurrentDoc(doc.urlId || doc.id, doc.openMode); } + public updateCurrentDocUsage(docUsage: FilteredDocUsageSummary) { + this.currentDocUsage.set(docUsage); + } + // Replace the URL without reloading the doc. public updateUrlNoReload(urlId: string, urlOpenMode: OpenDocMode, options: {replace: boolean}) { const state = urlState().state.get(); @@ -208,13 +204,6 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { return urlState().pushUrl(nextState, {avoidReload: true, ...options}); } - public updateDocUsage(docUsage: DocUsage) { - this.rowCount.set(docUsage.rowCount); - this.dataLimitStatus.set(docUsage.dataLimitStatus); - this.dataSizeBytes.set(docUsage.dataSizeBytes); - this.attachmentsSizeBytes.set(docUsage.attachmentsSizeBytes); - } - private _onOpenError(err: Error) { if (err instanceof CancelledError) { // This means that we started loading a new doc before the previous one finished loading. @@ -273,7 +262,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { this.currentDoc.set({...doc}); } if (openDocResponse.docUsage) { - this.updateDocUsage(openDocResponse.docUsage); + this.updateCurrentDocUsage(openDocResponse.docUsage); } const gdModule = await gristDocModulePromise; const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier); diff --git a/app/common/DocLimits.ts b/app/common/DocLimits.ts new file mode 100644 index 00000000..bf170b4d --- /dev/null +++ b/app/common/DocLimits.ts @@ -0,0 +1,74 @@ +import {ApiError} from 'app/common/ApiError'; +import {APPROACHING_LIMIT_RATIO, DataLimitStatus, DocumentUsage, getUsageRatio} from 'app/common/DocUsage'; +import {Features} from 'app/common/Features'; +import * as moment from 'moment-timezone'; + +/** + * Error class indicating failure due to limits being exceeded. + */ +export class LimitExceededError extends ApiError { + constructor(message: string) { + super(message, 413); + } +} + +export interface GetDataLimitStatusParams { + docUsage: DocumentUsage | null; + productFeatures: Features | undefined; + gracePeriodStart: Date | null; +} + +/** + * Given a set of params that includes document usage, current product features, and + * a grace-period start (if any), returns the data limit status of a document. + */ +export function getDataLimitStatus(params: GetDataLimitStatusParams): DataLimitStatus { + const {docUsage, productFeatures, gracePeriodStart} = params; + const ratio = getDataLimitRatio(docUsage, productFeatures); + if (ratio > 1) { + const start = gracePeriodStart; + const days = productFeatures?.gracePeriodDays; + if (start && days && moment().diff(moment(start), 'days') >= days) { + return 'deleteOnly'; + } else { + return 'gracePeriod'; + } + } else if (ratio > APPROACHING_LIMIT_RATIO) { + return 'approachingLimit'; + } else { + return null; + } +} + +/** + * Given `docUsage` and `productFeatures`, returns the highest usage ratio + * across all data-related limits (currently only row count and data size). + */ +export function getDataLimitRatio( + docUsage: DocumentUsage | null, + productFeatures: Features | undefined +): number { + if (!docUsage) { return 0; } + + const {rowCount, dataSizeBytes} = docUsage; + const maxRows = productFeatures?.baseMaxRowsPerDocument; + const maxDataSize = productFeatures?.baseMaxDataSizePerDocument; + const rowRatio = getUsageRatio(rowCount, maxRows); + const dataSizeRatio = getUsageRatio(dataSizeBytes, maxDataSize); + return Math.max(rowRatio, dataSizeRatio); +} + +/** + * Maps `dataLimitStatus` status to an integer and returns it; larger integer + * values indicate a more "severe" status. + * + * Useful for relatively comparing the severity of two statuses. + */ +export function getSeverity(dataLimitStatus: DataLimitStatus): number { + switch (dataLimitStatus) { + case null: { return 0; } + case 'approachingLimit': { return 1; } + case 'gracePeriod': { return 2; } + case 'deleteOnly': { return 3; } + } +} diff --git a/app/common/DocListAPI.ts b/app/common/DocListAPI.ts index 9645ed97..be76442e 100644 --- a/app/common/DocListAPI.ts +++ b/app/common/DocListAPI.ts @@ -1,6 +1,6 @@ import {MinimalActionGroup} from 'app/common/ActionGroup'; import {TableDataAction} from 'app/common/DocActions'; -import {DocUsage} from 'app/common/DocUsage'; +import {FilteredDocUsageSummary} from 'app/common/DocUsage'; import {Role} from 'app/common/roles'; import {StringUnion} from 'app/common/StringUnion'; import {FullUser} from 'app/common/UserAPI'; @@ -45,7 +45,7 @@ export interface OpenLocalDocResult { log: MinimalActionGroup[]; recoveryMode?: boolean; userOverride?: UserOverride; - docUsage?: DocUsage; + docUsage?: FilteredDocUsageSummary; } export interface UserOverride { diff --git a/app/common/DocUsage.ts b/app/common/DocUsage.ts index 02099d22..afe842bc 100644 --- a/app/common/DocUsage.ts +++ b/app/common/DocUsage.ts @@ -1,29 +1,60 @@ -import {ApiError} from 'app/common/ApiError'; - -export interface DocUsage { - dataLimitStatus: DataLimitStatus; - rowCount: RowCount; - dataSizeBytes: DataSize; - attachmentsSizeBytes: AttachmentsSize; +export interface DocumentUsage { + rowCount?: number; + dataSizeBytes?: number; + attachmentsSizeBytes?: number; } -type NumberOrStatus = number | 'hidden' | 'pending'; - -export type RowCount = NumberOrStatus; - -export type DataSize = NumberOrStatus; - -export type AttachmentsSize = NumberOrStatus; - export type DataLimitStatus = 'approachingLimit' | 'gracePeriod' | 'deleteOnly' | null; -export type NonHidden = Exclude; +type DocUsageOrPending = { + [Metric in keyof Required]: Required[Metric] | 'pending' +} + +export interface DocUsageSummary extends DocUsageOrPending { + dataLimitStatus: DataLimitStatus; +} + +// Count of non-removed documents in an org, grouped by data limit status. +export type OrgUsageSummary = Record, number>; + +type FilteredDocUsage = { + [Metric in keyof DocUsageOrPending]: DocUsageOrPending[Metric] | 'hidden' +} + +export interface FilteredDocUsageSummary extends FilteredDocUsage { + dataLimitStatus: DataLimitStatus; +} // Ratio of usage at which we start telling users that they're approaching limits. export const APPROACHING_LIMIT_RATIO = 0.9; -export class LimitExceededError extends ApiError { - constructor(message: string) { - super(message, 413); +/** + * Computes a ratio of `usage` to `limit`, if possible. Returns 0 if `usage` or `limit` + * is invalid or undefined. + */ +export function getUsageRatio(usage: number | undefined, limit: number | undefined): number { + if (!isEnforceableLimit(limit) || usage === undefined || usage < 0) { + // Treat undefined or invalid values as having 0 usage. + return 0; } + + return usage / limit; +} + +/** + * Returns an empty org usage summary with values initialized to 0. + */ + export function createEmptyOrgUsageSummary(): OrgUsageSummary { + return { + approachingLimit: 0, + gracePeriod: 0, + deleteOnly: 0, + }; +} + +/** + * Returns true if `limit` is defined and is a valid, positive number. + */ +function isEnforceableLimit(limit: number | undefined): limit is number { + return limit !== undefined && limit > 0; } diff --git a/app/common/UserAPI.ts b/app/common/UserAPI.ts index 9d392442..189a91d5 100644 --- a/app/common/UserAPI.ts +++ b/app/common/UserAPI.ts @@ -5,6 +5,7 @@ import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI'; import {BrowserSettings} from 'app/common/BrowserSettings'; import {BulkColValues, TableColValues, TableRecordValue, TableRecordValues, UserAction} from 'app/common/DocActions'; import {DocCreationInfo, OpenDocMode} from 'app/common/DocListAPI'; +import {OrgUsageSummary} from 'app/common/DocUsage'; import {Features} from 'app/common/Features'; import {ICustomWidget} from 'app/common/CustomWidget'; import {isClient} from 'app/common/gristUrls'; @@ -288,6 +289,7 @@ export interface UserAPI { getWorkspace(workspaceId: number): Promise; getOrg(orgId: number|string): Promise; getOrgWorkspaces(orgId: number|string): Promise; + getOrgUsageSummary(orgId: number|string): Promise; getTemplates(onlyFeatured?: boolean): Promise; getDoc(docId: string): Promise; newOrg(props: Partial): Promise; @@ -448,6 +450,10 @@ export class UserAPIImpl extends BaseAPI implements UserAPI { { method: 'GET' }); } + public async getOrgUsageSummary(orgId: number|string): Promise { + return this.requestJson(`${this._url}/api/orgs/${orgId}/usage`, { method: 'GET' }); + } + public async getTemplates(onlyFeatured: boolean = false): Promise { return this.requestJson(`${this._url}/api/templates?onlyFeatured=${onlyFeatured ? 1 : 0}`, { method: 'GET' }); } diff --git a/app/gen-server/ApiServer.ts b/app/gen-server/ApiServer.ts index 0ad019b5..e028561a 100644 --- a/app/gen-server/ApiServer.ts +++ b/app/gen-server/ApiServer.ts @@ -146,6 +146,15 @@ export class ApiServer { return sendReply(req, res, query); })); + // GET /api/orgs/:oid/usage + // Get usage summary of all un-deleted documents in the organization. + // Only accessible to org owners. + this._app.get('/api/orgs/:oid/usage', expressWrap(async (req, res) => { + const org = getOrgKey(req); + const usage = await this._dbManager.getOrgUsageSummary(getScope(req), org); + return sendOkReply(req, res, usage); + })); + // POST /api/orgs // Body params: name (required), domain // Create a new org. @@ -194,7 +203,7 @@ export class ApiServer { return sendReply(req, res, query); })); - // // DELETE /api/workspaces/:wid + // DELETE /api/workspaces/:wid // Delete the specified workspace and all included docs. this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => { const wsId = integerParam(req.params.wid, 'wid'); diff --git a/app/gen-server/entity/Document.ts b/app/gen-server/entity/Document.ts index a5b7289b..3b3101c0 100644 --- a/app/gen-server/entity/Document.ts +++ b/app/gen-server/entity/Document.ts @@ -1,4 +1,5 @@ import {ApiError} from 'app/common/ApiError'; +import {DocumentUsage} from 'app/common/DocUsage'; import {Role} from 'app/common/roles'; import {DocumentOptions, DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI"; import {nativeValues} from 'app/gen-server/lib/values'; @@ -65,6 +66,9 @@ export class Document extends Resource { @OneToMany(_type => Secret, secret => secret.doc) public secrets: Secret[]; + @Column({name: 'usage', type: nativeValues.jsonEntityType, nullable: true}) + public usage: DocumentUsage | null; + public checkProperties(props: any): props is Partial { return super.checkProperties(props, documentPropertyKeys); } diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 4b5f4bc7..c605cd89 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -1,5 +1,7 @@ import {ApiError} from 'app/common/ApiError'; import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate'; +import {getDataLimitStatus} from 'app/common/DocLimits'; +import {createEmptyOrgUsageSummary, DocumentUsage, OrgUsageSummary} from 'app/common/DocUsage'; import {normalizeEmail} from 'app/common/emails'; import {canAddOrgMembers, Features} from 'app/common/Features'; import {buildUrlId, MIN_URLID_PREFIX_LENGTH, parseUrlId} from 'app/common/gristUrls'; @@ -231,6 +233,12 @@ function stringifyUrlIdOrg(urlId: string, org?: string): string { return `${urlId}:${org}`; } +export interface DocumentMetadata { + // ISO 8601 UTC date (e.g. the output of new Date().toISOString()). + updatedAt?: string; + usage?: DocumentUsage|null; +} + /** * HomeDBManager handles interaction between the ApiServer and the Home database, * encapsulating the typeorm logic. @@ -922,6 +930,44 @@ export class HomeDBManager extends EventEmitter { return result; } + /** + * Returns an organization's usage summary (e.g. count of documents that are approaching or exceeding + * limits). + */ + public async getOrgUsageSummary(scope: Scope, orgKey: string|number): Promise { + // Check that an owner of the org is making the request. + const markPermissions = Permissions.OWNER; + let orgQuery = this.org(scope, orgKey, { + markPermissions, + needRealOrg: true + }); + orgQuery = this._addFeatures(orgQuery); + const orgQueryResult = await verifyIsPermitted(orgQuery); + const org: Organization = this.unwrapQueryResult(orgQueryResult); + const productFeatures = org.billingAccount.product.features; + + // Grab all the non-removed documents in the org. + let docsQuery = this._docs() + .innerJoin('docs.workspace', 'workspaces') + .innerJoin('workspaces.org', 'orgs') + .where('docs.workspace_id = workspaces.id') + .andWhere('workspaces.removed_at IS NULL AND docs.removed_at IS NULL'); + docsQuery = this._whereOrg(docsQuery, orgKey); + if (this.isMergedOrg(orgKey)) { + docsQuery = docsQuery.andWhere('orgs.owner_id = :userId', {userId: scope.userId}); + } + const docsQueryResult = await this._verifyAclPermissions(docsQuery, { scope, emptyAllowed: true }); + const docs: Document[] = this.unwrapQueryResult(docsQueryResult); + + // Return an aggregate count of documents, grouped by data limit status. + const summary = createEmptyOrgUsageSummary(); + for (const {usage: docUsage, gracePeriodStart} of docs) { + const dataLimitStatus = getDataLimitStatus({docUsage, gracePeriodStart, productFeatures}); + if (dataLimitStatus) { summary[dataLimitStatus] += 1; } + } + return summary; + } + /** * Compute the best access option for an organization, from the * users available to the client. If none of the options can access @@ -2364,12 +2410,13 @@ export class HomeDBManager extends EventEmitter { } /** - * Updates the updatedAt values for several docs. Takes a map where each entry maps a docId to - * an ISO date string representing the new updatedAt time. This is not a part of the API, it - * should be called only by the HostedMetadataManager when a change is made to a doc. + * Updates the updatedAt and usage values for several docs. Takes a map where each entry maps + * a docId to a metadata object containing the updatedAt and/or usage values. This is not a part + * of the API, it should be called only by the HostedMetadataManager when a change is made to a + * doc. */ - public async setDocsUpdatedAt( - docUpdateMap: {[docId: string]: string} + public async setDocsMetadata( + docUpdateMap: {[docId: string]: DocumentMetadata} ): Promise> { if (!docUpdateMap || Object.keys(docUpdateMap).length === 0) { return { @@ -2382,7 +2429,7 @@ export class HomeDBManager extends EventEmitter { const updateTasks = docIds.map(docId => { return manager.createQueryBuilder() .update(Document) - .set({updatedAt: docUpdateMap[docId]}) + .set(docUpdateMap[docId]) .where("id = :docId", {docId}) .execute(); }); diff --git a/app/gen-server/migration/1651469582887-DocumentUsage.ts b/app/gen-server/migration/1651469582887-DocumentUsage.ts new file mode 100644 index 00000000..0d349fee --- /dev/null +++ b/app/gen-server/migration/1651469582887-DocumentUsage.ts @@ -0,0 +1,18 @@ +import {nativeValues} from "app/gen-server/lib/values"; +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class DocumentUsage1651469582887 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn("docs", new TableColumn({ + name: "usage", + type: nativeValues.jsonType, + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn("docs", "usage"); + } + +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 1d6315ef..57f49cd8 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -33,21 +33,26 @@ import { UserAction } from 'app/common/DocActions'; import {DocData} from 'app/common/DocData'; +import { + getDataLimitRatio, + getDataLimitStatus, + getSeverity, + LimitExceededError, +} from 'app/common/DocLimits'; import {DocSnapshots} from 'app/common/DocSnapshot'; import {DocumentSettings} from 'app/common/DocumentSettings'; import { APPROACHING_LIMIT_RATIO, - AttachmentsSize, DataLimitStatus, - DataSize, - DocUsage, - LimitExceededError, - NonHidden, - RowCount + DocumentUsage, + DocUsageSummary, + FilteredDocUsageSummary, + getUsageRatio, } from 'app/common/DocUsage'; import {normalizeEmail} from 'app/common/emails'; import {Features} from 'app/common/Features'; import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause'; +import {parseUrlId} from 'app/common/gristUrls'; import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; import {schema, SCHEMA_VERSION} from 'app/common/schema'; @@ -180,12 +185,12 @@ export class ActiveDoc extends EventEmitter { private _fullyLoaded: boolean = false; // Becomes true once all columns are loaded/computed. private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured. private _lastDataSizeMeasurement: number = 0; // Timestamp when dbstat data size was last measured. + private _lastDataLimitStatus?: DataLimitStatus; private _fetchCache = new MapWithTTL>(DEFAULT_CACHE_TTL); - private _rowCount: NonHidden = 'pending'; - private _dataSize: NonHidden = 'pending'; - private _attachmentsSize: NonHidden = 'pending'; + private _docUsage: DocumentUsage|null = null; private _productFeatures?: Features; private _gracePeriodStart: Date|null = null; + private _isForkOrSnapshot: boolean = false; // Timer for shutting down the ActiveDoc a bit after all clients are gone. private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000); @@ -207,10 +212,30 @@ export class ActiveDoc extends EventEmitter { constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) { super(); + const {forkId, snapshotId} = parseUrlId(docName); + this._isForkOrSnapshot = Boolean(forkId || snapshotId); if (_options?.safeMode) { this._recoveryMode = true; } if (_options?.doc) { - this._productFeatures = _options.doc.workspace.org.billingAccount?.product.features; - this._gracePeriodStart = _options.doc.gracePeriodStart; + const {gracePeriodStart, workspace, usage} = _options.doc; + this._productFeatures = workspace.org.billingAccount?.product.features; + this._gracePeriodStart = gracePeriodStart; + + if (!this._isForkOrSnapshot) { + /* Note: We don't currently persist usage for forks or snapshots anywhere, so + * we need to hold off on setting _docUsage here. Normally, usage is set shortly + * after initialization finishes, after data/attachments size has finished + * calculating. However, this leaves a narrow window where forks can circumvent + * delete-only restrictions and replace the trunk document (even when the trunk + * is delete-only). This isn't very concerning today as the window is typically + * too narrow to easily exploit, and there are other ways to work around limits, + * like resetting gracePeriodStart by momentarily lowering usage. Regardless, it + * would be good to fix this eventually (perhaps around the same time we close + * up the gracePeriodStart loophole). + * + * TODO: Revisit this later and patch up the loophole. */ + this._docUsage = usage; + this._lastDataLimitStatus = this.dataLimitStatus; + } } this._docManager = docManager; this._docName = docName; @@ -253,55 +278,46 @@ export class ActiveDoc extends EventEmitter { public get isShuttingDown(): boolean { return this._shuttingDown; } - public get rowLimitRatio() { - if (!isEnforceableLimit(this._rowLimit) || this._rowCount === 'pending') { - // If limit can't be enforced (e.g. undefined, non-positive), assume no limit. - return 0; - } - return this._rowCount / this._rowLimit; + public get rowLimitRatio(): number { + return getUsageRatio( + this._docUsage?.rowCount, + this._productFeatures?.baseMaxRowsPerDocument + ); } - public get dataSizeLimitRatio() { - if (!isEnforceableLimit(this._dataSizeLimit) || this._dataSize === 'pending') { - // If limit can't be enforced (e.g. undefined, non-positive), assume no limit. - return 0; - } - - return this._dataSize / this._dataSizeLimit; + public get dataSizeLimitRatio(): number { + return getUsageRatio( + this._docUsage?.dataSizeBytes, + this._productFeatures?.baseMaxDataSizePerDocument + ); } - public get dataLimitRatio() { - return Math.max(this.rowLimitRatio, this.dataSizeLimitRatio); + public get dataLimitRatio(): number { + return getDataLimitRatio(this._docUsage, this._productFeatures); } public get dataLimitStatus(): DataLimitStatus { - const ratio = this.dataLimitRatio; - if (ratio > 1) { - const start = this._gracePeriodStart; - const days = this._productFeatures?.gracePeriodDays; - if (start && days && moment().diff(moment(start), 'days') >= days) { - return 'deleteOnly'; - } else { - return 'gracePeriod'; - } - } else if (ratio > APPROACHING_LIMIT_RATIO) { - return 'approachingLimit'; - } - return null; + return getDataLimitStatus({ + docUsage: this._docUsage, + productFeatures: this._productFeatures, + gracePeriodStart: this._gracePeriodStart, + }); } - public get docUsage(): DocUsage { + public getDocUsageSummary(): DocUsageSummary { return { dataLimitStatus: this.dataLimitStatus, - rowCount: this._rowCount, - dataSizeBytes: this._dataSize, - attachmentsSizeBytes: this._attachmentsSize, + rowCount: this._docUsage?.rowCount ?? 'pending', + dataSizeBytes: this._docUsage?.dataSizeBytes ?? 'pending', + attachmentsSizeBytes: this._docUsage?.attachmentsSizeBytes ?? 'pending', }; } - public getFilteredDocUsage(docSession: OptDocSession): Promise { - return this._granularAccess.filterDocUsage(docSession, this.docUsage); + public async getFilteredDocUsageSummary( + docSession: OptDocSession + ): Promise { + return this._granularAccess.filterDocUsageSummary(docSession, this.getDocUsageSummary()); } public async getUserOverride(docSession: OptDocSession) { @@ -431,15 +447,29 @@ export class ActiveDoc extends EventEmitter { clearInterval(interval); } + // Remove expired attachments, i.e. attachments that were soft deleted a while ago. This + // needs to happen periodically, and doing it here means we can guarantee that it happens + // even if the doc is only ever opened briefly, without having to slow down startup. + const removeAttachmentsPromise = this.removeUnusedAttachments(true, {syncUsageToDatabase: false}); + + // Update data size as well. We'll schedule a sync to the database once both this and the + // above promise settle. + const updateDataSizePromise = this._updateDataSize({syncUsageToDatabase: false}); + try { - // Remove expired attachments, i.e. attachments that were soft deleted a while ago. - // This needs to happen periodically, and doing it here means we can guarantee that it happens even if - // the doc is only ever opened briefly, without having to slow down startup. - await this.removeUnusedAttachments(true); + await removeAttachmentsPromise; } catch (e) { this._log.error(docSession, "Failed to remove expired attachments", e); } + try { + await updateDataSizePromise; + } catch (e) { + this._log.error(docSession, "Failed to update data size", e); + } + + this._syncDocUsageToDatabase(true); + try { await this._docManager.storageManager.closeDocument(this.docName); } catch (err) { @@ -715,7 +745,7 @@ export class ActiveDoc extends EventEmitter { // is potentially expensive, so this optimises for the common case of not exceeding the limit. const hadChanges = await this.updateUsedAttachmentsIfNeeded(); if (hadChanges) { - await this._updateAttachmentsSize(); + await this._updateAttachmentsSize({syncUsageToDatabase: false}); } else { // No point in retrying if nothing changed. throw new LimitExceededError("Exceeded attachments limit for document"); @@ -1401,10 +1431,13 @@ export class ActiveDoc extends EventEmitter { /** * Delete unused attachments from _grist_Attachments and gristsys_Files. * @param expiredOnly: if true, only delete attachments that were soft-deleted sufficiently long ago. + * @param options.syncUsageToDatabase: if true, schedule an update to the usage column of the docs table, if + * any unused attachments were soft-deleted. defaults to true. */ - public async removeUnusedAttachments(expiredOnly: boolean) { + public async removeUnusedAttachments(expiredOnly: boolean, options: {syncUsageToDatabase?: boolean} = {}) { + const {syncUsageToDatabase = true} = options; const hadChanges = await this.updateUsedAttachmentsIfNeeded(); - if (hadChanges) { await this._updateAttachmentsSize(); } + if (hadChanges) { await this._updateAttachmentsSize({syncUsageToDatabase}); } const rowIds = await this.docStorage.getSoftDeletedAttachmentIds(expiredOnly); if (rowIds.length) { const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds]; @@ -1468,7 +1501,7 @@ export class ActiveDoc extends EventEmitter { } public async updateRowCount(rowCount: number, docSession: OptDocSession | null) { - this._rowCount = rowCount; + this._updateDocUsage({rowCount}); log.rawInfo('Sandbox row count', {...this.getLogMeta(docSession), rowCount}); await this._checkDataLimitRatio(); @@ -1649,17 +1682,50 @@ export class ActiveDoc extends EventEmitter { }; } - private get _rowLimit(): number | undefined { - return this._productFeatures?.baseMaxRowsPerDocument; + /** + * Applies all metrics from `usage` to the current document usage state. + * Syncs updated usage to the home database by default, unless + * `options.syncUsageToDatabase` is set to false. + */ + private _updateDocUsage( + usage: Partial, + options: { + syncUsageToDatabase?: boolean + } = {} + ) { + const {syncUsageToDatabase = true} = options; + this._docUsage = {...(this._docUsage || {}), ...usage}; + if (this._lastDataLimitStatus === this.dataLimitStatus) { + // If status is unchanged, there's no need to sync usage to the database, as it currently + // won't result in any noticeable difference to site usage banners. On shutdown, we'll + // still schedule a sync so that the latest usage is persisted. + return; + } + + const lastStatus = this._lastDataLimitStatus; + this._lastDataLimitStatus = this.dataLimitStatus; + if (!syncUsageToDatabase) { return; } + + // If status decreased, we'll want to update usage in the DB with minimal delay, so that site + // usage banners show up-to-date statistics. If status increased or stayed the same, we'll + // schedule a delayed update, since it's less critical for such banners to update quickly + // when usage grows. + const didStatusDecrease = ( + lastStatus !== undefined && + getSeverity(this.dataLimitStatus) < getSeverity(lastStatus) + ); + this._syncDocUsageToDatabase(didStatusDecrease); } - private get _dataSizeLimit(): number | undefined { - return this._productFeatures?.baseMaxDataSizePerDocument; + private _syncDocUsageToDatabase(minimizeDelay = false) { + this._docManager.storageManager.scheduleUsageUpdate(this._docName, this._docUsage, minimizeDelay); } private async _updateGracePeriodStart(gracePeriodStart: Date | null) { this._gracePeriodStart = gracePeriodStart; - await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart); + if (!this._isForkOrSnapshot) { + await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart); + } } private async _checkDataLimitRatio() { @@ -1673,13 +1739,29 @@ export class ActiveDoc extends EventEmitter { private async _checkDataSizeLimitRatio(docSession: OptDocSession | null) { const start = Date.now(); - const dataSize = await this.docStorage.getDataSize(); + const dataSizeBytes = await this._updateDataSize(); const timeToMeasure = Date.now() - start; - this._dataSize = dataSize; - log.rawInfo('Data size from dbstat...', {...this.getLogMeta(docSession), dataSize, timeToMeasure}); + log.rawInfo('Data size from dbstat...', { + ...this.getLogMeta(docSession), + dataSizeBytes, + timeToMeasure, + }); await this._checkDataLimitRatio(); } + /** + * Calculates the total data size in bytes and sets it in _docUsage. Schedules + * a sync to the database, unless `options.syncUsageToDatabase` is set to false. + * + * Returns the calculated data size. + */ + private async _updateDataSize(options: {syncUsageToDatabase?: boolean} = {}): Promise { + const {syncUsageToDatabase = true} = options; + const dataSizeBytes = await this.docStorage.getDataSize(); + this._updateDocUsage({dataSizeBytes}, {syncUsageToDatabase}); + return dataSizeBytes; + } + /** * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply. */ @@ -1840,10 +1922,7 @@ export class ActiveDoc extends EventEmitter { const closeTimeout = Math.max(loadMs, 1000) * Deps.ACTIVEDOC_TIMEOUT; this._inactivityTimer.setDelay(closeTimeout); this._log.debug(docSession, `loaded in ${loadMs} ms, InactivityTimer set to ${closeTimeout} ms`); - // TODO: Initialize data and attachments size from Document.usage once it's available. - this._updateAttachmentsSize().catch(e => { - this._log.warn(docSession, 'failed to update attachments size', e); - }); + this._initializeDocUsageIfNeeded(docSession); } catch (err) { this._fullyLoaded = true; if (!this._shuttingDown) { @@ -1884,6 +1963,21 @@ export class ActiveDoc extends EventEmitter { } } + private _initializeDocUsageIfNeeded(docSession: OptDocSession) { + // TODO: Broadcast a message to clients after usage is fully calculated. + if (this._docUsage?.dataSizeBytes === undefined) { + this._updateDataSize().catch(e => { + this._log.warn(docSession, 'failed to update data size', e); + }); + } + + if (this._docUsage?.attachmentsSizeBytes === undefined) { + this._updateAttachmentsSize().catch(e => { + this._log.warn(docSession, 'failed to update attachments size', e); + }); + } + } + /** * Called before a migration. Makes sure a back-up is made. */ @@ -1990,14 +2084,22 @@ export class ActiveDoc extends EventEmitter { const maxSize = this._productFeatures?.baseMaxAttachmentsBytesPerDocument; if (!maxSize) { return true; } - const currentSize = this._attachmentsSize !== 'pending' - ? this._attachmentsSize - : await this.docStorage.getTotalAttachmentFileSizes(); + let currentSize = this._docUsage?.attachmentsSizeBytes; + currentSize = currentSize ?? await this._updateAttachmentsSize({syncUsageToDatabase: false}); return currentSize + uploadSizeBytes <= maxSize; } - private async _updateAttachmentsSize() { - this._attachmentsSize = await this.docStorage.getTotalAttachmentFileSizes(); + /** + * Calculates the total attachments size in bytes and sets it in _docUsage. Schedules + * a sync to the database, unless `options.syncUsageToDatabase` is set to false. + * + * Returns the calculated attachments size. + */ + private async _updateAttachmentsSize(options: {syncUsageToDatabase?: boolean} = {}): Promise { + const {syncUsageToDatabase = true} = options; + const attachmentsSizeBytes = await this.docStorage.getTotalAttachmentFileSizes(); + this._updateDocUsage({attachmentsSizeBytes}, {syncUsageToDatabase}); + return attachmentsSizeBytes; } } @@ -2023,8 +2125,3 @@ export function tableIdToRef(metaTables: { [p: string]: TableDataAction }, table } return tableRefs[tableRowIndex]; } - -// Helper that returns true if `limit` is set to a valid, positive number. -function isEnforceableLimit(limit: number | undefined): limit is number { - return limit !== undefined && limit > 0; -} diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index 3b5b0492..2391c2a3 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -851,6 +851,9 @@ export class DocWorkerApi { return true; } + // If Redis isn't configured, this is as far as we can go with checks. + if (!process.env.REDIS_URL) { return false; } + // Note the increased API usage on redis and in our local cache. // Update redis in the background so that the rest of the request can continue without waiting for redis. const multi = this._docWorkerMap.getRedisClient().multi(); diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index 3601c1d0..5d56e7fd 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -9,7 +9,7 @@ import {ApiError} from 'app/common/ApiError'; import {mapSetOrClear} from 'app/common/AsyncCreate'; import {BrowserSettings} from 'app/common/BrowserSettings'; import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI'; -import {DocUsage} from 'app/common/DocUsage'; +import {FilteredDocUsageSummary} from 'app/common/DocUsage'; import {Invite} from 'app/common/sharing'; import {tbind} from 'app/common/tbind'; import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; @@ -320,9 +320,9 @@ export class DocManager extends EventEmitter { activeDoc.getUserOverride(docSession), ]); - let docUsage: DocUsage | undefined; + let docUsage: FilteredDocUsageSummary | undefined; try { - docUsage = await activeDoc.getFilteredDocUsage(docSession); + docUsage = await activeDoc.getFilteredDocUsageSummary(docSession); } catch (e) { log.warn("DocManager.openDoc failed to get doc usage", e); } diff --git a/app/server/lib/DocStorageManager.ts b/app/server/lib/DocStorageManager.ts index 68875864..ca56033c 100644 --- a/app/server/lib/DocStorageManager.ts +++ b/app/server/lib/DocStorageManager.ts @@ -6,6 +6,7 @@ import * as path from 'path'; import {DocEntry, DocEntryTag} from 'app/common/DocListAPI'; import {DocSnapshots} from 'app/common/DocSnapshot'; +import {DocumentUsage} from 'app/common/DocUsage'; import * as gutil from 'app/common/gutil'; import * as Comm from 'app/server/lib/Comm'; import * as docUtils from 'app/server/lib/docUtils'; @@ -217,6 +218,14 @@ export class DocStorageManager implements IDocStorageManager { // nothing to do } + public scheduleUsageUpdate( + docName: string, + docUsage: DocumentUsage, + minimizeDelay = false + ): void { + // nothing to do + } + public testReopenStorage(): void { // nothing to do } diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index c966815d..3c8709b0 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -10,7 +10,7 @@ import { CellValue, ColValues, DocAction, getTableId, isSchemaAction } from 'app import { TableDataAction, UserAction } from 'app/common/DocActions'; import { DocData } from 'app/common/DocData'; import { UserOverride } from 'app/common/DocListAPI'; -import { DocUsage } from 'app/common/DocUsage'; +import { DocUsageSummary, FilteredDocUsageSummary } from 'app/common/DocUsage'; import { normalizeEmail } from 'app/common/emails'; import { ErrorWithCode } from 'app/common/ErrorWithCode'; import { AclMatchInput, InfoEditor, InfoView } from 'app/common/GranularAccessClause'; @@ -105,7 +105,7 @@ const OK_ACTIONS = new Set(['Calculate', 'UpdateCurrentTime']); interface DocUpdateMessage { actionGroup: ActionGroup; docActions: DocAction[]; - docUsage: DocUsage; + docUsage: DocUsageSummary; } /** @@ -115,7 +115,7 @@ export interface GranularAccessForBundle { canApplyBundle(): Promise; appliedBundle(): Promise; finishedBundle(): Promise; - sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage): Promise; + sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsageSummary): Promise; } /** @@ -395,14 +395,14 @@ export class GranularAccess implements GranularAccessForBundle { } /** - * Filter DocUsage to be sent to a client. + * Filter DocUsageSummary to be sent to a client. */ - public async filterDocUsage( + public async filterDocUsageSummary( docSession: OptDocSession, - docUsage: DocUsage, + docUsage: DocUsageSummary, options: {role?: Role | null} = {} - ): Promise { - const result: DocUsage = { ...docUsage }; + ): Promise { + const result: FilteredDocUsageSummary = { ...docUsage }; const role = options.role ?? await this.getNominalAccess(docSession); const hasEditRole = canEdit(role); if (!hasEditRole) { result.dataLimitStatus = null; } @@ -747,7 +747,7 @@ export class GranularAccess implements GranularAccessForBundle { /** * Broadcast document changes to all clients, with appropriate filtering. */ - public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsage) { + public async sendDocUpdateForBundle(actionGroup: ActionGroup, docUsage: DocUsageSummary) { if (!this._activeBundle) { throw new Error('no active bundle'); } const { docActions, docSession } = this._activeBundle; const client = docSession && docSession.client || null; @@ -909,7 +909,7 @@ export class GranularAccess implements GranularAccessForBundle { const role = await this.getNominalAccess(docSession); const result = { ...message, - docUsage: await this.filterDocUsage(docSession, message.docUsage, {role}), + docUsage: await this.filterDocUsageSummary(docSession, message.docUsage, {role}), }; if (!this._ruler.haveRules() && !this._activeBundle.hasDeliberateRuleChange) { return result; diff --git a/app/server/lib/HostedMetadataManager.ts b/app/server/lib/HostedMetadataManager.ts index b809c280..95789012 100644 --- a/app/server/lib/HostedMetadataManager.ts +++ b/app/server/lib/HostedMetadataManager.ts @@ -1,14 +1,14 @@ -import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; +import {DocumentMetadata, HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; import * as log from 'app/server/lib/log'; /** * HostedMetadataManager handles pushing document metadata changes to the Home database when - * a doc is updated. Currently only updates doc updatedAt time. + * a doc is updated. Currently updates doc updatedAt time and usage. */ export class HostedMetadataManager { - // updatedAt times as UTC ISO strings mapped by docId. - private _updatedAt: {[docId: string]: string} = {}; + // Document metadata mapped by docId. + private _metadata: {[docId: string]: DocumentMetadata} = {}; // Set if the class holder is closing and no further pushes should be scheduled. private _closing: boolean = false; @@ -22,60 +22,78 @@ export class HostedMetadataManager { // Maintains the update Promise to wait on it if the class is closing. private _push: Promise|null; + // The default delay in milliseconds between metadata pushes to the database. + private readonly _minPushDelayMs: number; + /** * Create an instance of HostedMetadataManager. - * The minPushDelay is the delay in seconds between metadata pushes to the database. + * The minPushDelay is the default delay in seconds between metadata pushes to the database. */ - constructor(private _dbManager: HomeDBManager, private _minPushDelay: number = 60) {} + constructor(private _dbManager: HomeDBManager, minPushDelay: number = 60) { + this._minPushDelayMs = minPushDelay * 1000; + } /** * Close the manager. Send out any pending updates and prevent more from being scheduled. */ public async close(): Promise { - // Finish up everything outgoing - this._closing = true; // Pushes will no longer be scheduled. + // Pushes will no longer be scheduled. + this._closing = true; + // Wait for outgoing pushes to finish before proceeding. + if (this._push) { await this._push; } if (this._timeout) { - clearTimeout(this._timeout); - this._timeout = null; // Since an update was scheduled, perform one final update now. this._update(); + if (this._push) { await this._push; } } - if (this._push) { await this._push; } } /** * Schedule a call to _update some time from now. When the update is made, it will - * store the given timestamp in the updated_at column of the docs table for the - * specified document. Timestamp should be an ISO 8601 format time, in UTC, e.g. - * the output of new Date().toISOString() + * store the given metadata in the updated_at and usage columns of the docs table for + * the specified document. + * + * If `minimizeDelay` is true, the push will be scheduled with minimum delay (0ms) and + * will cancel/overwrite an already scheduled push (if present). */ - public scheduleUpdate(docId: string, timestamp: string): void { - // Update updatedAt even if an update is already scheduled - if the update has not yet occurred, - // the more recent updatedAt time will be used. - this._updatedAt[docId] = timestamp; - if (this._timeout || this._closing) { return; } - const minDelay = this._minPushDelay * 1000; - // Set the push to occur at least the minDelay after the last push time. - const delay = Math.round(minDelay - (Date.now() - this._lastPushTime)); - this._timeout = setTimeout(() => this._update(), delay < 0 ? 0 : delay); + public scheduleUpdate(docId: string, metadata: DocumentMetadata, minimizeDelay = false): void { + if (this._closing) { return; } + + // Update metadata even if an update is already scheduled - if the update has not yet occurred, + // the more recent metadata will be used. + this._setOrUpdateMetadata(docId, metadata); + if (this._timeout && !minimizeDelay) { return; } + + this._schedulePush(minimizeDelay ? 0 : undefined); } - public setDocsUpdatedAt(docUpdateMap: {[docId: string]: string}): Promise { - return this._dbManager.setDocsUpdatedAt(docUpdateMap); + public setDocsMetadata(docUpdateMap: {[docId: string]: DocumentMetadata}): Promise { + return this._dbManager.setDocsMetadata(docUpdateMap); } /** * Push all metadata updates to the database. */ private _update(): void { - if (this._push) { return; } if (this._timeout) { clearTimeout(this._timeout); this._timeout = null; } + if (this._push) { return; } this._push = this._performUpdate() - .catch(err => { log.error("HostedMetadataManager error performing update: ", err); }) - .then(() => { this._push = null; }); + .catch(err => { + log.error("HostedMetadataManager error performing update: ", err); + }) + .then(() => { + this._push = null; + if (!this._closing && !this._timeout && Object.keys(this._metadata).length !== 0) { + // If we have metadata that hasn't been pushed up yet, but no push scheduled, + // go ahead and schedule an immediate push. This can happen if `scheduleUpdate` + // is called frequently with minimizeDelay set to true, particularly when + // _performUpdate is taking a bit longer than normal to complete. + this._schedulePush(0); + } + }); } /** @@ -84,9 +102,40 @@ export class HostedMetadataManager { */ private async _performUpdate(): Promise { // Await the database if it is not yet connected. - const docUpdates = this._updatedAt; - this._updatedAt = {}; + const docUpdates = this._metadata; + this._metadata = {}; this._lastPushTime = Date.now(); - await this.setDocsUpdatedAt(docUpdates); + await this.setDocsMetadata(docUpdates); + } + + /** + * Schedule a metadata push. + * + * If `delayMs` is specified, the push will be scheduled to occur at least that + * number of milliseconds in the future. If `delayMs` is unspecified, the push + * will be scheduled to occur at least `_minPushDelayMs` after the last push time. + * + * If called while a push is already scheduled, that push will be cancelled and + * replaced with this one. + */ + private _schedulePush(delayMs?: number): void { + if (delayMs === undefined) { + delayMs = Math.round(this._minPushDelayMs - (Date.now() - this._lastPushTime)); + } + if (this._timeout) { clearTimeout(this._timeout); } + this._timeout = setTimeout(() => this._update(), delayMs < 0 ? 0 : delayMs); + } + + /** + * Adds `docId` and its `metadata` to the list of queued updates, merging any existing values. + */ + private _setOrUpdateMetadata(docId: string, metadata: DocumentMetadata): void { + if (!this._metadata[docId]) { + this._metadata[docId] = metadata; + } else { + const {updatedAt, usage} = metadata; + if (updatedAt) { this._metadata[docId].updatedAt = updatedAt; } + if (usage !== undefined) { this._metadata[docId].usage = usage; } + } } } diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts index 7649dba3..e129f578 100644 --- a/app/server/lib/HostedStorageManager.ts +++ b/app/server/lib/HostedStorageManager.ts @@ -3,6 +3,7 @@ import {mapGetOrSet} from 'app/common/AsyncCreate'; import {delay} from 'app/common/delay'; import {DocEntry} from 'app/common/DocListAPI'; import {DocSnapshots} from 'app/common/DocSnapshot'; +import {DocumentUsage} from 'app/common/DocUsage'; import {buildUrlId, parseUrlId} from 'app/common/gristUrls'; import {KeyedOps} from 'app/common/KeyedOps'; import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; @@ -323,6 +324,8 @@ export class HostedStorageManager implements IDocStorageManager { // NOTE: fse.remove succeeds also when the file does not exist. await fse.remove(this._getHashFile(this.getPath(docId))); this.markAsChanged(docId, 'edit'); + // Invalidate usage; it'll get re-computed the next time the document is opened. + this.scheduleUsageUpdate(docId, null, true); } catch (err) { this._log.error(docId, "problem replacing doc: %s", err); await fse.move(tmpPath, docPath, {overwrite: true}); @@ -483,6 +486,27 @@ export class HostedStorageManager implements IDocStorageManager { } } + /** + * Schedule an update to a document's usage column. + * + * If `minimizeDelay` is true, HostedMetadataManager will attempt to + * minimize delays by scheduling the update to occur as soon as possible. + */ + public scheduleUsageUpdate( + docName: string, + docUsage: DocumentUsage|null, + minimizeDelay = false + ): void { + const {forkId, snapshotId} = parseUrlId(docName); + if (!this._metadataManager || forkId || snapshotId) { return; } + + this._metadataManager.scheduleUpdate( + docName, + {usage: docUsage}, + minimizeDelay + ); + } + /** * Check if there is a pending change to be pushed to S3. */ @@ -524,9 +548,9 @@ export class HostedStorageManager implements IDocStorageManager { * This is called when a document was edited by the user. */ private _markAsEdited(docName: string, timestamp: string): void { - if (parseUrlId(docName).snapshotId) { return; } + if (parseUrlId(docName).snapshotId || !this._metadataManager) { return; } // Schedule a metadata update for the modified doc. - if (this._metadataManager) { this._metadataManager.scheduleUpdate(docName, timestamp); } + this._metadataManager.scheduleUpdate(docName, {updatedAt: timestamp}); } /** diff --git a/app/server/lib/IDocStorageManager.ts b/app/server/lib/IDocStorageManager.ts index 8d7f44eb..9b03e272 100644 --- a/app/server/lib/IDocStorageManager.ts +++ b/app/server/lib/IDocStorageManager.ts @@ -1,5 +1,6 @@ import {DocEntry} from 'app/common/DocListAPI'; import {DocSnapshots} from 'app/common/DocSnapshot'; +import {DocumentUsage} from 'app/common/DocUsage'; import {DocReplacementOptions} from 'app/common/UserAPI'; export interface IDocStorageManager { @@ -24,6 +25,7 @@ export interface IDocStorageManager { // Mark document as needing a backup (due to edits, migrations, etc). // If reason is set to 'edit' the user-facing timestamp on the document should be updated. markAsChanged(docName: string, reason?: 'edit'): void; + scheduleUsageUpdate(docName: string, usage: DocumentUsage|null, minimizeDelay?: boolean): void; testReopenStorage(): void; // restart storage during tests addToStorage(docName: string): Promise; // add a new local document to storage prepareToCloseStorage(): void; // speed up sync with remote store diff --git a/app/server/lib/Sharing.ts b/app/server/lib/Sharing.ts index a9cfa47c..5ddd2370 100644 --- a/app/server/lib/Sharing.ts +++ b/app/server/lib/Sharing.ts @@ -314,7 +314,7 @@ export class Sharing { }); actionGroup.actionSummary = actionSummary; await accessControl.appliedBundle(); - await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.docUsage); + await accessControl.sendDocUpdateForBundle(actionGroup, this._activeDoc.getDocUsageSummary()); if (docSession) { docSession.linkId = docSession.shouldBundleActions ? localActionBundle.actionNum : 0; } diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index 156d4cc1..7ca50d93 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -21,7 +21,7 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? const INTERNAL_FIELDS = new Set([ 'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', - 'authSubject', + 'authSubject', 'usage' ]); /**