diff --git a/app/client/components/GristDoc.ts b/app/client/components/GristDoc.ts index 6dc643ce..7840686b 100644 --- a/app/client/components/GristDoc.ts +++ b/app/client/components/GristDoc.ts @@ -157,7 +157,7 @@ export class GristDoc extends DisposableWithEvents { } = {} ) { super(); - console.log("RECEIVED DOC RESPONSE", openDocResponse.doc); + console.log("RECEIVED DOC RESPONSE", openDocResponse); this.docData = new DocData(this.docComm, openDocResponse.doc); this.docModel = new DocModel(this.docData); this.querySetManager = QuerySetManager.create(this, this.docModel, this.docComm); diff --git a/app/client/models/DocPageModel.ts b/app/client/models/DocPageModel.ts index 9a7ae63a..bb397c74 100644 --- a/app/client/models/DocPageModel.ts +++ b/app/client/models/DocPageModel.ts @@ -14,6 +14,7 @@ import {bigBasicButton} from 'app/client/ui2018/buttons'; import {testId} from 'app/client/ui2018/cssVars'; import {menu, menuDivider, menuIcon, menuItem, menuText} from 'app/client/ui2018/menus'; import {confirmModal} from 'app/client/ui2018/modals'; +import {DataLimitStatus} from 'app/common/ActiveDocAPI'; import {AsyncFlow, CancelledError, FlowRunner} from 'app/common/AsyncFlow'; import {delay} from 'app/common/delay'; import {OpenDocMode, UserOverride} from 'app/common/DocListAPI'; @@ -66,6 +67,7 @@ export interface DocPageModel { gristDoc: Observable; // Instance of GristDoc once it exists. rowCount: Observable; + dataLimitStatus: Observable; createLeftPane(leftPanelOpen: Observable): DomArg; renameDoc(value: string): Promise; @@ -108,6 +110,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { public readonly gristDoc = Observable.create(this, null); public readonly rowCount = Observable.create(this, undefined); + public readonly dataLimitStatus = Observable.create(this, null); // Combination of arguments needed to open a doc (docOrUrlId + openMod). It's obtained from the // URL, and when it changes, we need to re-open. @@ -258,6 +261,7 @@ export class DocPageModelImpl extends Disposable implements DocPageModel { this.currentDoc.set({...doc}); } this.rowCount.set(openDocResponse.rowCount); + this.dataLimitStatus.set(openDocResponse.dataLimitStatus); const gdModule = await gristDocModulePromise; const docComm = gdModule.DocComm.create(flow, comm, openDocResponse, doc.id, this.appModel.notifier); flow.checkIfCancelled(); diff --git a/app/common/ActiveDocAPI.ts b/app/common/ActiveDocAPI.ts index b81b7928..672fa135 100644 --- a/app/common/ActiveDocAPI.ts +++ b/app/common/ActiveDocAPI.ts @@ -153,6 +153,8 @@ export interface PermissionDataWithExtraUsers extends PermissionData { exampleUsers: UserAccessData[]; } +export type DataLimitStatus = null | 'approachingLimit' | 'gracePeriod' | 'deleteOnly'; + export interface ActiveDocAPI { /** * Closes a document, and unsubscribes from its userAction events. diff --git a/app/common/DocListAPI.ts b/app/common/DocListAPI.ts index 91e85576..1cb40e7d 100644 --- a/app/common/DocListAPI.ts +++ b/app/common/DocListAPI.ts @@ -1,4 +1,5 @@ import {MinimalActionGroup} from 'app/common/ActionGroup'; +import {DataLimitStatus} from 'app/common/ActiveDocAPI'; import {TableDataAction} from 'app/common/DocActions'; import {Role} from 'app/common/roles'; import {StringUnion} from 'app/common/StringUnion'; @@ -45,6 +46,7 @@ export interface OpenLocalDocResult { recoveryMode?: boolean; userOverride?: UserOverride; rowCount?: number; + dataLimitStatus?: DataLimitStatus; } export interface UserOverride { diff --git a/app/common/Features.ts b/app/common/Features.ts index 21577386..5b8a1631 100644 --- a/app/common/Features.ts +++ b/app/common/Features.ts @@ -45,9 +45,9 @@ export interface Features { baseMaxRowsPerDocument?: number; // If set, establishes a default maximum on the // number of rows (total) in a single document. // Actual max for a document may be higher. - // TODO: not honored at time of writing. - // TODO: nuances about how rows are counted. baseMaxApiUnitsPerDocumentPerDay?: number; // Similar for api calls. + + gracePeriodDays?: number; // Duration of the grace period in days, before entering delete-only mode } // Check whether it is possible to add members at the org level. There's no flag diff --git a/app/gen-server/entity/Document.ts b/app/gen-server/entity/Document.ts index c0a1e0f3..a5b7289b 100644 --- a/app/gen-server/entity/Document.ts +++ b/app/gen-server/entity/Document.ts @@ -53,6 +53,9 @@ export class Document extends Resource { @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true}) public removedAt: Date|null; + @Column({name: 'grace_period_start', type: nativeValues.dateTimeType, nullable: true}) + public gracePeriodStart: Date|null; + @OneToMany(type => Alias, alias => alias.doc) public aliases: Alias[]; diff --git a/app/gen-server/entity/Product.ts b/app/gen-server/entity/Product.ts index 3c7b35f9..aa8838fc 100644 --- a/app/gen-server/entity/Product.ts +++ b/app/gen-server/entity/Product.ts @@ -37,7 +37,8 @@ export const teamFreeFeatures: Features = { maxDocsPerOrg: 20, snapshotWindow: { count: 1, unit: 'month' }, baseMaxRowsPerDocument: 5000, - baseMaxApiUnitsPerDocumentPerDay: 5000 + baseMaxApiUnitsPerDocumentPerDay: 5000, + gracePeriodDays: 14, }; /** diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts index 2cf13325..db681d6e 100644 --- a/app/gen-server/lib/HomeDBManager.ts +++ b/app/gen-server/lib/HomeDBManager.ts @@ -2362,6 +2362,14 @@ export class HomeDBManager extends EventEmitter { }); } + public async setDocGracePeriodStart(docId: string, gracePeriodStart: Date | null) { + return await this._connection.createQueryBuilder() + .update(Document) + .set({gracePeriodStart}) + .where({id: docId}) + .execute(); + } + public async getDocProduct(docId: string): Promise { return await this._connection.createQueryBuilder() .select('product') diff --git a/app/gen-server/migration/1647883793388-GracePeriodStart.ts b/app/gen-server/migration/1647883793388-GracePeriodStart.ts new file mode 100644 index 00000000..5fead934 --- /dev/null +++ b/app/gen-server/migration/1647883793388-GracePeriodStart.ts @@ -0,0 +1,18 @@ +import {nativeValues} from "app/gen-server/lib/values"; +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm"; + +export class GracePeriodStart1647883793388 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.addColumn("docs", new TableColumn({ + name: "grace_period_start", + type: nativeValues.dateTimeType, + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropColumn("docs", "grace_period_start"); + } + +} diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index ca5bbae4..f1b4b1c5 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -10,6 +10,7 @@ import {ActionSummary} from "app/common/ActionSummary"; import { ApplyUAOptions, ApplyUAResult, + DataLimitStatus, DataSourceTransformed, ForkResult, ImportOptions, @@ -34,6 +35,7 @@ import {DocData} from 'app/common/DocData'; import {DocSnapshots} from 'app/common/DocSnapshot'; import {DocumentSettings} from 'app/common/DocumentSettings'; import {normalizeEmail} from 'app/common/emails'; +import {Features} from 'app/common/Features'; import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause'; import {byteString, countIf, safeJsonParse} from 'app/common/gutil'; import {InactivityTimer} from 'app/common/InactivityTimer'; @@ -51,6 +53,7 @@ import {Authorizer} from 'app/server/lib/Authorizer'; import {checksumFile} from 'app/server/lib/checksumFile'; import {Client} from 'app/server/lib/Client'; import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager'; +import {ICreateActiveDocOptions} from 'app/server/lib/ICreate'; import {makeForkIds} from 'app/server/lib/idUtils'; import {GRIST_DOC_SQL, GRIST_DOC_WITH_TABLE1_SQL} from 'app/server/lib/initialDocSql'; import {ISandbox} from 'app/server/lib/ISandbox'; @@ -160,18 +163,21 @@ export class ActiveDoc extends EventEmitter { private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured. private _fetchCache = new MapWithTTL>(DEFAULT_CACHE_TTL); private _rowCount?: number; + private _productFeatures?: Features; + private _gracePeriodStart: Date|null = null; // Timer for shutting down the ActiveDoc a bit after all clients are gone. private _inactivityTimer = new InactivityTimer(() => this.shutdown(), Deps.ACTIVEDOC_TIMEOUT * 1000); private _recoveryMode: boolean = false; private _shuttingDown: boolean = false; - constructor(docManager: DocManager, docName: string, private _options?: { - safeMode?: boolean, - docUrl?: string - }) { + constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) { super(); if (_options?.safeMode) { this._recoveryMode = true; } + if (_options?.doc) { + this._productFeatures = _options.doc.workspace.org.billingAccount?.product.features; + this._gracePeriodStart = _options.doc.gracePeriodStart; + } this._docManager = docManager; this._docName = docName; this.docStorage = new DocStorage(docManager.storageManager, docName); @@ -219,6 +225,24 @@ export class ActiveDoc extends EventEmitter { } } + public async getDataLimitStatus(): Promise { + if (this._rowLimit && this._rowCount) { + const ratio = this._rowCount / this._rowLimit; + if (ratio > 1) { + const start = this._gracePeriodStart; + const days = this._productFeatures?.gracePeriodDays; + if (start && days && moment().diff(moment(start), 'days') >= days) { + return 'deleteOnly'; + } else { + return 'gracePeriod'; + } + } else if (ratio > 0.9) { + return 'approachingLimit'; + } + } + return null; + } + public async getUserOverride(docSession: OptDocSession) { return this._granularAccess.getUserOverride(docSession); } @@ -900,6 +924,17 @@ export class ActiveDoc extends EventEmitter { // Be careful not to sneak into user action queue before Calculate action, otherwise // there'll be a deadlock. await this.waitForInitialization(); + + if ( + await this.getDataLimitStatus() === "deleteOnly" && + !actions.every(action => [ + 'RemoveTable', 'RemoveColumn', 'RemoveRecord', 'BulkRemoveRecord', + 'RemoveViewSection', 'RemoveView', 'ApplyUndoActions', + ].includes(action[0] as string)) + ) { + throw new Error("Document is in delete-only mode"); + } + // Granular access control implemented in _applyUserActions. return await this._applyUserActions(docSession, actions, options); } @@ -1223,7 +1258,7 @@ export class ActiveDoc extends EventEmitter { ...this.getLogMeta(docSession), rowCount: sandboxActionBundle.rowCount }); - this._rowCount = sandboxActionBundle.rowCount; + await this._updateRowCount(sandboxActionBundle.rowCount); await this._reportDataEngineMemory(); } else { // Create default SandboxActionBundle to use if the data engine is not called. @@ -1451,6 +1486,25 @@ export class ActiveDoc extends EventEmitter { }; } + private get _rowLimit(): number | undefined { + return this._productFeatures?.baseMaxRowsPerDocument; + } + + private async _updateGracePeriodStart(gracePeriodStart: Date | null) { + this._gracePeriodStart = gracePeriodStart; + await this.getHomeDbManager()?.setDocGracePeriodStart(this.docName, gracePeriodStart); + } + + private async _updateRowCount(rowCount: number) { + this._rowCount = rowCount; + const exceedingRowLimit = this._rowLimit && rowCount > this._rowLimit; + if (exceedingRowLimit && !this._gracePeriodStart) { + await this._updateGracePeriodStart(new Date()); + } else if (!exceedingRowLimit && this._gracePeriodStart) { + await this._updateGracePeriodStart(null); + } + } + /** * Prepares a single attachment by adding it DocStorage and returns a UserAction to apply. */ diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index 21caa0fa..0279d007 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -313,11 +313,12 @@ export class DocManager extends EventEmitter { } } - const [metaTables, recentActions, userOverride, rowCount] = await Promise.all([ + const [metaTables, recentActions, userOverride, rowCount, dataLimitStatus] = await Promise.all([ activeDoc.fetchMetaTables(docSession), activeDoc.getRecentMinimalActions(docSession), activeDoc.getUserOverride(docSession), activeDoc.getRowCount(docSession), + activeDoc.getDataLimitStatus(), ]); const result = { @@ -328,6 +329,7 @@ export class DocManager extends EventEmitter { recoveryMode: activeDoc.recoveryMode, userOverride, rowCount, + dataLimitStatus, } as OpenLocalDocResult; if (!activeDoc.muted) { @@ -510,7 +512,7 @@ export class DocManager extends EventEmitter { const doc = await this._getDoc(docSession, docName); // Get URL for document for use with SELF_HYPERLINK(). const docUrl = doc && await this._getDocUrl(doc); - return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode}); + return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode, doc}); } /** diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index 514048fa..44f90ea9 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -1,3 +1,4 @@ +import {Document} from 'app/gen-server/entity/Document'; import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {DocManager} from 'app/server/lib/DocManager'; @@ -32,4 +33,5 @@ export interface ICreate { export interface ICreateActiveDocOptions { safeMode?: boolean; docUrl?: string; + doc?: Document; } diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index bae8aa88..144cb88a 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -18,9 +18,10 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ? parseInt(process.env.GRIST_TEST_HTTPS_OFFSET, 10) : undefined; // Database fields that we permit in entities but don't want to cross the api. -const INTERNAL_FIELDS = new Set(['apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', - 'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId', - 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin']); +const INTERNAL_FIELDS = new Set([ + 'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId', + 'stripeSubscriptionId', 'stripePlanId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin', +]); /** * Adapt a home-server or doc-worker URL to match the hostname in the request URL. For custom