(core) Add API endpoint to get site usage summary

Summary:
The summary includes a count of documents that are approaching
limits, in grace period, or delete-only. The endpoint is only accessible
to site owners, and is currently unused. A follow-up diff will add usage
banners to the site home page, which will use the response from the
endpoint to communicate usage information to owners.

Test Plan: Browser and server tests.

Reviewers: alexmojaki

Reviewed By: alexmojaki

Differential Revision: https://phab.getgrist.com/D3420

app/gen-server/ApiServer.ts

@@ -146,6 +146,15 @@ export class ApiServer {
       return sendReply(req, res, query);
     }));
 
+    // GET /api/orgs/:oid/usage
+    // Get usage summary of all un-deleted documents in the organization.
+    // Only accessible to org owners.
+    this._app.get('/api/orgs/:oid/usage', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const usage = await this._dbManager.getOrgUsageSummary(getScope(req), org);
+      return sendOkReply(req, res, usage);
+    }));
+
     // POST /api/orgs
     // Body params: name (required), domain
     // Create a new org.
@@ -194,7 +203,7 @@ export class ApiServer {
       return sendReply(req, res, query);
     }));
 
-    // // DELETE /api/workspaces/:wid
+    // DELETE /api/workspaces/:wid
     // Delete the specified workspace and all included docs.
     this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => {
       const wsId = integerParam(req.params.wid, 'wid');

app/gen-server/entity/Document.ts

@@ -1,4 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
+import {DocumentUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {DocumentOptions, DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
 import {nativeValues} from 'app/gen-server/lib/values';
@@ -65,6 +66,9 @@ export class Document extends Resource {
   @OneToMany(_type => Secret, secret => secret.doc)
   public secrets: Secret[];
 
+  @Column({name: 'usage', type: nativeValues.jsonEntityType, nullable: true})
+  public usage: DocumentUsage | null;
+
   public checkProperties(props: any): props is Partial<DocumentProperties> {
     return super.checkProperties(props, documentPropertyKeys);
   }

app/gen-server/lib/HomeDBManager.ts

@@ -1,5 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {mapGetOrSet, mapSetOrClear, MapWithTTL} from 'app/common/AsyncCreate';
+import {getDataLimitStatus} from 'app/common/DocLimits';
+import {createEmptyOrgUsageSummary, DocumentUsage, OrgUsageSummary} from 'app/common/DocUsage';
 import {normalizeEmail} from 'app/common/emails';
 import {canAddOrgMembers, Features} from 'app/common/Features';
 import {buildUrlId, MIN_URLID_PREFIX_LENGTH, parseUrlId} from 'app/common/gristUrls';
@@ -231,6 +233,12 @@ function stringifyUrlIdOrg(urlId: string, org?: string): string {
   return `${urlId}:${org}`;
 }
 
+export interface DocumentMetadata {
+  // ISO 8601 UTC date (e.g. the output of new Date().toISOString()).
+  updatedAt?: string;
+  usage?: DocumentUsage|null;
+}
+
 /**
  * HomeDBManager handles interaction between the ApiServer and the Home database,
  * encapsulating the typeorm logic.
@@ -922,6 +930,44 @@ export class HomeDBManager extends EventEmitter {
     return result;
   }
 
+  /**
+   * Returns an organization's usage summary (e.g. count of documents that are approaching or exceeding
+   * limits).
+   */
+  public async getOrgUsageSummary(scope: Scope, orgKey: string|number): Promise<OrgUsageSummary> {
+    // Check that an owner of the org is making the request.
+    const markPermissions = Permissions.OWNER;
+    let orgQuery = this.org(scope, orgKey, {
+      markPermissions,
+      needRealOrg: true
+    });
+    orgQuery = this._addFeatures(orgQuery);
+    const orgQueryResult = await verifyIsPermitted(orgQuery);
+    const org: Organization = this.unwrapQueryResult(orgQueryResult);
+    const productFeatures = org.billingAccount.product.features;
+
+    // Grab all the non-removed documents in the org.
+    let docsQuery = this._docs()
+      .innerJoin('docs.workspace', 'workspaces')
+      .innerJoin('workspaces.org', 'orgs')
+      .where('docs.workspace_id = workspaces.id')
+      .andWhere('workspaces.removed_at IS NULL AND docs.removed_at IS NULL');
+    docsQuery = this._whereOrg(docsQuery, orgKey);
+    if (this.isMergedOrg(orgKey)) {
+      docsQuery = docsQuery.andWhere('orgs.owner_id = :userId', {userId: scope.userId});
+    }
+    const docsQueryResult = await this._verifyAclPermissions(docsQuery, { scope, emptyAllowed: true });
+    const docs: Document[] = this.unwrapQueryResult(docsQueryResult);
+
+    // Return an aggregate count of documents, grouped by data limit status.
+    const summary = createEmptyOrgUsageSummary();
+    for (const {usage: docUsage, gracePeriodStart} of docs) {
+      const dataLimitStatus = getDataLimitStatus({docUsage, gracePeriodStart, productFeatures});
+      if (dataLimitStatus) { summary[dataLimitStatus] += 1; }
+    }
+    return summary;
+  }
+
   /**
    * Compute the best access option for an organization, from the
    * users available to the client.  If none of the options can access
@@ -2364,12 +2410,13 @@ export class HomeDBManager extends EventEmitter {
   }
 
   /**
-   * Updates the updatedAt values for several docs. Takes a map where each entry maps a docId to
-   * an ISO date string representing the new updatedAt time. This is not a part of the API, it
-   * should be called only by the HostedMetadataManager when a change is made to a doc.
+   * Updates the updatedAt and usage values for several docs. Takes a map where each entry maps
+   * a docId to a metadata object containing the updatedAt and/or usage values. This is not a part
+   * of the API, it should be called only by the HostedMetadataManager when a change is made to a
+   * doc.
    */
-  public async setDocsUpdatedAt(
-    docUpdateMap: {[docId: string]: string}
+  public async setDocsMetadata(
+    docUpdateMap: {[docId: string]: DocumentMetadata}
   ): Promise<QueryResult<void>> {
     if (!docUpdateMap || Object.keys(docUpdateMap).length === 0) {
       return {
@@ -2382,7 +2429,7 @@ export class HomeDBManager extends EventEmitter {
       const updateTasks = docIds.map(docId => {
         return manager.createQueryBuilder()
           .update(Document)
-          .set({updatedAt: docUpdateMap[docId]})
+          .set(docUpdateMap[docId])
           .where("id = :docId", {docId})
           .execute();
       });

app/gen-server/migration/1651469582887-DocumentUsage.ts

@@ -0,0 +1,18 @@
+import {nativeValues} from "app/gen-server/lib/values";
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class DocumentUsage1651469582887 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn("docs", new TableColumn({
+      name: "usage",
+      type: nativeValues.jsonType,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn("docs", "usage");
+  }
+
+}