(core) Add telemetry

Test Plan: Server tests.

Reviewers: jarek

Differential Revision: https://phab.getgrist.com/D3818

app/gen-server/entity/Document.ts

@@ -2,7 +2,8 @@ import {ApiError} from 'app/common/ApiError';
 import {DocumentUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {DocumentOptions, DocumentProperties, documentPropertyKeys,
-        DocumentType, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+        DocumentType, NEW_DOCUMENT_CODE, TutorialMetadata} from "app/common/UserAPI";
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {nativeValues} from 'app/gen-server/lib/values';
 import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
 import {AclRuleDoc} from "./AclRule";
@@ -90,7 +91,7 @@ export class Document extends Resource {
     return super.checkProperties(props, documentPropertyKeys);
   }
 
-  public updateFromProperties(props: Partial<DocumentProperties>) {
+  public updateFromProperties(props: Partial<DocumentProperties>, dbManager?: HomeDBManager) {
     super.updateFromProperties(props);
     if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
     if (props.urlId !== undefined) {
@@ -131,6 +132,12 @@ export class Document extends Resource {
             if (props.options.tutorial.lastSlideIndex !== undefined) {
               this.options.tutorial.lastSlideIndex = props.options.tutorial.lastSlideIndex;
             }
+            if (props.options.tutorial.numSlides !== undefined) {
+              this.options.tutorial.numSlides = props.options.tutorial.numSlides;
+              if (dbManager && props.options?.tutorial?.lastSlideIndex !== undefined) {
+                this._emitTutorialProgressChangeEvent(dbManager, this.options.tutorial);
+              }
+            }
           }
         }
         // Normalize so that null equates with absence.
@@ -146,6 +153,24 @@ export class Document extends Resource {
       }
     }
   }
+
+  private _emitTutorialProgressChangeEvent(
+    dbManager: HomeDBManager,
+    tutorialMetadata: TutorialMetadata
+  ) {
+    const lastSlideIndex = tutorialMetadata.lastSlideIndex;
+    const numSlides = tutorialMetadata.numSlides;
+    const percentComplete = lastSlideIndex !== undefined && numSlides !== undefined
+      ? Math.floor((lastSlideIndex / numSlides) * 100)
+      : undefined;
+    dbManager?.emit('tutorialProgressChange', {
+      tutorialForkId: this.id,
+      tutorialTrunkId: this.trunkId,
+      lastSlideIndex,
+      numSlides,
+      percentComplete,
+    });
+  }
 }
 
 // Check that icon points to an expected location.  This will definitely

app/gen-server/lib/HomeDBManager.ts

@@ -89,6 +89,14 @@ export const NotifierEvents = StringUnion(
 
 export type NotifierEvent = typeof NotifierEvents.type;
 
+export const TelemetryEvents = StringUnion(
+  'tutorialProgressChange',
+);
+
+export type TelemetryEvent = typeof TelemetryEvents.type;
+
+export type Event = NotifierEvent | TelemetryEvent;
+
 // Nominal email address of a user who can view anything (for thumbnails).
 export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com';
 
@@ -276,6 +284,7 @@ export class HomeDBManager extends EventEmitter {
   // In restricted mode, documents should be read-only.
   private _restrictedMode: boolean = false;
 
+
   /**
    * Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers',
    * 'guests', and 'members') are created by default on every new entity (Organization,
@@ -315,7 +324,7 @@ export class HomeDBManager extends EventEmitter {
     orgOnly: true
   }];
 
-  public emit(event: NotifierEvent, ...args: any[]): boolean {
+  public emit(event: Event, ...args: any[]): boolean {
     return super.emit(event, ...args);
   }
 
@@ -1944,7 +1953,7 @@ export class HomeDBManager extends EventEmitter {
       // Update the name and save.
       const doc: Document = queryResult.data;
       doc.checkProperties(props);
-      doc.updateFromProperties(props);
+      doc.updateFromProperties(props, this);
       if (forkId) {
         await manager.save(doc);
         return {status: 200};

app/gen-server/lib/Housekeeper.ts

@@ -1,6 +1,7 @@
 import { ApiError } from 'app/common/ApiError';
 import { buildUrlId } from 'app/common/gristUrls';
 import { Document } from 'app/gen-server/entity/Document';
+import { Organization } from 'app/gen-server/entity/Organization';
 import { Workspace } from 'app/gen-server/entity/Workspace';
 import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
 import { fromNow } from 'app/gen-server/sqlUtils';
@@ -14,6 +15,7 @@ import { optStringParam, stringParam } from 'app/server/lib/requestUtils';
 import * as express from 'express';
 import fetch from 'node-fetch';
 import * as Fetch from 'node-fetch';
+import { EntityManager } from 'typeorm';
 
 const HOUSEKEEPER_PERIOD_MS = 1 * 60 * 60 * 1000;   // operate every 1 hour
 const AGE_THRESHOLD_OFFSET = '-30 days';            // should be an interval known by postgres + sqlite
@@ -23,6 +25,7 @@ const AGE_THRESHOLD_OFFSET = '-30 days';            // should be an interval kno
  *
  *  - deleting old soft-deleted documents
  *  - deleting old soft-deleted workspaces
+ *  - logging metrics
  *
  * Call start(), keep the object around, and call stop() when shutting down.
  *
@@ -42,7 +45,9 @@ export class Housekeeper {
    */
   public async start() {
     await this.stop();
-    this._interval = setInterval(() => this.deleteTrashExclusively().catch(log.warn.bind(log)), HOUSEKEEPER_PERIOD_MS);
+    this._interval = setInterval(() => {
+      this.doHousekeepingExclusively().catch(log.warn.bind(log));
+    }, HOUSEKEEPER_PERIOD_MS);
   }
 
   /**
@@ -56,16 +61,18 @@ export class Housekeeper {
   }
 
   /**
-   * Deletes old trash if no other server is working on it or worked on it recently.
+   * Deletes old trash and logs metrics if no other server is working on it or worked on it
+   * recently.
    */
-  public async deleteTrashExclusively(): Promise<boolean> {
-    const electionKey = await this._electionStore.getElection('trash', HOUSEKEEPER_PERIOD_MS / 2.0);
+  public async doHousekeepingExclusively(): Promise<boolean> {
+    const electionKey = await this._electionStore.getElection('housekeeping', HOUSEKEEPER_PERIOD_MS / 2.0);
     if (!electionKey) {
-      log.info('Skipping deleteTrash since another server is working on it or worked on it recently');
+      log.info('Skipping housekeeping since another server is working on it or worked on it recently');
       return false;
     }
     this._electionKey = electionKey;
     await this.deleteTrash();
+    await this.logMetrics();
     return true;
   }
 
@@ -145,6 +152,39 @@ export class Housekeeper {
     }
   }
 
+  /**
+   * Logs metrics regardless of what other servers may be doing.
+   */
+  public async logMetrics() {
+    await this._dbManager.connection.transaction('READ UNCOMMITTED', async (manager) => {
+      const telemetryManager = this._server.getTelemetryManager();
+      const usageSummaries = await this._getOrgUsageSummaries(manager);
+      for (const summary of usageSummaries) {
+        telemetryManager?.logEvent('siteUsage', {
+          siteId: summary.site_id,
+          siteType: summary.site_type,
+          inGoodStanding: summary.in_good_standing,
+          stripePlanId: summary.stripe_plan_id,
+          numDocs: summary.num_docs,
+          numWorkspaces: summary.num_workspaces,
+          numMembers: summary.num_members,
+          lastActivity: summary.last_activity,
+        });
+      }
+
+      const membershipSummaries = await this._getOrgMembershipSummaries(manager);
+      for (const summary of membershipSummaries) {
+        telemetryManager?.logEvent('siteMembership', {
+          siteId: summary.site_id,
+          siteType: summary.site_type,
+          numOwners: summary.num_owners,
+          numEditors: summary.num_editors,
+          numViewers: summary.num_viewers,
+        });
+      }
+    });
+  }
+
   public addEndpoints(app: express.Application) {
     // Allow support user to perform housekeeping tasks for a specific
     // document.  The tasks necessarily bypass user access controls.
@@ -208,7 +248,7 @@ export class Housekeeper {
    */
   public async testClearExclusivity(): Promise<void> {
     if (this._electionKey) {
-      await this._electionStore.removeElection('trash', this._electionKey);
+      await this._electionStore.removeElection('housekeeping', this._electionKey);
       this._electionKey = undefined;
     }
   }
@@ -249,6 +289,52 @@ export class Housekeeper {
     return forks;
   }
 
+  private async _getOrgUsageSummaries(manager: EntityManager) {
+    const orgs = await manager.createQueryBuilder()
+      .select('orgs.id', 'site_id')
+      .addSelect('products.name', 'site_type')
+      .addSelect('billing_accounts.in_good_standing', 'in_good_standing')
+      .addSelect('billing_accounts.stripe_plan_id', 'stripe_plan_id')
+      .addSelect('COUNT(DISTINCT docs.id)', 'num_docs')
+      .addSelect('COUNT(DISTINCT workspaces.id)', 'num_workspaces')
+      .addSelect('COUNT(DISTINCT org_member_users.id)', 'num_members')
+      .addSelect('MAX(docs.updated_at)', 'last_activity')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.workspaces', 'workspaces')
+      .leftJoin('workspaces.docs', 'docs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .leftJoin('billing_accounts.product', 'products')
+      .leftJoin('orgs.aclRules', 'acl_rules')
+      .leftJoin('acl_rules.group', 'org_groups')
+      .leftJoin('org_groups.memberUsers', 'org_member_users')
+      .where('org_member_users.id IS NOT NULL')
+      .groupBy('orgs.id')
+      .addGroupBy('products.id')
+      .addGroupBy('billing_accounts.id')
+      .getRawMany();
+    return orgs;
+  }
+
+  private async _getOrgMembershipSummaries(manager: EntityManager) {
+    const orgs = await manager.createQueryBuilder()
+      .select('orgs.id', 'site_id')
+      .addSelect('products.name', 'site_type')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'owners' THEN 1 ELSE 0 END)", 'num_owners')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'editors' THEN 1 ELSE 0 END)", 'num_editors')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'viewers' THEN 1 ELSE 0 END)", 'num_viewers')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .leftJoin('billing_accounts.product', 'products')
+      .leftJoin('orgs.aclRules', 'acl_rules')
+      .leftJoin('acl_rules.group', 'org_groups')
+      .leftJoin('org_groups.memberUsers', 'org_member_users')
+      .where('org_member_users.id IS NOT NULL')
+      .groupBy('orgs.id')
+      .addGroupBy('products.id')
+      .getRawMany();
+    return orgs;
+  }
+
   /**
    * TypeORM isn't very adept at handling date representation for
    * comparisons, so we construct the threshold date in SQL so that we