(core) Add optional telemetry to grist-core

Summary: Adds support for optional telemetry to grist-core. A new environment variable, GRIST_TELEMETRY_LEVEL, controls the level of telemetry collected. Test Plan: Server and unit tests. Reviewers: paulfitz Reviewed By: paulfitz Subscribers: dsagal, anaisconce Differential Revision: https://phab.getgrist.com/D3880
2026-03-02 04:09:24 +00:00 · 2023-06-06 13:08:50 -04:00
parent 0d082c9cfc
commit 10f5f0cb37
38 changed files with 2177 additions and 201 deletions
--- a/app/server/generateInitialDocSql.ts
+++ b/app/server/generateInitialDocSql.ts
@@ -3,6 +3,7 @@ import { create } from 'app/server/lib/create';
 import { DocManager } from 'app/server/lib/DocManager';
 import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
 import { DocStorageManager } from 'app/server/lib/DocStorageManager';
+import { createDummyTelemetry } from 'app/server/lib/GristServer';
 import { PluginManager } from 'app/server/lib/PluginManager';

 import * as childProcess from 'child_process';
@@ -33,7 +34,7 @@ export async function main(baseName: string) {
    }
    const docManager = new DocManager(storageManager, pluginManager, null as any, {
      create,
-      getTelemetryManager: () => undefined,
+      getTelemetry() { return createDummyTelemetry(); },
    } as any);
    const activeDoc = new ActiveDoc(docManager, baseName);
    const session = makeExceptionalDocSession('nascent');
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -74,7 +74,7 @@ import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord, SingleCell} from 'app/common/TableData';
-import {TelemetryEventName} from 'app/common/Telemetry';
+import {TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
 import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
@@ -1395,11 +1395,13 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
      // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
      const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
      this.logTelemetryEvent(docSession, 'documentForked', {
-        forkIdDigest: hashId(forkIds.forkId),
-        forkDocIdDigest: hashId(forkIds.docId),
-        trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
-        isTemplate,
-        lastActivity: doc.updatedAt,
+        limited: {
+          forkIdDigest: hashId(forkIds.forkId),
+          forkDocIdDigest: hashId(forkIds.docId),
+          trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
+          isTemplate,
+          lastActivity: doc.updatedAt,
+        },
      });
    } finally {
      await permitStore.removePermit(permitKey);
@@ -1789,13 +1791,14 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {

  public logTelemetryEvent(
    docSession: OptDocSession | null,
-    eventName: TelemetryEventName,
-    metadata?: Record<string, any>
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadataByLevel
  ) {
-    this._docManager.gristServer.getTelemetryManager()?.logEvent(eventName, {
-      ...this._getTelemetryMeta(docSession),
-      ...metadata,
-    });
+    this._docManager.gristServer.getTelemetry().logEvent(event, merge(
+      this._getTelemetryMeta(docSession),
+      metadata,
+    ))
+    .catch(e => this._log.error(docSession, `failed to log telemetry event ${event}`, e));
  }

  /**
@@ -2332,18 +2335,20 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {

  private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
    this.logTelemetryEvent(docSession, 'documentUsage', {
-      triggeredBy,
-      isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
-      rowCount: this._docUsage?.rowCount?.total,
-      dataSizeBytes: this._docUsage?.dataSizeBytes,
-      attachmentsSize: this._docUsage?.attachmentsSizeBytes,
-      ...this._getAccessRuleMetrics(),
-      ...this._getAttachmentMetrics(),
-      ...this._getChartMetrics(),
-      ...this._getWidgetMetrics(),
-      ...this._getColumnMetrics(),
-      ...this._getTableMetrics(),
-      ...this._getCustomWidgetMetrics(),
+      limited: {
+        triggeredBy,
+        isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
+        rowCount: this._docUsage?.rowCount?.total,
+        dataSizeBytes: this._docUsage?.dataSizeBytes,
+        attachmentsSize: this._docUsage?.attachmentsSizeBytes,
+        ...this._getAccessRuleMetrics(),
+        ...this._getAttachmentMetrics(),
+        ...this._getChartMetrics(),
+        ...this._getWidgetMetrics(),
+        ...this._getColumnMetrics(),
+        ...this._getTableMetrics(),
+        ...this._getCustomWidgetMetrics(),
+      },
    });
  }

@@ -2365,10 +2370,10 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
      // Exclude the leading ".", if any.
      .map(r => r.fileExt?.trim()?.slice(1))
      .filter(ext => Boolean(ext));
-
+    const uniqueAttachmentTypes = [...new Set(attachmentTypes ?? [])];
    return {
      numAttachments,
-      attachmentTypes,
+      attachmentTypes: uniqueAttachmentTypes,
    };
  }

@@ -2528,15 +2533,19 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
    this._logDocMetrics(docSession, 'docOpen');
  }

-  private _getTelemetryMeta(docSession: OptDocSession|null) {
+  private _getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel {
    const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined;
    return merge(
      docSession ? getTelemetryMetaFromDocSession(docSession) : {},
-      altSessionId ? {altSessionId} : undefined,
+      altSessionId ? {altSessionId} : {},
      {
-        docIdDigest: hashId(this._docName),
-        siteId: this._doc?.workspace.org.id,
-        siteType: this._product?.name,
+        limited: {
+          docIdDigest: hashId(this._docName),
+        },
+        full: {
+          siteId: this._doc?.workspace.org.id,
+          siteType: this._product?.name,
+        },
      },
    );
  }
--- a/app/server/lib/AppEndpoint.ts
+++ b/app/server/lib/AppEndpoint.ts
@@ -11,7 +11,7 @@ import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gr
 import {removeTrailingSlash} from 'app/common/gutil';
 import {hashId} from 'app/common/hashingUtils';
 import {LocalPlugin} from "app/common/plugin";
-import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
+import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
 import {Document as APIDocument} from 'app/common/UserAPI';
 import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
@@ -308,18 +308,23 @@ export function attachAppEndpoint(options: AttachOptions): void {
    // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
    const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
    if (isPublic || isTemplate) {
-      gristServer.getTelemetryManager()?.logEvent('documentOpened', {
-        docIdDigest: hashId(docId),
-        siteId: doc.workspace.org.id,
-        siteType: doc.workspace.org.billingAccount.product.name,
-        userId: mreq.userId,
-        altSessionId: mreq.altSessionId,
-        access: doc.access,
-        isPublic,
-        isSnapshot,
-        isTemplate,
-        lastUpdated: doc.updatedAt,
-      });
+      gristServer.getTelemetry().logEvent('documentOpened', {
+        limited: {
+          docIdDigest: hashId(docId),
+          access: doc.access,
+          isPublic,
+          isSnapshot,
+          isTemplate,
+          lastUpdated: doc.updatedAt,
+        },
+        full: {
+          siteId: doc.workspace.org.id,
+          siteType: doc.workspace.org.billingAccount.product.name,
+          userId: mreq.userId,
+          altSessionId: mreq.altSessionId,
+        },
+      })
+      .catch(e => log.error('failed to log telemetry event documentOpened', e));
    }

    if (isTemplate) {
@@ -330,7 +335,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
        isAnonymous: isAnonymousUser(mreq),
        templateId: docId,
      };
-      res.cookie(TelemetryTemplateSignupCookieName, JSON.stringify(value), {
+      res.cookie(TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME, JSON.stringify(value), {
        maxAge: 1000 * 60 * 60,
        httpOnly: true,
        path: '/',
--- a/app/server/lib/Authorizer.ts
+++ b/app/server/lib/Authorizer.ts
@@ -396,11 +396,14 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
  };
  log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
  if (hasApiKey) {
-    options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
-      method: mreq.method,
-      userId: mreq.userId,
-      userAgent: req.headers['user-agent'],
-    });
+    options.gristServer.getTelemetry().logEvent('apiUsage', {
+      full: {
+        method: mreq.method,
+        userId: mreq.userId,
+        userAgent: mreq.headers['user-agent'],
+      },
+    })
+    .catch(e => log.error('failed to log telemetry event apiUsage', e));
  }

  return next();
--- a/app/server/lib/Client.ts
+++ b/app/server/lib/Client.ts
@@ -4,6 +4,7 @@ import {delay} from 'app/common/delay';
 import {CommClientConnect, CommMessage, CommResponse, CommResponseError} from 'app/common/CommTypes';
 import {ErrorWithCode} from 'app/common/ErrorWithCode';
 import {UserProfile} from 'app/common/LoginSessionAPI';
+import {TelemetryMetadata} from 'app/common/Telemetry';
 import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI';
 import {User} from 'app/gen-server/entity/User';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
@@ -433,8 +434,8 @@ export class Client {
    return meta;
  }

-  public getFullTelemetryMeta() {
-    const meta: Record<string, any> = {};
+  public getFullTelemetryMeta(): TelemetryMetadata {
+    const meta: TelemetryMetadata = {};
    // We assume the _userId has already been cached, which will be true always (for all practical
    // purposes) because it's set when the Authorizer checks this client.
    if (this._userId) { meta.userId = this._userId; }
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -916,8 +916,10 @@ export class DocWorkerApi {
          });
          const {forkId} = parseUrlId(scope.urlId);
          activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
-            tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
-            tutorialTrunkIdDigest: hashId(tutorialTrunkId),
+            full: {
+              tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
+              tutorialTrunkIdDigest: hashId(tutorialTrunkId),
+            },
          });
        }
      }
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -1,12 +1,12 @@
 import {ApiError} from 'app/common/ApiError';
 import {delay} from 'app/common/delay';
 import {DocCreationInfo} from 'app/common/DocListAPI';
-import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPathOnly,
-        parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
+import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
+        GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
+        sanitizePathTail} from 'app/common/gristUrls';
 import {getOrgUrlInfo} from 'app/common/gristUrls';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {tbind} from 'app/common/tbind';
-import {TelemetryEventName, TelemetryEventNames} from 'app/common/Telemetry';
 import * as version from 'app/common/version';
 import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
@@ -57,7 +57,7 @@ import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
-import {TelemetryManager} from 'app/server/lib/TelemetryManager';
+import {ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
@@ -117,7 +117,9 @@ export class FlexServer implements GristServer {
  public electronServerMethods: ElectronServerMethods;
  public readonly docsRoot: string;
  public readonly i18Instance: i18n;
+  private _activations: Activations;
  private _comm: Comm;
+  private _deploymentType: GristDeploymentType;
  private _dbManager: HomeDBManager;
  private _defaultBaseDomain: string|undefined;
  private _pluginUrl: string|undefined;
@@ -130,7 +132,7 @@ export class FlexServer implements GristServer {
  private _sessions: Sessions;
  private _sessionStore: SessionStore;
  private _storageManager: IDocStorageManager;
-  private _telemetryManager: TelemetryManager|undefined;
+  private _telemetry: ITelemetry;
  private _processMonitorStop?: () => void;    // Callback to stop the ProcessMonitor
  private _docWorkerMap: IDocWorkerMap;
  private _widgetRepository: IWidgetRepository;
@@ -199,6 +201,11 @@ export class FlexServer implements GristServer {
    this.docsRoot = fse.realpathSync(docsRoot);
    this.info.push(['docsRoot', this.docsRoot]);

+    this._deploymentType = this.create.deploymentType();
+    if (process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE) {
+      this._deploymentType = GristDeploymentTypes.check(process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE);
+    }
+
    const homeUrl = process.env.APP_HOME_URL;
    // The "base domain" is only a thing if orgs are encoded as a subdomain.
    if (process.env.GRIST_ORG_IN_PATH === 'true' || process.env.GRIST_SINGLE_ORG) {
@@ -328,11 +335,20 @@ export class FlexServer implements GristServer {
    return this._comm;
  }

+  public getDeploymentType(): GristDeploymentType {
+    return this._deploymentType;
+  }
+
  public getHosts(): Hosts {
    if (!this._hosts) { throw new Error('no hosts available'); }
    return this._hosts;
  }

+  public getActivations(): Activations {
+    if (!this._activations) { throw new Error('no activations available'); }
+    return this._activations;
+  }
+
  public getHomeDBManager(): HomeDBManager {
    if (!this._dbManager) { throw new Error('no home db available'); }
    return this._dbManager;
@@ -343,8 +359,9 @@ export class FlexServer implements GristServer {
    return this._storageManager;
  }

-  public getTelemetryManager(): TelemetryManager|undefined {
-    return this._telemetryManager;
+  public getTelemetry(): ITelemetry {
+    if (!this._telemetry) { throw new Error('no telemetry available'); }
+    return this._telemetry;
  }

  public getWidgetRepository(): IWidgetRepository {
@@ -553,8 +570,8 @@ export class FlexServer implements GristServer {
    // Report which database we are using, without sensitive credentials.
    this.info.push(['database', getDatabaseUrl(this._dbManager.connection.options, false)]);
    // If the installation appears to be new, give it an id and a creation date.
-    const activations = new Activations(this._dbManager);
-    await activations.current();
+    this._activations = new Activations(this._dbManager);
+    await this._activations.current();
  }

  public addDocWorkerMap() {
@@ -689,24 +706,14 @@ export class FlexServer implements GristServer {
    });
  }

-  public addTelemetryEndpoint() {
-    if (this._check('telemetry-endpoint', 'json', 'api-mw', 'homedb')) { return; }
+  public addTelemetry() {
+    if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }

-    this._telemetryManager = new TelemetryManager(this._dbManager);
+    this._telemetry = this.create.Telemetry(this._dbManager, this);
+    this._telemetry.addEndpoints(this.app);

    // Start up a monitor for memory and cpu usage.
-    this._processMonitorStop = ProcessMonitor.start(this._telemetryManager);
-
-    this.app.post('/api/telemetry', async (req, resp) => {
-      const mreq = req as RequestWithLogin;
-      const name = stringParam(req.body.name, 'name', TelemetryEventNames);
-      this._telemetryManager?.logEvent(name as TelemetryEventName, {
-        userId: mreq.userId,
-        altSessionId: mreq.altSessionId,
-        ...req.body.metadata,
-      });
-      return resp.status(200).send();
-    });
+    this._processMonitorStop = ProcessMonitor.start(this._telemetry);
  }

  public async close() {
@@ -828,7 +835,7 @@ export class FlexServer implements GristServer {

    // Initialize _sendAppPage helper.
    this._sendAppPage = makeSendAppPage({
-      server: isSingleUserMode() ? null : this,
+      server: this,
      staticDir: getAppPathTo(this.appRoot, 'static'),
      tag: this.tag,
      testLogin: allowTestLogin(),
@@ -1108,7 +1115,7 @@ export class FlexServer implements GristServer {
  // Add document-related endpoints and related support.
  public async addDoc() {
    this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ?
-      null : 'homedb', 'api-mw', 'map', 'telemetry-endpoint');
+      null : 'homedb', 'api-mw', 'map', 'telemetry');
    // add handlers for cleanup, if we are in charge of the doc manager.
    if (!this._docManager) { this.addCleanup(); }
    await this.loadConfig();
@@ -1368,7 +1375,11 @@ export class FlexServer implements GristServer {
  }

  public getGristConfig(): GristLoadConfig {
-    return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
+    return makeGristConfig({
+      homeUrl: this.getDefaultHomeUrl(),
+      extra: {},
+      baseDomain: this._defaultBaseDomain,
+    });
  }

  /**
--- a/app/server/lib/GristServer.ts
+++ b/app/server/lib/GristServer.ts
@@ -1,8 +1,9 @@
-import { GristLoadConfig } from 'app/common/gristUrls';
+import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
 import { FullUser, UserProfile } from 'app/common/UserAPI';
 import { Document } from 'app/gen-server/entity/Document';
 import { Organization } from 'app/gen-server/entity/Organization';
 import { Workspace } from 'app/gen-server/entity/Workspace';
+import { Activations } from 'app/gen-server/lib/Activations';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { IAccessTokens } from 'app/server/lib/AccessTokens';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
@@ -16,7 +17,7 @@ import { IPermitStore } from 'app/server/lib/Permit';
 import { ISendAppPageOptions } from 'app/server/lib/sendAppPage';
 import { fromCallback } from 'app/server/lib/serverUtils';
 import { Sessions } from 'app/server/lib/Sessions';
-import { TelemetryManager } from 'app/server/lib/TelemetryManager';
+import { ITelemetry } from 'app/server/lib/Telemetry';
 import * as express from 'express';
 import { IncomingMessage } from 'http';

@@ -40,10 +41,12 @@ export interface GristServer {
  getExternalPermitStore(): IPermitStore;
  getSessions(): Sessions;
  getComm(): Comm;
+  getDeploymentType(): GristDeploymentType;
  getHosts(): Hosts;
+  getActivations(): Activations;
  getHomeDBManager(): HomeDBManager;
  getStorageManager(): IDocStorageManager;
-  getTelemetryManager(): TelemetryManager|undefined;
+  getTelemetry(): ITelemetry;
  getNotifier(): INotifier;
  getDocTemplate(): Promise<DocTemplate>;
  getTag(): string;
@@ -117,10 +120,12 @@ export function createDummyGristServer(): GristServer {
    getResourceUrl() { return Promise.resolve(''); },
    getSessions() { throw new Error('no sessions'); },
    getComm() { throw new Error('no comms'); },
+    getDeploymentType() { return 'core'; },
    getHosts() { throw new Error('no hosts'); },
+    getActivations() { throw new Error('no activations'); },
    getHomeDBManager() { throw new Error('no db'); },
    getStorageManager() { throw new Error('no storage manager'); },
-    getTelemetryManager() { return undefined; },
+    getTelemetry() { return createDummyTelemetry(); },
    getNotifier() { throw new Error('no notifier'); },
    getDocTemplate() { throw new Error('no doc template'); },
    getTag() { return 'tag'; },
@@ -128,3 +133,11 @@ export function createDummyGristServer(): GristServer {
    getAccessTokens() { throw new Error('no access tokens'); },
  };
 }
+
+export function createDummyTelemetry(): ITelemetry {
+  return {
+    logEvent() { return Promise.resolve(); },
+    addEndpoints() { /* do nothing */ },
+    getTelemetryLevel() { return 'off'; },
+  };
+}
--- a/app/server/lib/ICreate.ts
+++ b/app/server/lib/ICreate.ts
@@ -1,19 +1,22 @@
+import {GristDeploymentType} from 'app/common/gristUrls';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {ExternalStorage} from 'app/server/lib/ExternalStorage';
-import {GristServer} from 'app/server/lib/GristServer';
+import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
 import {INotifier} from 'app/server/lib/INotifier';
 import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox';
 import {IShell} from 'app/server/lib/IShell';
 import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
+import {ITelemetry} from 'app/server/lib/Telemetry';

 export interface ICreate {

  Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
  Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
+  Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
  Shell?(): IShell;  // relevant to electron version of Grist only.

  // Create a space to store files externally, for storing either:
@@ -25,6 +28,7 @@ export interface ICreate {

  NSandbox(options: ISandboxCreationOptions): ISandbox;

+  deploymentType(): GristDeploymentType;
  sessionSecret(): string;
  // Check configuration of the app early enough to show on startup.
  configure?(): Promise<void>;
@@ -57,19 +61,26 @@ export interface ICreateBillingOptions {
  create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined;
 }

+export interface ICreateTelemetryOptions {
+  create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
+}
+
 export function makeSimpleCreator(opts: {
+  deploymentType: GristDeploymentType,
  sessionSecret?: string,
  storage?: ICreateStorageOptions[],
  billing?: ICreateBillingOptions,
  notifier?: ICreateNotifierOptions,
+  telemetry?: ICreateTelemetryOptions,
  sandboxFlavor?: string,
  shell?: IShell,
  getExtraHeadHtml?: () => string,
  getSqliteVariant?: () => SqliteVariant,
  getSandboxVariants?: () => Record<string, SpawnFn>,
 }): ICreate {
-  const {sessionSecret, storage, notifier, billing} = opts;
+  const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts;
  return {
+    deploymentType() { return deploymentType; },
    Billing(dbManager, gristConfig) {
      return billing?.create(dbManager, gristConfig) ?? {
        addEndpoints() { /* do nothing */ },
@@ -93,6 +104,9 @@ export function makeSimpleCreator(opts: {
      }
      return undefined;
    },
+    Telemetry(dbManager, gristConfig) {
+      return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();
+    },
    NSandbox(options) {
      return createSandbox(opts.sandboxFlavor || 'unsandboxed', options);
    },
--- a/app/server/lib/ProcessMonitor.ts
+++ b/app/server/lib/ProcessMonitor.ts
@@ -1,4 +1,5 @@
-import { TelemetryManager } from 'app/server/lib/TelemetryManager';
+import log from 'app/server/lib/log';
+import { ITelemetry } from 'app/server/lib/Telemetry';

 const MONITOR_PERIOD_MS = 5_000;        // take a look at memory usage this often
 const MEMORY_DELTA_FRACTION = 0.1;      // fraction by which usage should change to get reported
@@ -16,7 +17,7 @@ let _lastReportedCpuAverage: number = 0;
 * Monitor process memory (heap) and CPU usage, reporting as telemetry on an interval, and more
 * often when usage ticks up or down by a big enough delta.
 *
- * There is a single global process monitor, reporting to the telemetryManager passed into the
+ * There is a single global process monitor, reporting to the `telemetry` object passed into the
 * first call to start().
 *
 * Returns a function that stops the monitor, or null if there was already a process monitor
@@ -30,12 +31,12 @@ let _lastReportedCpuAverage: number = 0;
 *  - intervalMs:   Interval (in milliseconds) over which cpuAverage is reported. Being much
 *                  higher than MONITOR_PERIOD_MS is a sign of being CPU bound for that long.
 */
-export function start(telemetryManager: TelemetryManager): (() => void) | undefined {
+export function start(telemetry: ITelemetry): (() => void) | undefined {
  if (!_timer) {
    // Initialize variables needed for accurate first-tick measurement.
    _lastTickTime = Date.now();
    _lastCpuUsage = process.cpuUsage();
-    _timer = setInterval(() => monitor(telemetryManager), MONITOR_PERIOD_MS);
+    _timer = setInterval(() => monitor(telemetry), MONITOR_PERIOD_MS);

    return function stop() {
      clearInterval(_timer);
@@ -44,7 +45,7 @@ export function start(telemetryManager: TelemetryManager): (() => void) | undefi
  }
 }

-function monitor(telemetryManager: TelemetryManager) {
+function monitor(telemetry: ITelemetry) {
  const memoryUsage = process.memoryUsage();
  const heapUsed = memoryUsage.heapUsed;
  const cpuUsage = process.cpuUsage();
@@ -66,12 +67,15 @@ function monitor(telemetryManager: TelemetryManager) {
    Math.abs(heapUsed - _lastReportedHeapUsed) > _lastReportedHeapUsed * MEMORY_DELTA_FRACTION ||
    Math.abs(cpuAverage - _lastReportedCpuAverage) > CPU_DELTA_FRACTION
  ) {
-    telemetryManager.logEvent('processMonitor', {
-      heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
-      heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
-      cpuAverage: Math.round(cpuAverage * 100) / 100,
-      intervalMs,
-    });
+    telemetry.logEvent('processMonitor', {
+      full: {
+        heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
+        heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
+        cpuAverage: Math.round(cpuAverage * 100) / 100,
+        intervalMs,
+      },
+    })
+    .catch(e => log.error('failed to log telemetry event processMonitor', e));
    _lastReportedHeapUsed = heapUsed;
    _lastReportedCpuAverage = cpuAverage;
    _lastReportTime = now;
--- a/app/server/lib/Telemetry.ts
+++ b/app/server/lib/Telemetry.ts
@@ -0,0 +1,210 @@
+import {ApiError} from 'app/common/ApiError';
+import {
+  buildTelemetryEventChecker,
+  filterMetadata,
+  removeNullishKeys,
+  TelemetryEvent,
+  TelemetryEventChecker,
+  TelemetryEvents,
+  TelemetryLevel,
+  TelemetryLevels,
+  TelemetryMetadata,
+  TelemetryMetadataByLevel,
+} from 'app/common/Telemetry';
+import {HomeDBManager, HomeDBTelemetryEvents} from 'app/gen-server/lib/HomeDBManager';
+import {RequestWithLogin} from 'app/server/lib/Authorizer';
+import {GristServer} from 'app/server/lib/GristServer';
+import {LogMethods} from 'app/server/lib/LogMethods';
+import {stringParam} from 'app/server/lib/requestUtils';
+import * as express from 'express';
+import merge = require('lodash/merge');
+
+export interface ITelemetry {
+  logEvent(name: TelemetryEvent, metadata?: TelemetryMetadataByLevel): Promise<void>;
+  addEndpoints(app: express.Express): void;
+  getTelemetryLevel(): TelemetryLevel;
+}
+
+/**
+ * Manages telemetry for Grist.
+ */
+export class Telemetry implements ITelemetry {
+  private _telemetryLevel: TelemetryLevel;
+  private _deploymentType = this._gristServer.getDeploymentType();
+  private _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
+  private _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
+  'https://telemetry.getgrist.com/api/telemetry';
+
+  private _installationId: string | undefined;
+
+  private _errorLogger = new LogMethods('Telemetry ', () => ({}));
+  private _telemetryLogger = new LogMethods('Telemetry ', () => ({
+    eventType: 'telemetry',
+  }));
+
+  private _checkEvent: TelemetryEventChecker | undefined;
+
+  constructor(private _dbManager: HomeDBManager, private _gristServer: GristServer) {
+    this._initialize().catch((e) => {
+      this._errorLogger.error(undefined, 'failed to initialize', e);
+    });
+  }
+
+  /**
+   * Logs a telemetry `event` and its `metadata`.
+   *
+   * Depending on the deployment type, this will either forward the
+   * data to an endpoint (set via GRIST_TELEMETRY_URL) or log it
+   * directly. In hosted Grist, telemetry is logged directly, and
+   * subsequently sent to an OpenSearch instance via CloudWatch. In
+   * other deployment types, telemetry is forwarded to an endpoint
+   * of hosted Grist, which then handles logging to OpenSearch.
+   *
+   * Note that `metadata` is grouped by telemetry level, with only the
+   * groups meeting the current telemetry level being included in
+   * what's logged. If the current telemetry level is `off`, nothing
+   * will be logged. Otherwise, `metadata` will be filtered according
+   * to the current telemetry level, keeping only the groups that are
+   * less than or equal to the current level.
+   *
+   * Additionally, runtime checks are also performed to verify that the
+   * event and metadata being passed in are being logged appropriately
+   * for the configured telemetry level. If any checks fail, an error
+   * is thrown.
+   *
+   * Example:
+   *
+   * The following will only log the `rowCount` if the telemetry level is set
+   * to `limited`, and will log both the `method` and `userId` if the telemetry
+   * level is set to `full`:
+   *
+   * ```
+   * logEvent('documentUsage', {
+   *   limited: {
+   *     rowCount: 123,
+   *   },
+   *   full: {
+   *     userId: 1586,
+   *   },
+   * });
+   * ```
+   */
+  public async logEvent(
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadataByLevel
+  ) {
+    if (this._telemetryLevel === 'off') { return; }
+
+    metadata = filterMetadata(metadata, this._telemetryLevel);
+    this._checkTelemetryEvent(event, metadata);
+
+    if (this._shouldForwardTelemetryEvents) {
+      await this.forwardEvent(event, metadata);
+    } else {
+      this._telemetryLogger.rawLog('info', null, event, {
+        eventName: event,
+        eventSource: `grist-${this._deploymentType}`,
+        ...metadata,
+      });
+    }
+  }
+
+  /**
+   * Forwards a telemetry event and its metadata to another server.
+   */
+  public async forwardEvent(
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadata
+  ) {
+    try {
+      await fetch(this._forwardTelemetryEventsUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          event,
+          metadata,
+        }),
+      });
+    } catch (e) {
+      this._errorLogger.error(undefined, `failed to forward telemetry event ${event}`, e);
+    }
+  }
+
+  public addEndpoints(app: express.Application) {
+    /**
+     * Logs telemetry events and their metadata.
+     *
+     * Clients of this endpoint may be external Grist instances, so the behavior
+     * varies based on the presence of an `eventSource` key in the event metadata.
+     *
+     * If an `eventSource` key is present, the telemetry event will be logged
+     * directly, as the request originated from an external source; runtime checks
+     * of telemetry data are skipped since they should have already occured at the
+     * source. Otherwise, the event will only be logged after passing various
+     * checks.
+     */
+    app.post('/api/telemetry', async (req, resp) => {
+      const mreq = req as RequestWithLogin;
+      const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
+      if ('eventSource' in req.body.metadata) {
+        this._telemetryLogger.rawLog('info', null, event, {
+          eventName: event,
+          ...(removeNullishKeys(req.body.metadata)),
+        });
+      } else {
+        try {
+          await this.logEvent(event as TelemetryEvent, merge(
+            {
+              limited: {
+                eventSource: `grist-${this._deploymentType}`,
+                ...(this._deploymentType !== 'saas' ? {installationId: this._installationId} : {}),
+              },
+              full: {
+                userId: mreq.userId,
+                altSessionId: mreq.altSessionId,
+              },
+            },
+            req.body.metadata,
+          ));
+        } catch (e) {
+          this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e);
+          throw new ApiError(`Telemetry failed to log telemetry event ${event}`, 500);
+        }
+      }
+      return resp.status(200).send();
+    });
+  }
+
+  public getTelemetryLevel() {
+    return this._telemetryLevel;
+  }
+
+  private async _initialize() {
+    if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
+      this._telemetryLevel = TelemetryLevels.check(process.env.GRIST_TELEMETRY_LEVEL);
+      this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryLevel);
+    } else {
+      this._telemetryLevel = 'off';
+    }
+
+    const {id} = await this._gristServer.getActivations().current();
+    this._installationId = id;
+
+    for (const event of HomeDBTelemetryEvents.values) {
+      this._dbManager.on(event, async (metadata) => {
+        this.logEvent(event, metadata).catch(e =>
+          this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e));
+      });
+    }
+  }
+
+  private _checkTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadata) {
+    if (!this._checkEvent) {
+      throw new Error('Telemetry._checkEvent is undefined');
+    }
+
+    this._checkEvent(event, metadata);
+  }
+}
--- a/app/server/lib/Triggers.ts
+++ b/app/server/lib/Triggers.ts
@@ -630,7 +630,7 @@ export class DocTriggers {
        meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
        this._log("Sending batch of webhook events", meta);
        this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
-          numEvents: batch.length,
+          limited: {numEvents: meta.numEvents},
        });
        success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
        if (this._loopAbort.signal.aborted) {
--- a/app/server/lib/sendAppPage.ts
+++ b/app/server/lib/sendAppPage.ts
@@ -3,7 +3,7 @@ import {isAffirmative} from 'app/common/gutil';
 import {getTagManagerSnippet} from 'app/common/tagManager';
 import {Document} from 'app/common/UserAPI';
 import {SUPPORT_EMAIL} from 'app/gen-server/lib/HomeDBManager';
-import {isAnonymousUser, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {isAnonymousUser, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {GristServer} from 'app/server/lib/GristServer';
 import {getSupportedEngineChoices} from 'app/server/lib/serverUtils';
@@ -31,9 +31,16 @@ export interface ISendAppPageOptions {
  googleTagManager?: true | false | 'anon';
 }

-export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadConfig>,
-                                baseDomain?: string, req?: express.Request
-): GristLoadConfig {
+export interface MakeGristConfigOptons {
+  homeUrl: string|null;
+  extra: Partial<GristLoadConfig>;
+  baseDomain?: string;
+  req?: express.Request;
+  server?: GristServer|null;
+}
+
+export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
+  const {homeUrl, extra, baseDomain, req, server} = options;
  // .invalid is a TLD the IETF promises will never exist.
  const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
  const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") ||
@@ -69,6 +76,8 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
    featureFormulaAssistant: isAffirmative(process.env.GRIST_FORMULA_ASSISTANT),
    supportEmail: SUPPORT_EMAIL,
    userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
+    telemetry: server ? getTelemetryConfig(server) : undefined,
+    deploymentType: server?.getDeploymentType(),
    ...extra,
  };
 }
@@ -94,14 +103,18 @@ export function makeMessagePage(staticDir: string) {
 * placeholders replaced.
 */
 export function makeSendAppPage(opts: {
-  server: GristServer|null, staticDir: string, tag: string, testLogin?: boolean,
+  server: GristServer, staticDir: string, tag: string, testLogin?: boolean,
  baseDomain?: string
 }) {
  const {server, staticDir, tag, testLogin} = opts;
  return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
-    // .invalid is a TLD the IETF promises will never exist.
-    const config = makeGristConfig(server ? server.getHomeUrl(req) : null, options.config,
-      opts.baseDomain, req);
+      const config = makeGristConfig({
+        homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
+        extra: options.config,
+        baseDomain: opts.baseDomain,
+        req,
+        server,
+      });

    // We could cache file contents in memory, but the filesystem does caching too, and compared
    // to that, the performance gain is unlikely to be meaningful. So keep it simple here.
@@ -112,7 +125,7 @@ export function makeSendAppPage(opts: {
    const tagManagerSnippet = needTagManager ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) : '';
    const staticOrigin = process.env.APP_STATIC_URL || "";
    const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
-    const customHeadHtmlSnippet = server?.create.getExtraHeadHtml?.() ?? "";
+    const customHeadHtmlSnippet = server.create.getExtraHeadHtml?.() ?? "";
    const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
    // Preload all languages that will be used or are requested by client.
    const preloads = req.languages
@@ -127,7 +140,7 @@ export function makeSendAppPage(opts: {
      .replace("<!-- INSERT WARNING -->", warning)
      .replace("<!-- INSERT TITLE -->", getPageTitle(req, config))
      .replace("<!-- INSERT META -->", getPageMetadataHtmlSnippet(config))
-      .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server?.getGristConfig()))
+      .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server.getGristConfig()))
      .replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet)
      .replace("<!-- INSERT LOCALE -->", preloads)
      .replace("<!-- INSERT CUSTOM -->", customHeadHtmlSnippet)
@@ -150,6 +163,13 @@ function getFeatures(): IFeature[] {
  return Features.checkAll(difference(enabledFeatures, disabledFeatures));
 }

+function getTelemetryConfig(server: GristServer) {
+  const telemetry = server.getTelemetry();
+  return {
+    telemetryLevel: telemetry.getTelemetryLevel(),
+  };
+}
+
 function configuredPageTitleSuffix() {
  const result = process.env.GRIST_PAGE_TITLE_SUFFIX;
  return result === "_blank" ? "" : result;
--- a/app/server/lib/serverUtils.ts
+++ b/app/server/lib/serverUtils.ts
@@ -7,6 +7,7 @@ import uuidv4 from 'uuid/v4';
 import {AbortSignal} from 'node-abort-controller';

 import {EngineCode} from 'app/common/DocumentSettings';
+import {TelemetryMetadataByLevel} from 'app/common/Telemetry';
 import log from 'app/server/lib/log';
 import {OpenMode, SQLiteDB} from 'app/server/lib/SQLiteDB';
 import {getDocSessionAccessOrNull, getDocSessionUser, OptDocSession} from './DocSession';
@@ -166,14 +167,18 @@ export function getLogMetaFromDocSession(docSession: OptDocSession) {
 /**
 * Extract telemetry metadata from session.
 */
-export function getTelemetryMetaFromDocSession(docSession: OptDocSession) {
+export function getTelemetryMetaFromDocSession(docSession: OptDocSession): TelemetryMetadataByLevel {
  const client = docSession.client;
  const access = getDocSessionAccessOrNull(docSession);
  const user = getDocSessionUser(docSession);
  return {
-    access,
-    ...(user ? {userId: user.id} : {}),
-    ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+    limited: {
+      access,
+    },
+    full: {
+      ...(user ? {userId: user.id} : {}),
+      ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+    },
  };
 }

--- a/app/server/mergedServerMain.ts
+++ b/app/server/mergedServerMain.ts
@@ -119,19 +119,19 @@ export async function main(port: number, serverTypes: ServerType[],
    server.addHomeApi();
    server.addBillingApi();
    server.addNotifier();
+    server.addTelemetry();
    await server.addHousekeeper();
    await server.addLoginRoutes();
    server.addAccountPage();
    server.addBillingPages();
    server.addWelcomePaths();
    server.addLogEndpoint();
-    server.addTelemetryEndpoint();
    server.addGoogleAuthEndpoint();
  }

  if (includeDocs) {
    server.addJsonSupport();
-    server.addTelemetryEndpoint();
+    server.addTelemetry();
    await server.addDoc();
  }