(core) Add optional telemetry to grist-core

Summary:
Adds support for optional telemetry to grist-core.

A new environment variable, GRIST_TELEMETRY_LEVEL, controls the level of telemetry collected.

Test Plan: Server and unit tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: dsagal, anaisconce

Differential Revision: https://phab.getgrist.com/D3880

README.md

@@ -285,6 +285,7 @@ GRIST_SINGLE_ORG | set to an org "domain" to pin client to that org
 GRIST_HELP_CENTER | set the help center link ref
 GRIST_SUPPORT_ANON | if set to 'true', show UI for anonymous access (not shown by default)
 GRIST_SUPPORT_EMAIL | if set, give a user with the specified email support powers. The main extra power is the ability to share sites, workspaces, and docs with all users in a listed way.
+GRIST_TELEMETRY_LEVEL | the telemetry level. Can be set to: `off` (default), `limited`, or `full`.
 GRIST_THROTTLE_CPU | if set, CPU throttling is enabled
 GRIST_USER_ROOT     | an extra path to look for plugins in.
 GRIST_UI_FEATURES | comma-separated list of UI features to enable. Allowed names of parts: `helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,tutorials`. If a part also exists in GRIST_HIDE_UI_ELEMENTS, it won't be enabled.

app/client/lib/helpScout.ts

@@ -197,11 +197,11 @@ function _beaconOpen(userObj: IUserObj|null, options: IBeaconOpenOptions) {
 
   Beacon('once', 'open', () => logTelemetryEvent('beaconOpen'));
   Beacon('on', 'article-viewed', (article) => logTelemetryEvent('beaconArticleViewed', {
-    articleId: article!.id,
+    full: {articleId: article!.id},
   }));
   Beacon('on', 'email-sent', () => logTelemetryEvent('beaconEmailSent'));
   Beacon('on', 'search', (search) => logTelemetryEvent('beaconSearch', {
-    searchQuery: search!.query,
+    full: {searchQuery: search!.query},
   }));
 }
 

app/client/lib/telemetry.ts

@@ -1,14 +1,20 @@
 import {logError} from 'app/client/models/errors';
-import {TelemetryEventName} from 'app/common/Telemetry';
+import {Level, TelemetryContracts, TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
-import {fetchFromHome, pageHasHome} from 'app/common/urlUtils';
+import {fetchFromHome, getGristConfig, pageHasHome} from 'app/common/urlUtils';
 
-export function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<string, any>) {
+export function logTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadataByLevel) {
   if (!pageHasHome()) { return; }
 
+  const {telemetry} = getGristConfig();
+  if (!telemetry) { return; }
+
+  const {telemetryLevel} = telemetry;
+  if (Level[telemetryLevel] < TelemetryContracts[event].minimumTelemetryLevel) { return; }
+
   fetchFromHome('/api/telemetry', {
     method: 'POST',
     body: JSON.stringify({
-      name,
+      event,
       metadata,
     }),
     credentials: 'include',
@@ -17,7 +23,7 @@ export function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<st
       'X-Requested-With': 'XMLHttpRequest',
     },
   }).catch((e: Error) => {
-    console.warn(`Failed to log telemetry event ${name}`, e);
+    console.warn(`Failed to log telemetry event ${event}`, e);
     logError(e);
   });
 }

app/client/models/errors.ts

@@ -215,6 +215,6 @@ export function logError(error: Error|string) {
   }).catch(e => {
     // There ... isn't much we can do about this.
     // tslint:disable-next-line:no-console
-    console.warn('Failed to log event', event);
+    console.warn('Failed to log event', e);
   });
 }

app/client/ui/AccountWidget.ts

@@ -1,7 +1,6 @@
 import {AppModel} from 'app/client/models/AppModel';
 import {DocPageModel} from 'app/client/models/DocPageModel';
 import {getLoginOrSignupUrl, getLoginUrl, getLogoutUrl, urlState} from 'app/client/models/gristUrlState';
-import {buildUserMenuBillingItem} from 'app/client/ui/BillingButtons';
 import {manageTeamUsers} from 'app/client/ui/OpenUserManager';
 import {createUserImage} from 'app/client/ui/UserImage';
 import * as viewport from 'app/client/ui/viewport';
@@ -16,6 +15,7 @@ import {Disposable, dom, DomElementArg, styled} from 'grainjs';
 import {cssMenuItem} from 'popweasel';
 import {maybeAddSiteSwitcherSection} from 'app/client/ui/SiteSwitcher';
 import {makeT} from 'app/client/lib/localization';
+import {getGristConfig} from 'app/common/urlUtils';
 
 const t = makeT('AccountWidget');
 
@@ -98,7 +98,8 @@ export class AccountWidget extends Disposable {
         // Don't show on doc pages, or for personal orgs.
         null),
 
-      buildUserMenuBillingItem(this._appModel),
+      this._maybeBuildBillingPageMenuItem(),
+      this._maybeBuildActivationPageMenuItem(),
 
       mobileModeToggle,
 
@@ -141,6 +142,33 @@ export class AccountWidget extends Disposable {
     }
     this._appModel.topAppModel.initialize();
   }
+
+  private _maybeBuildBillingPageMenuItem() {
+    const {deploymentType} = getGristConfig();
+    if (deploymentType !== 'saas') { return null; }
+
+    const {currentValidUser, currentOrg, isTeamSite} = this._appModel;
+    const isBillingManager = Boolean(currentOrg && currentOrg.billingAccount &&
+      (currentOrg.billingAccount.isManager || currentValidUser?.isSupport));
+
+    return isTeamSite ?
+      // For links, disabling with just a class is hard; easier to just not make it a link.
+      // TODO weasel menus should support disabling menuItemLink.
+      (isBillingManager ?
+        menuItemLink(urlState().setLinkUrl({billing: 'billing'}), 'Billing Account') :
+        menuItem(() => null, 'Billing Account', dom.cls('disabled', true))
+      ) :
+      menuItem(() => this._appModel.showUpgradeModal(), 'Upgrade Plan');
+  }
+
+  private _maybeBuildActivationPageMenuItem() {
+    const {activation, deploymentType} = getGristConfig();
+    if (deploymentType !== 'enterprise' || !activation?.isManager) {
+      return null;
+    }
+
+    return menuItemLink('Activation', urlState().setLinkUrl({activation: 'activation'}));
+  }
 }
 
 const cssAccountWidget = styled('div', `

app/client/ui/AppHeader.ts

@@ -1,5 +1,4 @@
 import {getWelcomeHomeUrl, urlState} from 'app/client/models/gristUrlState';
-import {buildAppMenuBillingItem} from 'app/client/ui/BillingButtons';
 import {getTheme} from 'app/client/ui/CustomThemes';
 import {cssLeftPane} from 'app/client/ui/PagePanels';
 import {colors, testId, theme, vars} from 'app/client/ui2018/cssVars';
@@ -14,6 +13,7 @@ import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
 import {maybeAddSiteSwitcherSection} from 'app/client/ui/SiteSwitcher';
 import {BindableValue, Disposable, dom, DomContents, styled} from 'grainjs';
 import {makeT} from 'app/client/lib/localization';
+import {getGristConfig} from 'app/common/urlUtils';
 
 const t = makeT('AppHeader');
 
@@ -71,7 +71,8 @@ export class AppHeader extends Disposable {
             // Don't show on doc pages, or for personal orgs.
             null),
 
-          buildAppMenuBillingItem(this._appModel, testId('orgmenu-billing')),
+          this._maybeBuildBillingPageMenuItem(),
+          this._maybeBuildActivationPageMenuItem(),
 
           maybeAddSiteSwitcherSection(this._appModel),
         ], { placement: 'bottom-start' }),
@@ -88,6 +89,40 @@ export class AppHeader extends Disposable {
       return {href: getWelcomeHomeUrl()};
     }
   }
+
+  private _maybeBuildBillingPageMenuItem() {
+    const {deploymentType} = getGristConfig();
+    if (deploymentType !== 'saas') { return null; }
+
+    const {currentOrg} = this._appModel;
+    const isBillingManager = this._appModel.isBillingManager() || this._appModel.isSupport();
+    return currentOrg && !currentOrg.owner ?
+      // For links, disabling with just a class is hard; easier to just not make it a link.
+      // TODO weasel menus should support disabling menuItemLink.
+      (isBillingManager
+        ? menuItemLink(
+          urlState().setLinkUrl({billing: 'billing'}),
+          'Billing Account',
+          testId('orgmenu-billing'),
+        )
+        : menuItem(
+          () => null,
+          'Billing Account',
+          dom.cls('disabled', true),
+          testId('orgmenu-billing'),
+        )
+      ) :
+      null;
+  }
+
+  private _maybeBuildActivationPageMenuItem() {
+    const {activation, deploymentType} = getGristConfig();
+    if (deploymentType !== 'enterprise' || !activation?.isManager) {
+      return null;
+    }
+
+    return menuItemLink('Activation', urlState().setLinkUrl({activation: 'activation'}));
+  }
 }
 
 export function productPill(org: Organization|null, options: {large?: boolean} = {}): DomContents {

app/client/ui/OpenVideoTour.ts

@@ -37,7 +37,7 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
         if (youtubePlayer.isLoading()) { return; }
 
         logTelemetryEvent('watchedVideoTour', {
-          watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime()),
+          limited: {watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime())},
         });
       });
 

app/client/ui/WelcomeCoachingCall.ts

@@ -0,0 +1,162 @@
+import {AppModel} from 'app/client/models/AppModel';
+import {bigBasicButton, bigPrimaryButtonLink} from 'app/client/ui2018/buttons';
+import {testId, theme, vars} from 'app/client/ui2018/cssVars';
+import {cardPopup, cssPopupBody, cssPopupButtons, cssPopupCloseButton,
+        cssPopupTitle} from 'app/client/ui2018/popups';
+import {icon} from 'app/client/ui2018/icons';
+import {getGristConfig} from 'app/common/urlUtils';
+import {dom, styled} from 'grainjs';
+
+const FREE_COACHING_CALL_URL = 'https://calendly.com/grist-team/grist-free-coaching-call';
+
+export function shouldShowWelcomeCoachingCall(appModel: AppModel) {
+  const {deploymentType} = getGristConfig();
+  if (deploymentType !== 'saas') { return false; }
+
+  const {behavioralPromptsManager, dismissedWelcomePopups} = appModel;
+
+  // Defer showing coaching call until Add New tip is dismissed.
+  const hasSeenAddNewTip = behavioralPromptsManager.hasSeenTip('addNew');
+  const shouldShowTips = behavioralPromptsManager.shouldShowTips();
+  if (!hasSeenAddNewTip && shouldShowTips) { return false; }
+
+  const popup = dismissedWelcomePopups.get().find(p => p.id === 'coachingCall');
+  return (
+    // Only show if the user is an owner.
+    appModel.isOwner() && (
+      // And preferences for the popup haven't been saved before.
+      popup === undefined ||
+      // Or the popup has been shown before, and it's time to shown it again.
+      popup.nextAppearanceAt !== null && popup.nextAppearanceAt <= Date.now()
+    )
+  );
+}
+
+/**
+ * Shows a popup with an offer for a free coaching call.
+ */
+export function showWelcomeCoachingCall(triggerElement: Element, appModel: AppModel) {
+  const {dismissedWelcomePopups} = appModel;
+
+  cardPopup(triggerElement, (ctl) => {
+    const dismissPopup = (scheduleNextAppearance?: boolean) => {
+      const dismissedPopups = dismissedWelcomePopups.get();
+      const newDismissedPopups = [...dismissedPopups];
+      const coachingPopup = newDismissedPopups.find(p => p.id === 'coachingCall');
+      if (!coachingPopup) {
+        newDismissedPopups.push({
+          id: 'coachingCall',
+          lastDismissedAt: Date.now(),
+          timesDismissed: 1,
+          nextAppearanceAt: scheduleNextAppearance
+            ? new Date().setDate(new Date().getDate() + 7)
+            : null,
+        });
+      } else {
+        Object.assign(coachingPopup, {
+          lastDismissedAt: Date.now(),
+          timesDismissed: coachingPopup.timesDismissed + 1,
+          nextAppearanceAt: scheduleNextAppearance && coachingPopup.timesDismissed + 1 <= 1
+            ? new Date().setDate(new Date().getDate() + 7)
+            : null,
+        });
+      }
+      dismissedWelcomePopups.set(newDismissedPopups);
+      ctl.close();
+    };
+
+    // TODO: i18n
+    return [
+      cssPopup.cls(''),
+      cssPopupHeader(
+        cssLogoAndName(
+          cssLogo(),
+          cssName('Grist'),
+        ),
+        cssPopupCloseButton(
+          cssCloseIcon('CrossBig'),
+          dom.on('click', () => dismissPopup(true)),
+          testId('popup-close-button'),
+        ),
+      ),
+      cssPopupTitle('Free Coaching Call', testId('popup-title')),
+      cssPopupBody(
+        cssBody(
+          dom('div',
+            'Schedule your ', cssBoldText('free coaching call'), ' with a member of our team.'
+          ),
+          dom('div',
+            "On the call, we'll take the time to understand your needs and "
+            + 'tailor the call to you. We can show you the Grist basics, or start '
+            + 'working with your data right away to build the dashboards you need.'
+          ),
+        ),
+        testId('popup-body'),
+      ),
+      cssPopupButtons(
+        bigPrimaryButtonLink(
+          'Schedule Call',
+          dom.on('click', () => dismissPopup(false)),
+          {
+            href: FREE_COACHING_CALL_URL,
+            target: '_blank',
+          },
+          testId('popup-primary-button'),
+        ),
+        bigBasicButton(
+          'Maybe Later',
+          dom.on('click', () => dismissPopup(true)),
+          testId('popup-basic-button'),
+        ),
+      ),
+      testId('coaching-call'),
+    ];
+  });
+}
+
+const cssBody = styled('div', `
+  display: flex;
+  flex-direction: column;
+  row-gap: 16px;
+`);
+
+const cssBoldText = styled('span', `
+  font-weight: 600;
+`);
+
+const cssCloseIcon = styled(icon, `
+  padding: 12px;
+`);
+
+const cssName = styled('div', `
+  color: ${theme.popupCloseButtonFg};
+  font-size: ${vars.largeFontSize};
+  font-weight: 600;
+`);
+
+const cssLogo = styled('div', `
+  flex: none;
+  height: 32px;
+  width: 32px;
+  background-image: var(--icon-GristLogo);
+  background-size: ${vars.logoSize};
+  background-repeat: no-repeat;
+  background-position: center;
+`);
+
+const cssLogoAndName = styled('div', `
+  display: flex;
+  align-items: center;
+  gap: 4px;
+`);
+
+const cssPopup = styled('div', `
+  display: flex;
+  flex-direction: column;
+`);
+
+const cssPopupHeader = styled('div', `
+  display: flex;
+  justify-content: space-between;
+  margin-bottom: 16px;
+`);

app/client/ui/WelcomeCoachingCallStub.ts

@@ -1,9 +0,0 @@
-import {AppModel} from 'app/client/models/AppModel';
-
-export function shouldShowWelcomeCoachingCall(_app: AppModel) {
-  return false;
-}
-
-export function showWelcomeCoachingCall(_triggerElement: Element, _app: AppModel) {
-
-}

app/common/Telemetry.ts

@@ -1,6 +1,796 @@
-export const TelemetryTemplateSignupCookieName = 'gr_template_signup_trk';
+import {StringUnion} from 'app/common/StringUnion';
+import pickBy = require('lodash/pickBy');
 
-export const TelemetryEventNames = [
+/**
+ * Telemetry levels, in increasing order of data collected.
+ */
+export enum Level {
+  off = 0,
+  limited = 1,
+  full = 2,
+}
+
+/**
+ * A set of contracts that all telemetry events must follow prior to being
+ * logged.
+ *
+ * Currently, this includes meeting minimum telemetry levels for events
+ * and their metadata, and passing in the correct data type for the value of
+ * each metadata property.
+ *
+ * The `minimumTelemetryLevel` defined at the event level will also be applied
+ * to all metadata properties of an event, and can be overridden at the metadata
+ * level.
+ */
+export const TelemetryContracts: TelemetryContracts = {
+  /**
+   * Triggered when an HTTP request with an API key is made.
+   */
+  apiUsage: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * The HTTP request method (e.g. GET, POST, PUT).
+       */
+      method: {
+        dataType: 'string',
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+      /**
+       * The User-Agent HTTP request header.
+       */
+      userAgent: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered when HelpScout Beacon is opened.
+   */
+  beaconOpen: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered when an article is opened in HelpScout Beacon.
+   */
+  beaconArticleViewed: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * The id of the article.
+       */
+      articleId: {
+        dataType: 'string',
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered when an email is sent in HelpScout Beacon.
+   */
+  beaconEmailSent: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered when a search is made in HelpScout Beacon.
+   */
+  beaconSearch: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * The search query.
+       */
+      searchQuery: {
+        dataType: 'string',
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered when a document is forked.
+   */
+  documentForked: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * A hash of the doc id.
+       */
+      docIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The id of the site containing the forked document.
+       */
+      siteId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The type of the site.
+       */
+      siteType: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A hash of the fork id.
+       */
+      forkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * A hash of the full id of the fork, including the trunk id and fork id.
+       */
+      forkDocIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * A hash of the trunk id.
+       */
+      trunkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * Whether the trunk is a template.
+       */
+      isTemplate: {
+        dataType: 'boolean',
+      },
+      /**
+       * Timestamp of the last update to the trunk document.
+       */
+      lastActivity: {
+        dataType: 'date',
+      },
+    },
+  },
+  /**
+   * Triggered when a public document or template is opened.
+   */
+  documentOpened: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * A hash of the doc id.
+       */
+      docIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The document access level of the user that triggered this event.
+       */
+      access: {
+        dataType: 'boolean',
+      },
+      /**
+       * Whether the document is public.
+       */
+      isPublic: {
+        dataType: 'boolean',
+      },
+      /**
+       * Whether a snapshot was opened.
+       */
+      isSnapshot: {
+        dataType: 'boolean',
+      },
+      /**
+       * Whether the document is a template.
+       */
+      isTemplate: {
+        dataType: 'boolean',
+      },
+      /**
+       * Timestamp of when the document was last updated.
+       */
+      lastUpdated: {
+        dataType: 'date',
+      },
+    },
+  },
+  /**
+   * Triggered on doc open and close, as well as hourly while a document is open.
+   */
+  documentUsage: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * A hash of the doc id.
+       */
+      docIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * What caused this event to trigger.
+       *
+       * May be either "docOpen", "interval", or "docClose".
+       */
+      triggeredBy: {
+        dataType: 'string',
+      },
+      /**
+       * Whether the document is public.
+       */
+      isPublic: {
+        dataType: 'boolean',
+      },
+      /**
+       * The number of rows in the document.
+       */
+      rowCount: {
+        dataType: 'number',
+      },
+      /**
+       * The total size of all data in the document, excluding attachments.
+       */
+      dataSizeBytes: {
+        dataType: 'number',
+      },
+      /**
+       * The total size of all attachments in the document.
+       */
+      attachmentsSize: {
+        dataType: 'number',
+      },
+      /**
+       * The number of access rules in the document.
+       */
+      numAccessRules: {
+        dataType: 'number',
+      },
+      /**
+       * The number of user attributes in the document.
+       */
+      numUserAttributes: {
+        dataType: 'number',
+      },
+      /**
+       * The number of attachments in the document.
+       */
+      numAttachments: {
+        dataType: 'number',
+      },
+      /**
+       * A list of unique file extensions compiled from all of the document's attachments.
+       */
+      attachmentTypes: {
+        dataType: 'string[]',
+      },
+      /**
+       * The number of charts in the document.
+       */
+      numCharts: {
+        dataType: 'number',
+      },
+      /**
+       * A list of chart types of every chart in the document.
+       */
+      chartTypes: {
+        dataType: 'string[]',
+      },
+      /**
+       * The number of linked charts in the document.
+       */
+      numLinkedCharts: {
+        dataType: 'number',
+      },
+      /**
+       * The number of linked widgets in the document.
+       */
+      numLinkedWidgets: {
+        dataType: 'number',
+      },
+      /**
+       * The number of columns in the document.
+       */
+      numColumns: {
+        dataType: 'number',
+      },
+      /**
+       * The number of columns with conditional formatting in the document.
+       */
+      numColumnsWithConditionalFormatting: {
+        dataType: 'number',
+      },
+      /**
+       * The number of formula columns in the document.
+       */
+      numFormulaColumns: {
+        dataType: 'number',
+      },
+      /**
+       * The number of trigger formula columns in the document.
+       */
+      numTriggerFormulaColumns: {
+        dataType: 'number',
+      },
+      /**
+       * The number of summary formula columns in the document.
+       */
+      numSummaryFormulaColumns: {
+        dataType: 'number',
+      },
+      /**
+       * The number of fields with conditional formatting in the document.
+       */
+      numFieldsWithConditionalFormatting: {
+        dataType: 'number',
+      },
+      /**
+       * The number of tables in the document.
+       */
+      numTables: {
+        dataType: 'number',
+      },
+      /**
+       * The number of on-demand tables in the document.
+       */
+      numOnDemandTables: {
+        dataType: 'number',
+      },
+      /**
+       * The number of tables with conditional formatting in the document.
+       */
+      numTablesWithConditionalFormatting: {
+        dataType: 'number',
+      },
+      /**
+       * The number of summary tables in the document.
+       */
+      numSummaryTables: {
+        dataType: 'number',
+      },
+      /**
+       * The number of custom widgets in the document.
+       */
+      numCustomWidgets: {
+        dataType: 'number',
+      },
+      /**
+       * A list of plugin ids for every custom widget in the document.
+       *
+       * The ids of widgets not created by Grist Labs are replaced with "externalId".
+       */
+      customWidgetIds: {
+        dataType: 'string[]',
+      },
+    },
+  },
+  /**
+   * Triggered every 5 seconds.
+   */
+  processMonitor: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /** Size of JS heap in use, in MiB. */
+      heapUsedMB: {
+        dataType: 'number',
+      },
+      /** Total heap size, in MiB, allocated for JS by V8. */
+      heapTotalMB: {
+        dataType: 'number',
+      },
+      /** Fraction (typically between 0 and 1) of CPU usage. Includes all threads, so may exceed 1. */
+      cpuAverage: {
+        dataType: 'number',
+      },
+      /** Interval (in milliseconds) over which `cpuAverage` is reported. */
+      intervalMs: {
+        dataType: 'number',
+      },
+    },
+  },
+  /**
+   * Triggered when sending webhooks.
+   */
+  sendingWebhooks: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * The number of events in the batch of webhooks being sent.
+       */
+      numEvents: {
+        dataType: 'number',
+      },
+      /**
+       * A hash of the doc id.
+       */
+      docIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+    },
+  },
+  /**
+   * Triggered after a user successfully verifies their account during sign-up.
+   *
+   * Not triggered in grist-core.
+   */
+  signupVerified: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * Whether the user viewed any templates before signing up.
+       */
+      isAnonymousTemplateSignup: {
+        dataType: 'boolean',
+      },
+      /**
+       * The doc id of the template the user last viewed before signing up, if any.
+       */
+      templateId: {
+        dataType: 'string',
+      },
+    },
+  },
+  /**
+   * Triggered daily.
+   */
+  siteMembership: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+      },
+      /**
+       * The number of users with an owner role in this site.
+       */
+      numOwners: {
+        dataType: 'number',
+      },
+      /**
+       * The number of users with an editor role in this site.
+       */
+      numEditors: {
+        dataType: 'number',
+      },
+      /**
+       * The number of users with a viewer role in this site.
+       */
+      numViewers: {
+        dataType: 'number',
+      },
+    },
+  },
+  /**
+   * Triggered daily.
+   */
+  siteUsage: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+      },
+      /**
+       * Whether the site's subscription is in good standing.
+       */
+      inGoodStanding: {
+        dataType: 'boolean',
+      },
+      /**
+       * The Stripe Plan id associated with this site.
+       */
+      stripePlanId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * The number of docs in this site.
+       */
+      numDocs: {
+        dataType: 'number',
+      },
+      /**
+       * The number of workspaces in this site.
+       */
+      numWorkspaces: {
+        dataType: 'number',
+      },
+      /**
+       * The number of site members.
+       */
+      numMembers: {
+        dataType: 'number',
+      },
+      /**
+       * A timestamp of the most recent update made to a site document.
+       */
+      lastActivity: {
+        dataType: 'date',
+      },
+    },
+  },
+  /**
+   * Triggered on changes to tutorial progress.
+   */
+  tutorialProgressChanged: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * A hash of the tutorial fork id.
+       */
+      tutorialForkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * A hash of the tutorial trunk id.
+       */
+      tutorialTrunkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The 0-based index of the last tutorial slide the user had open.
+       */
+      lastSlideIndex: {
+        dataType: 'number',
+      },
+      /**
+       * The total number of slides in the tutorial.
+       */
+      numSlides: {
+        dataType: 'number',
+      },
+      /**
+       * Percentage of tutorial completion.
+       */
+      percentComplete: {
+        dataType: 'number',
+      },
+    },
+  },
+  /**
+   * Triggered when a tutorial is restarted.
+   */
+  tutorialRestarted: {
+    minimumTelemetryLevel: Level.full,
+    metadataContracts: {
+      /**
+       * A hash of the tutorial fork id.
+       */
+      tutorialForkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * A hash of the tutorial trunk id.
+       */
+      tutorialTrunkIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * A hash of the doc id.
+       */
+      docIdDigest: {
+        dataType: 'string',
+      },
+      /**
+       * The site id.
+       */
+      siteId: {
+        dataType: 'number',
+      },
+      /**
+       * The site type.
+       */
+      siteType: {
+        dataType: 'string',
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+      },
+    },
+  },
+  /**
+   * Triggered when the video tour is closed.
+   */
+  watchedVideoTour: {
+    minimumTelemetryLevel: Level.limited,
+    metadataContracts: {
+      /**
+       * The number of seconds elapsed in the video player.
+       */
+      watchTimeSeconds: {
+        dataType: 'number',
+      },
+      /**
+       * The id of the user that triggered this event.
+       */
+      userId: {
+        dataType: 'number',
+        minimumTelemetryLevel: Level.full,
+      },
+      /**
+       * A random, session-based identifier for the user that triggered this event.
+       */
+      altSessionId: {
+        dataType: 'string',
+        minimumTelemetryLevel: Level.full,
+      },
+    },
+  },
+};
+
+type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
+
+export const TelemetryEvents = StringUnion(
   'apiUsage',
   'beaconOpen',
   'beaconArticleViewed',
@@ -17,6 +807,142 @@ export const TelemetryEventNames = [
   'tutorialProgressChanged',
   'tutorialRestarted',
   'watchedVideoTour',
-] as const;
+);
+export type TelemetryEvent = typeof TelemetryEvents.type;
 
-export type TelemetryEventName = typeof TelemetryEventNames[number];
+interface TelemetryEventContract {
+  minimumTelemetryLevel: Level;
+  metadataContracts?: Record<string, MetadataContract>;
+}
+
+interface MetadataContract {
+  dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date';
+  minimumTelemetryLevel?: Level;
+}
+
+export type TelemetryMetadataByLevel = Partial<Record<EnabledTelemetryLevel, TelemetryMetadata>>;
+
+export type EnabledTelemetryLevel = Exclude<TelemetryLevel, 'off'>;
+
+export const TelemetryLevels = StringUnion('off', 'limited', 'full');
+export type TelemetryLevel = typeof TelemetryLevels.type;
+
+export type TelemetryMetadata = Record<string, any>;
+
+/**
+ * The name of a cookie that's set whenever a template is opened.
+ *
+ * The cookie remembers the last template that was opened, which is then read during
+ * sign-up to track which templates were viewed before sign-up.
+ */
+export const TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME = 'gr_template_signup_trk';
+
+// A set of metadata keys that are always allowed when logging.
+const ALLOWED_METADATA_KEYS = new Set(['eventSource', 'installationId']);
+
+/**
+ * Returns a function that accepts a telemetry event and metadata, and performs various
+ * checks on it based on a set of contracts and the `telemetryLevel`.
+ *
+ * The function throws if any checks fail.
+ */
+export function buildTelemetryEventChecker(telemetryLevel: TelemetryLevel) {
+  const currentTelemetryLevel = Level[telemetryLevel];
+
+  return (event: TelemetryEvent, metadata?: TelemetryMetadata) => {
+    const eventContract = TelemetryContracts[event];
+    if (!eventContract) {
+      throw new Error(`Unknown telemetry event: ${event}`);
+    }
+
+    const eventMinimumTelemetryLevel = eventContract.minimumTelemetryLevel;
+    if (currentTelemetryLevel < eventMinimumTelemetryLevel) {
+      throw new Error(
+        `Telemetry event ${event} requires a minimum telemetry level of ${eventMinimumTelemetryLevel} ` +
+        `but the current level is ${currentTelemetryLevel}`
+      );
+    }
+
+    for (const [key, value] of Object.entries(metadata ?? {})) {
+      if (ALLOWED_METADATA_KEYS.has(key))  { continue; }
+
+      const metadataContract = eventContract.metadataContracts?.[key];
+      if (!metadataContract) {
+        throw new Error(`Unknown metadata for telemetry event ${event}: ${key}`);
+      }
+
+      const metadataMinimumTelemetryLevel = metadataContract.minimumTelemetryLevel;
+      if (metadataMinimumTelemetryLevel && currentTelemetryLevel < metadataMinimumTelemetryLevel) {
+        throw new Error(
+          `Telemetry metadata ${key} of event ${event} requires a minimum telemetry level of ` +
+          `${metadataMinimumTelemetryLevel} but the current level is ${currentTelemetryLevel}`
+        );
+      }
+
+      const {dataType} = metadataContract;
+      if (dataType.endsWith('[]')) {
+        if (!Array.isArray(value)) {
+          throw new Error(
+            `Telemetry metadata ${key} of event ${event} expected a value of type array ` +
+            `but received a value of type ${typeof value}`
+          );
+        }
+
+        const elementDataType = dataType.slice(0, -2);
+        if (value.some(element => typeof element !== elementDataType)) {
+          throw new Error(
+            `Telemetry metadata ${key} of event ${event} expected a value of type ${elementDataType}[] ` +
+            `but received a value of type ${typeof value}[]`
+          );
+        }
+      } else if (dataType === 'date') {
+        if (!(value instanceof Date) && typeof value !== 'string') {
+          throw new Error(
+            `Telemetry metadata ${key} of event ${event} expected a value of type Date or string ` +
+            `but received a value of type ${typeof value}`
+          );
+        }
+      } else if (dataType !== typeof value) {
+        throw new Error(
+          `Telemetry metadata ${key} of event ${event} expected a value of type ${dataType} ` +
+          `but received a value of type ${typeof value}`
+        );
+      }
+    }
+  };
+}
+
+export type TelemetryEventChecker = (event: TelemetryEvent, metadata?: TelemetryMetadata) => void;
+
+/**
+ * Returns a new, filtered metadata object.
+ *
+ * Metadata in groups that don't meet `telemetryLevel` are removed from the
+ * returned object, and the returned object is flattened.
+ *
+ * Returns undefined if `metadata` is undefined.
+ */
+export function filterMetadata(
+  metadata: TelemetryMetadataByLevel | undefined,
+  telemetryLevel: TelemetryLevel
+): TelemetryMetadata | undefined {
+  if (!metadata) { return; }
+
+  let filteredMetadata = {};
+  for (const level of ['limited', 'full'] as const) {
+    if (Level[telemetryLevel] < Level[level]) { break; }
+
+    filteredMetadata = {...filteredMetadata, ...metadata[level]};
+  }
+
+  filteredMetadata = removeNullishKeys(filteredMetadata);
+
+  return removeNullishKeys(filteredMetadata);
+}
+
+/**
+ * Returns a copy of `object` with all null and undefined keys removed.
+ */
+export function removeNullishKeys(object: Record<string, any>) {
+  return pickBy(object, value => value !== null && value !== undefined);
+}

app/common/gristUrls.ts

@@ -4,6 +4,7 @@ import {EngineCode} from 'app/common/DocumentSettings';
 import {encodeQueryParams, isAffirmative} from 'app/common/gutil';
 import {LocalPlugin} from 'app/common/plugin';
 import {StringUnion} from 'app/common/StringUnion';
+import {TelemetryLevel} from 'app/common/Telemetry';
 import {UIRowId} from 'app/common/UIRowId';
 import {getGristConfig} from 'app/common/urlUtils';
 import {Document} from 'app/common/UserAPI';
@@ -627,6 +628,12 @@ export interface GristLoadConfig {
 
   // Current user locale, read from the user options;
   userLocale?: string;
+
+  // Telemetry config.
+  telemetry?: TelemetryConfig;
+
+  // The Grist deployment type (e.g. core, enterprise).
+  deploymentType?: GristDeploymentType;
 }
 
 export const Features = StringUnion(
@@ -648,6 +655,13 @@ export function getPageTitleSuffix(config?: GristLoadConfig) {
   return config?.pageTitleSuffix ?? " - Grist";
 }
 
+export interface TelemetryConfig {
+  telemetryLevel: TelemetryLevel;
+}
+
+export const GristDeploymentTypes = StringUnion('saas', 'core', 'enterprise', 'electron', 'static');
+export type GristDeploymentType = typeof GristDeploymentTypes.type;
+
 /**
  * For a packaged version of Grist that requires activation, this
  * summarizes the current state. Not applicable to grist-core.

app/common/orgNameUtils.ts

@@ -16,7 +16,7 @@ const BLACKLISTED_SUBDOMAINS = new Set([
   'docs', 'api', 'static',
   'ftp', 'imap', 'pop', 'smtp', 'mail', 'git', 'blog', 'wiki', 'support', 'kb', 'help',
   'admin', 'store', 'dev', 'beta',
-  'community', 'try', 'wpx',
+  'community', 'try', 'wpx', 'telemetry',
 
   // a few random tech brands
   'google', 'apple', 'microsoft', 'ms', 'facebook', 'fb', 'twitter', 'youtube', 'yt',

app/gen-server/entity/Document.ts

@@ -164,12 +164,14 @@ export class Document extends Resource {
     const percentComplete = lastSlideIndex !== undefined && numSlides !== undefined
       ? Math.floor((lastSlideIndex / numSlides) * 100)
       : undefined;
-    dbManager?.emit('tutorialProgressChange', {
+    dbManager?.emit('tutorialProgressChanged', {
-      tutorialForkIdDigest: hashId(this.id),
+      full: {
-      tutorialTrunkIdDigest: this.trunkId ? hashId(this.trunkId) : undefined,
+        tutorialForkIdDigest: hashId(this.id),
-      lastSlideIndex,
+        tutorialTrunkIdDigest: this.trunkId ? hashId(this.trunkId) : undefined,
-      numSlides,
+        lastSlideIndex,
-      percentComplete,
+        numSlides,
+        percentComplete,
+      },
     });
   }
 }

app/gen-server/lib/HomeDBManager.ts

@@ -89,13 +89,13 @@ export const NotifierEvents = StringUnion(
 
 export type NotifierEvent = typeof NotifierEvents.type;
 
-export const TelemetryEvents = StringUnion(
+export const HomeDBTelemetryEvents = StringUnion(
-  'tutorialProgressChange',
+  'tutorialProgressChanged',
 );
 
-export type TelemetryEvent = typeof TelemetryEvents.type;
+export type HomeDBTelemetryEvent = typeof HomeDBTelemetryEvents.type;
 
-export type Event = NotifierEvent | TelemetryEvent;
+export type Event = NotifierEvent | HomeDBTelemetryEvent;
 
 // Nominal email address of a user who can view anything (for thumbnails).
 export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com';

app/gen-server/lib/Housekeeper.ts

@@ -37,6 +37,7 @@ export class Housekeeper {
   private _deleteTrashinterval?: NodeJS.Timeout;
   private _logMetricsInterval?: NodeJS.Timeout;
   private _electionKey?: string;
+  private _telemetry = this._server.getTelemetry();
 
   public constructor(private _dbManager: HomeDBManager, private _server: GristServer,
                      private _permitStore: IPermitStore, private _electionStore: IElectionStore) {
@@ -174,30 +175,37 @@ export class Housekeeper {
    */
   public async logMetrics() {
     await this._dbManager.connection.transaction('READ UNCOMMITTED', async (manager) => {
-      const telemetryManager = this._server.getTelemetryManager();
       const usageSummaries = await this._getOrgUsageSummaries(manager);
       for (const summary of usageSummaries) {
-        telemetryManager?.logEvent('siteUsage', {
+        this._telemetry.logEvent('siteUsage', {
-          siteId: summary.site_id,
+          limited: {
-          siteType: summary.site_type,
+            siteId: summary.site_id,
-          inGoodStanding: Boolean(summary.in_good_standing),
+            siteType: summary.site_type,
-          stripePlanId: summary.stripe_plan_id,
+            inGoodStanding: Boolean(summary.in_good_standing),
-          numDocs: Number(summary.num_docs),
+            numDocs: Number(summary.num_docs),
-          numWorkspaces: Number(summary.num_workspaces),
+            numWorkspaces: Number(summary.num_workspaces),
-          numMembers: Number(summary.num_members),
+            numMembers: Number(summary.num_members),
-          lastActivity: summary.last_activity,
+            lastActivity: summary.last_activity,
-        });
+          },
+          full: {
+            stripePlanId: summary.stripe_plan_id,
+          },
+        })
+        .catch(e => log.error('failed to log telemetry event siteUsage', e));
       }
 
       const membershipSummaries = await this._getOrgMembershipSummaries(manager);
       for (const summary of membershipSummaries) {
-        telemetryManager?.logEvent('siteMembership', {
+        this._telemetry.logEvent('siteMembership', {
-          siteId: summary.site_id,
+          limited: {
-          siteType: summary.site_type,
+            siteId: summary.site_id,
-          numOwners: Number(summary.num_owners),
+            siteType: summary.site_type,
-          numEditors: Number(summary.num_editors),
+            numOwners: Number(summary.num_owners),
-          numViewers: Number(summary.num_viewers),
+            numEditors: Number(summary.num_editors),
-        });
+            numViewers: Number(summary.num_viewers),
+          },
+        })
+        .catch(e => log.error('failed to log telemetry event siteMembership', e));
       }
     });
   }

app/server/generateInitialDocSql.ts

@@ -3,6 +3,7 @@ import { create } from 'app/server/lib/create';
 import { DocManager } from 'app/server/lib/DocManager';
 import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
 import { DocStorageManager } from 'app/server/lib/DocStorageManager';
+import { createDummyTelemetry } from 'app/server/lib/GristServer';
 import { PluginManager } from 'app/server/lib/PluginManager';
 
 import * as childProcess from 'child_process';
@@ -33,7 +34,7 @@ export async function main(baseName: string) {
     }
     const docManager = new DocManager(storageManager, pluginManager, null as any, {
       create,
-      getTelemetryManager: () => undefined,
+      getTelemetry() { return createDummyTelemetry(); },
     } as any);
     const activeDoc = new ActiveDoc(docManager, baseName);
     const session = makeExceptionalDocSession('nascent');

app/server/lib/ActiveDoc.ts

@@ -74,7 +74,7 @@ import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord, SingleCell} from 'app/common/TableData';
-import {TelemetryEventName} from 'app/common/Telemetry';
+import {TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
 import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
 import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
@@ -1395,11 +1395,13 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
       // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
       const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
       this.logTelemetryEvent(docSession, 'documentForked', {
-        forkIdDigest: hashId(forkIds.forkId),
+        limited: {
-        forkDocIdDigest: hashId(forkIds.docId),
+          forkIdDigest: hashId(forkIds.forkId),
-        trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
+          forkDocIdDigest: hashId(forkIds.docId),
-        isTemplate,
+          trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
-        lastActivity: doc.updatedAt,
+          isTemplate,
+          lastActivity: doc.updatedAt,
+        },
       });
     } finally {
       await permitStore.removePermit(permitKey);
@@ -1789,13 +1791,14 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
 
   public logTelemetryEvent(
     docSession: OptDocSession | null,
-    eventName: TelemetryEventName,
+    event: TelemetryEvent,
-    metadata?: Record<string, any>
+    metadata?: TelemetryMetadataByLevel
   ) {
-    this._docManager.gristServer.getTelemetryManager()?.logEvent(eventName, {
+    this._docManager.gristServer.getTelemetry().logEvent(event, merge(
-      ...this._getTelemetryMeta(docSession),
+      this._getTelemetryMeta(docSession),
-      ...metadata,
+      metadata,
-    });
+    ))
+    .catch(e => this._log.error(docSession, `failed to log telemetry event ${event}`, e));
   }
 
   /**
@@ -2332,18 +2335,20 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
 
   private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
     this.logTelemetryEvent(docSession, 'documentUsage', {
-      triggeredBy,
+      limited: {
-      isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
+        triggeredBy,
-      rowCount: this._docUsage?.rowCount?.total,
+        isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
-      dataSizeBytes: this._docUsage?.dataSizeBytes,
+        rowCount: this._docUsage?.rowCount?.total,
-      attachmentsSize: this._docUsage?.attachmentsSizeBytes,
+        dataSizeBytes: this._docUsage?.dataSizeBytes,
-      ...this._getAccessRuleMetrics(),
+        attachmentsSize: this._docUsage?.attachmentsSizeBytes,
-      ...this._getAttachmentMetrics(),
+        ...this._getAccessRuleMetrics(),
-      ...this._getChartMetrics(),
+        ...this._getAttachmentMetrics(),
-      ...this._getWidgetMetrics(),
+        ...this._getChartMetrics(),
-      ...this._getColumnMetrics(),
+        ...this._getWidgetMetrics(),
-      ...this._getTableMetrics(),
+        ...this._getColumnMetrics(),
-      ...this._getCustomWidgetMetrics(),
+        ...this._getTableMetrics(),
+        ...this._getCustomWidgetMetrics(),
+      },
     });
   }
 
@@ -2365,10 +2370,10 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
       // Exclude the leading ".", if any.
       .map(r => r.fileExt?.trim()?.slice(1))
       .filter(ext => Boolean(ext));
-
+    const uniqueAttachmentTypes = [...new Set(attachmentTypes ?? [])];
     return {
       numAttachments,
-      attachmentTypes,
+      attachmentTypes: uniqueAttachmentTypes,
     };
   }
 
@@ -2528,15 +2533,19 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
     this._logDocMetrics(docSession, 'docOpen');
   }
 
-  private _getTelemetryMeta(docSession: OptDocSession|null) {
+  private _getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel {
     const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined;
     return merge(
       docSession ? getTelemetryMetaFromDocSession(docSession) : {},
-      altSessionId ? {altSessionId} : undefined,
+      altSessionId ? {altSessionId} : {},
       {
-        docIdDigest: hashId(this._docName),
+        limited: {
-        siteId: this._doc?.workspace.org.id,
+          docIdDigest: hashId(this._docName),
-        siteType: this._product?.name,
+        },
+        full: {
+          siteId: this._doc?.workspace.org.id,
+          siteType: this._product?.name,
+        },
       },
     );
   }

app/server/lib/AppEndpoint.ts

@@ -11,7 +11,7 @@ import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gr
 import {removeTrailingSlash} from 'app/common/gutil';
 import {hashId} from 'app/common/hashingUtils';
 import {LocalPlugin} from "app/common/plugin";
-import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
+import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
 import {Document as APIDocument} from 'app/common/UserAPI';
 import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
@@ -308,18 +308,23 @@ export function attachAppEndpoint(options: AttachOptions): void {
     // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
     const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
     if (isPublic || isTemplate) {
-      gristServer.getTelemetryManager()?.logEvent('documentOpened', {
+      gristServer.getTelemetry().logEvent('documentOpened', {
-        docIdDigest: hashId(docId),
+        limited: {
-        siteId: doc.workspace.org.id,
+          docIdDigest: hashId(docId),
-        siteType: doc.workspace.org.billingAccount.product.name,
+          access: doc.access,
-        userId: mreq.userId,
+          isPublic,
-        altSessionId: mreq.altSessionId,
+          isSnapshot,
-        access: doc.access,
+          isTemplate,
-        isPublic,
+          lastUpdated: doc.updatedAt,
-        isSnapshot,
+        },
-        isTemplate,
+        full: {
-        lastUpdated: doc.updatedAt,
+          siteId: doc.workspace.org.id,
-      });
+          siteType: doc.workspace.org.billingAccount.product.name,
+          userId: mreq.userId,
+          altSessionId: mreq.altSessionId,
+        },
+      })
+      .catch(e => log.error('failed to log telemetry event documentOpened', e));
     }
 
     if (isTemplate) {
@@ -330,7 +335,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
         isAnonymous: isAnonymousUser(mreq),
         templateId: docId,
       };
-      res.cookie(TelemetryTemplateSignupCookieName, JSON.stringify(value), {
+      res.cookie(TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME, JSON.stringify(value), {
         maxAge: 1000 * 60 * 60,
         httpOnly: true,
         path: '/',

app/server/lib/Authorizer.ts

@@ -396,11 +396,14 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
   };
   log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
   if (hasApiKey) {
-    options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
+    options.gristServer.getTelemetry().logEvent('apiUsage', {
-      method: mreq.method,
+      full: {
-      userId: mreq.userId,
+        method: mreq.method,
-      userAgent: req.headers['user-agent'],
+        userId: mreq.userId,
-    });
+        userAgent: mreq.headers['user-agent'],
+      },
+    })
+    .catch(e => log.error('failed to log telemetry event apiUsage', e));
   }
 
   return next();

app/server/lib/Client.ts

@@ -4,6 +4,7 @@ import {delay} from 'app/common/delay';
 import {CommClientConnect, CommMessage, CommResponse, CommResponseError} from 'app/common/CommTypes';
 import {ErrorWithCode} from 'app/common/ErrorWithCode';
 import {UserProfile} from 'app/common/LoginSessionAPI';
+import {TelemetryMetadata} from 'app/common/Telemetry';
 import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI';
 import {User} from 'app/gen-server/entity/User';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
@@ -433,8 +434,8 @@ export class Client {
     return meta;
   }
 
-  public getFullTelemetryMeta() {
+  public getFullTelemetryMeta(): TelemetryMetadata {
-    const meta: Record<string, any> = {};
+    const meta: TelemetryMetadata = {};
     // We assume the _userId has already been cached, which will be true always (for all practical
     // purposes) because it's set when the Authorizer checks this client.
     if (this._userId) { meta.userId = this._userId; }

app/server/lib/DocApi.ts

@@ -916,8 +916,10 @@ export class DocWorkerApi {
           });
           const {forkId} = parseUrlId(scope.urlId);
           activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
-            tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
+            full: {
-            tutorialTrunkIdDigest: hashId(tutorialTrunkId),
+              tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
+              tutorialTrunkIdDigest: hashId(tutorialTrunkId),
+            },
           });
         }
       }

app/server/lib/FlexServer.ts

@@ -1,12 +1,12 @@
 import {ApiError} from 'app/common/ApiError';
 import {delay} from 'app/common/delay';
 import {DocCreationInfo} from 'app/common/DocListAPI';
-import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPathOnly,
+import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
-        parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
+        GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
+        sanitizePathTail} from 'app/common/gristUrls';
 import {getOrgUrlInfo} from 'app/common/gristUrls';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {tbind} from 'app/common/tbind';
-import {TelemetryEventName, TelemetryEventNames} from 'app/common/Telemetry';
 import * as version from 'app/common/version';
 import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
@@ -57,7 +57,7 @@ import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
-import {TelemetryManager} from 'app/server/lib/TelemetryManager';
+import {ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
@@ -117,7 +117,9 @@ export class FlexServer implements GristServer {
   public electronServerMethods: ElectronServerMethods;
   public readonly docsRoot: string;
   public readonly i18Instance: i18n;
+  private _activations: Activations;
   private _comm: Comm;
+  private _deploymentType: GristDeploymentType;
   private _dbManager: HomeDBManager;
   private _defaultBaseDomain: string|undefined;
   private _pluginUrl: string|undefined;
@@ -130,7 +132,7 @@ export class FlexServer implements GristServer {
   private _sessions: Sessions;
   private _sessionStore: SessionStore;
   private _storageManager: IDocStorageManager;
-  private _telemetryManager: TelemetryManager|undefined;
+  private _telemetry: ITelemetry;
   private _processMonitorStop?: () => void;    // Callback to stop the ProcessMonitor
   private _docWorkerMap: IDocWorkerMap;
   private _widgetRepository: IWidgetRepository;
@@ -199,6 +201,11 @@ export class FlexServer implements GristServer {
     this.docsRoot = fse.realpathSync(docsRoot);
     this.info.push(['docsRoot', this.docsRoot]);
 
+    this._deploymentType = this.create.deploymentType();
+    if (process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE) {
+      this._deploymentType = GristDeploymentTypes.check(process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE);
+    }
+
     const homeUrl = process.env.APP_HOME_URL;
     // The "base domain" is only a thing if orgs are encoded as a subdomain.
     if (process.env.GRIST_ORG_IN_PATH === 'true' || process.env.GRIST_SINGLE_ORG) {
@@ -328,11 +335,20 @@ export class FlexServer implements GristServer {
     return this._comm;
   }
 
+  public getDeploymentType(): GristDeploymentType {
+    return this._deploymentType;
+  }
+
   public getHosts(): Hosts {
     if (!this._hosts) { throw new Error('no hosts available'); }
     return this._hosts;
   }
 
+  public getActivations(): Activations {
+    if (!this._activations) { throw new Error('no activations available'); }
+    return this._activations;
+  }
+
   public getHomeDBManager(): HomeDBManager {
     if (!this._dbManager) { throw new Error('no home db available'); }
     return this._dbManager;
@@ -343,8 +359,9 @@ export class FlexServer implements GristServer {
     return this._storageManager;
   }
 
-  public getTelemetryManager(): TelemetryManager|undefined {
+  public getTelemetry(): ITelemetry {
-    return this._telemetryManager;
+    if (!this._telemetry) { throw new Error('no telemetry available'); }
+    return this._telemetry;
   }
 
   public getWidgetRepository(): IWidgetRepository {
@@ -553,8 +570,8 @@ export class FlexServer implements GristServer {
     // Report which database we are using, without sensitive credentials.
     this.info.push(['database', getDatabaseUrl(this._dbManager.connection.options, false)]);
     // If the installation appears to be new, give it an id and a creation date.
-    const activations = new Activations(this._dbManager);
+    this._activations = new Activations(this._dbManager);
-    await activations.current();
+    await this._activations.current();
   }
 
   public addDocWorkerMap() {
@@ -689,24 +706,14 @@ export class FlexServer implements GristServer {
     });
   }
 
-  public addTelemetryEndpoint() {
+  public addTelemetry() {
-    if (this._check('telemetry-endpoint', 'json', 'api-mw', 'homedb')) { return; }
+    if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }
 
-    this._telemetryManager = new TelemetryManager(this._dbManager);
+    this._telemetry = this.create.Telemetry(this._dbManager, this);
+    this._telemetry.addEndpoints(this.app);
 
     // Start up a monitor for memory and cpu usage.
-    this._processMonitorStop = ProcessMonitor.start(this._telemetryManager);
+    this._processMonitorStop = ProcessMonitor.start(this._telemetry);
-
-    this.app.post('/api/telemetry', async (req, resp) => {
-      const mreq = req as RequestWithLogin;
-      const name = stringParam(req.body.name, 'name', TelemetryEventNames);
-      this._telemetryManager?.logEvent(name as TelemetryEventName, {
-        userId: mreq.userId,
-        altSessionId: mreq.altSessionId,
-        ...req.body.metadata,
-      });
-      return resp.status(200).send();
-    });
   }
 
   public async close() {
@@ -828,7 +835,7 @@ export class FlexServer implements GristServer {
 
     // Initialize _sendAppPage helper.
     this._sendAppPage = makeSendAppPage({
-      server: isSingleUserMode() ? null : this,
+      server: this,
       staticDir: getAppPathTo(this.appRoot, 'static'),
       tag: this.tag,
       testLogin: allowTestLogin(),
@@ -1108,7 +1115,7 @@ export class FlexServer implements GristServer {
   // Add document-related endpoints and related support.
   public async addDoc() {
     this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ?
-      null : 'homedb', 'api-mw', 'map', 'telemetry-endpoint');
+      null : 'homedb', 'api-mw', 'map', 'telemetry');
     // add handlers for cleanup, if we are in charge of the doc manager.
     if (!this._docManager) { this.addCleanup(); }
     await this.loadConfig();
@@ -1368,7 +1375,11 @@ export class FlexServer implements GristServer {
   }
 
   public getGristConfig(): GristLoadConfig {
-    return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
+    return makeGristConfig({
+      homeUrl: this.getDefaultHomeUrl(),
+      extra: {},
+      baseDomain: this._defaultBaseDomain,
+    });
   }
 
   /**

app/server/lib/GristServer.ts

@@ -1,8 +1,9 @@
-import { GristLoadConfig } from 'app/common/gristUrls';
+import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
 import { FullUser, UserProfile } from 'app/common/UserAPI';
 import { Document } from 'app/gen-server/entity/Document';
 import { Organization } from 'app/gen-server/entity/Organization';
 import { Workspace } from 'app/gen-server/entity/Workspace';
+import { Activations } from 'app/gen-server/lib/Activations';
 import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { IAccessTokens } from 'app/server/lib/AccessTokens';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
@@ -16,7 +17,7 @@ import { IPermitStore } from 'app/server/lib/Permit';
 import { ISendAppPageOptions } from 'app/server/lib/sendAppPage';
 import { fromCallback } from 'app/server/lib/serverUtils';
 import { Sessions } from 'app/server/lib/Sessions';
-import { TelemetryManager } from 'app/server/lib/TelemetryManager';
+import { ITelemetry } from 'app/server/lib/Telemetry';
 import * as express from 'express';
 import { IncomingMessage } from 'http';
 
@@ -40,10 +41,12 @@ export interface GristServer {
   getExternalPermitStore(): IPermitStore;
   getSessions(): Sessions;
   getComm(): Comm;
+  getDeploymentType(): GristDeploymentType;
   getHosts(): Hosts;
+  getActivations(): Activations;
   getHomeDBManager(): HomeDBManager;
   getStorageManager(): IDocStorageManager;
-  getTelemetryManager(): TelemetryManager|undefined;
+  getTelemetry(): ITelemetry;
   getNotifier(): INotifier;
   getDocTemplate(): Promise<DocTemplate>;
   getTag(): string;
@@ -117,10 +120,12 @@ export function createDummyGristServer(): GristServer {
     getResourceUrl() { return Promise.resolve(''); },
     getSessions() { throw new Error('no sessions'); },
     getComm() { throw new Error('no comms'); },
+    getDeploymentType() { return 'core'; },
     getHosts() { throw new Error('no hosts'); },
+    getActivations() { throw new Error('no activations'); },
     getHomeDBManager() { throw new Error('no db'); },
     getStorageManager() { throw new Error('no storage manager'); },
-    getTelemetryManager() { return undefined; },
+    getTelemetry() { return createDummyTelemetry(); },
     getNotifier() { throw new Error('no notifier'); },
     getDocTemplate() { throw new Error('no doc template'); },
     getTag() { return 'tag'; },
@@ -128,3 +133,11 @@ export function createDummyGristServer(): GristServer {
     getAccessTokens() { throw new Error('no access tokens'); },
   };
 }
+
+export function createDummyTelemetry(): ITelemetry {
+  return {
+    logEvent() { return Promise.resolve(); },
+    addEndpoints() { /* do nothing */ },
+    getTelemetryLevel() { return 'off'; },
+  };
+}

app/server/lib/ICreate.ts

@@ -1,19 +1,22 @@
+import {GristDeploymentType} from 'app/common/gristUrls';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {ExternalStorage} from 'app/server/lib/ExternalStorage';
-import {GristServer} from 'app/server/lib/GristServer';
+import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
 import {INotifier} from 'app/server/lib/INotifier';
 import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox';
 import {IShell} from 'app/server/lib/IShell';
 import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
+import {ITelemetry} from 'app/server/lib/Telemetry';
 
 export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
+  Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
   Shell?(): IShell;  // relevant to electron version of Grist only.
 
   // Create a space to store files externally, for storing either:
@@ -25,6 +28,7 @@ export interface ICreate {
 
   NSandbox(options: ISandboxCreationOptions): ISandbox;
 
+  deploymentType(): GristDeploymentType;
   sessionSecret(): string;
   // Check configuration of the app early enough to show on startup.
   configure?(): Promise<void>;
@@ -57,19 +61,26 @@ export interface ICreateBillingOptions {
   create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined;
 }
 
+export interface ICreateTelemetryOptions {
+  create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
+}
+
 export function makeSimpleCreator(opts: {
+  deploymentType: GristDeploymentType,
   sessionSecret?: string,
   storage?: ICreateStorageOptions[],
   billing?: ICreateBillingOptions,
   notifier?: ICreateNotifierOptions,
+  telemetry?: ICreateTelemetryOptions,
   sandboxFlavor?: string,
   shell?: IShell,
   getExtraHeadHtml?: () => string,
   getSqliteVariant?: () => SqliteVariant,
   getSandboxVariants?: () => Record<string, SpawnFn>,
 }): ICreate {
-  const {sessionSecret, storage, notifier, billing} = opts;
+  const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts;
   return {
+    deploymentType() { return deploymentType; },
     Billing(dbManager, gristConfig) {
       return billing?.create(dbManager, gristConfig) ?? {
         addEndpoints() { /* do nothing */ },
@@ -93,6 +104,9 @@ export function makeSimpleCreator(opts: {
       }
       return undefined;
     },
+    Telemetry(dbManager, gristConfig) {
+      return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();
+    },
     NSandbox(options) {
       return createSandbox(opts.sandboxFlavor || 'unsandboxed', options);
     },

app/server/lib/ProcessMonitor.ts

@@ -1,4 +1,5 @@
-import { TelemetryManager } from 'app/server/lib/TelemetryManager';
+import log from 'app/server/lib/log';
+import { ITelemetry } from 'app/server/lib/Telemetry';
 
 const MONITOR_PERIOD_MS = 5_000;        // take a look at memory usage this often
 const MEMORY_DELTA_FRACTION = 0.1;      // fraction by which usage should change to get reported
@@ -16,7 +17,7 @@ let _lastReportedCpuAverage: number = 0;
  * Monitor process memory (heap) and CPU usage, reporting as telemetry on an interval, and more
  * often when usage ticks up or down by a big enough delta.
  *
- * There is a single global process monitor, reporting to the telemetryManager passed into the
+ * There is a single global process monitor, reporting to the `telemetry` object passed into the
  * first call to start().
  *
  * Returns a function that stops the monitor, or null if there was already a process monitor
@@ -30,12 +31,12 @@ let _lastReportedCpuAverage: number = 0;
  *  - intervalMs:   Interval (in milliseconds) over which cpuAverage is reported. Being much
  *                  higher than MONITOR_PERIOD_MS is a sign of being CPU bound for that long.
  */
-export function start(telemetryManager: TelemetryManager): (() => void) | undefined {
+export function start(telemetry: ITelemetry): (() => void) | undefined {
   if (!_timer) {
     // Initialize variables needed for accurate first-tick measurement.
     _lastTickTime = Date.now();
     _lastCpuUsage = process.cpuUsage();
-    _timer = setInterval(() => monitor(telemetryManager), MONITOR_PERIOD_MS);
+    _timer = setInterval(() => monitor(telemetry), MONITOR_PERIOD_MS);
 
     return function stop() {
       clearInterval(_timer);
@@ -44,7 +45,7 @@ export function start(telemetryManager: TelemetryManager): (() => void) | undefi
   }
 }
 
-function monitor(telemetryManager: TelemetryManager) {
+function monitor(telemetry: ITelemetry) {
   const memoryUsage = process.memoryUsage();
   const heapUsed = memoryUsage.heapUsed;
   const cpuUsage = process.cpuUsage();
@@ -66,12 +67,15 @@ function monitor(telemetryManager: TelemetryManager) {
     Math.abs(heapUsed - _lastReportedHeapUsed) > _lastReportedHeapUsed * MEMORY_DELTA_FRACTION ||
     Math.abs(cpuAverage - _lastReportedCpuAverage) > CPU_DELTA_FRACTION
   ) {
-    telemetryManager.logEvent('processMonitor', {
+    telemetry.logEvent('processMonitor', {
-      heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
+      full: {
-      heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
+        heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
-      cpuAverage: Math.round(cpuAverage * 100) / 100,
+        heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
-      intervalMs,
+        cpuAverage: Math.round(cpuAverage * 100) / 100,
-    });
+        intervalMs,
+      },
+    })
+    .catch(e => log.error('failed to log telemetry event processMonitor', e));
     _lastReportedHeapUsed = heapUsed;
     _lastReportedCpuAverage = cpuAverage;
     _lastReportTime = now;

app/server/lib/Telemetry.ts

@@ -0,0 +1,210 @@
+import {ApiError} from 'app/common/ApiError';
+import {
+  buildTelemetryEventChecker,
+  filterMetadata,
+  removeNullishKeys,
+  TelemetryEvent,
+  TelemetryEventChecker,
+  TelemetryEvents,
+  TelemetryLevel,
+  TelemetryLevels,
+  TelemetryMetadata,
+  TelemetryMetadataByLevel,
+} from 'app/common/Telemetry';
+import {HomeDBManager, HomeDBTelemetryEvents} from 'app/gen-server/lib/HomeDBManager';
+import {RequestWithLogin} from 'app/server/lib/Authorizer';
+import {GristServer} from 'app/server/lib/GristServer';
+import {LogMethods} from 'app/server/lib/LogMethods';
+import {stringParam} from 'app/server/lib/requestUtils';
+import * as express from 'express';
+import merge = require('lodash/merge');
+
+export interface ITelemetry {
+  logEvent(name: TelemetryEvent, metadata?: TelemetryMetadataByLevel): Promise<void>;
+  addEndpoints(app: express.Express): void;
+  getTelemetryLevel(): TelemetryLevel;
+}
+
+/**
+ * Manages telemetry for Grist.
+ */
+export class Telemetry implements ITelemetry {
+  private _telemetryLevel: TelemetryLevel;
+  private _deploymentType = this._gristServer.getDeploymentType();
+  private _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
+  private _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
+  'https://telemetry.getgrist.com/api/telemetry';
+
+  private _installationId: string | undefined;
+
+  private _errorLogger = new LogMethods('Telemetry ', () => ({}));
+  private _telemetryLogger = new LogMethods('Telemetry ', () => ({
+    eventType: 'telemetry',
+  }));
+
+  private _checkEvent: TelemetryEventChecker | undefined;
+
+  constructor(private _dbManager: HomeDBManager, private _gristServer: GristServer) {
+    this._initialize().catch((e) => {
+      this._errorLogger.error(undefined, 'failed to initialize', e);
+    });
+  }
+
+  /**
+   * Logs a telemetry `event` and its `metadata`.
+   *
+   * Depending on the deployment type, this will either forward the
+   * data to an endpoint (set via GRIST_TELEMETRY_URL) or log it
+   * directly. In hosted Grist, telemetry is logged directly, and
+   * subsequently sent to an OpenSearch instance via CloudWatch. In
+   * other deployment types, telemetry is forwarded to an endpoint
+   * of hosted Grist, which then handles logging to OpenSearch.
+   *
+   * Note that `metadata` is grouped by telemetry level, with only the
+   * groups meeting the current telemetry level being included in
+   * what's logged. If the current telemetry level is `off`, nothing
+   * will be logged. Otherwise, `metadata` will be filtered according
+   * to the current telemetry level, keeping only the groups that are
+   * less than or equal to the current level.
+   *
+   * Additionally, runtime checks are also performed to verify that the
+   * event and metadata being passed in are being logged appropriately
+   * for the configured telemetry level. If any checks fail, an error
+   * is thrown.
+   *
+   * Example:
+   *
+   * The following will only log the `rowCount` if the telemetry level is set
+   * to `limited`, and will log both the `method` and `userId` if the telemetry
+   * level is set to `full`:
+   *
+   * ```
+   * logEvent('documentUsage', {
+   *   limited: {
+   *     rowCount: 123,
+   *   },
+   *   full: {
+   *     userId: 1586,
+   *   },
+   * });
+   * ```
+   */
+  public async logEvent(
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadataByLevel
+  ) {
+    if (this._telemetryLevel === 'off') { return; }
+
+    metadata = filterMetadata(metadata, this._telemetryLevel);
+    this._checkTelemetryEvent(event, metadata);
+
+    if (this._shouldForwardTelemetryEvents) {
+      await this.forwardEvent(event, metadata);
+    } else {
+      this._telemetryLogger.rawLog('info', null, event, {
+        eventName: event,
+        eventSource: `grist-${this._deploymentType}`,
+        ...metadata,
+      });
+    }
+  }
+
+  /**
+   * Forwards a telemetry event and its metadata to another server.
+   */
+  public async forwardEvent(
+    event: TelemetryEvent,
+    metadata?: TelemetryMetadata
+  ) {
+    try {
+      await fetch(this._forwardTelemetryEventsUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          event,
+          metadata,
+        }),
+      });
+    } catch (e) {
+      this._errorLogger.error(undefined, `failed to forward telemetry event ${event}`, e);
+    }
+  }
+
+  public addEndpoints(app: express.Application) {
+    /**
+     * Logs telemetry events and their metadata.
+     *
+     * Clients of this endpoint may be external Grist instances, so the behavior
+     * varies based on the presence of an `eventSource` key in the event metadata.
+     *
+     * If an `eventSource` key is present, the telemetry event will be logged
+     * directly, as the request originated from an external source; runtime checks
+     * of telemetry data are skipped since they should have already occured at the
+     * source. Otherwise, the event will only be logged after passing various
+     * checks.
+     */
+    app.post('/api/telemetry', async (req, resp) => {
+      const mreq = req as RequestWithLogin;
+      const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
+      if ('eventSource' in req.body.metadata) {
+        this._telemetryLogger.rawLog('info', null, event, {
+          eventName: event,
+          ...(removeNullishKeys(req.body.metadata)),
+        });
+      } else {
+        try {
+          await this.logEvent(event as TelemetryEvent, merge(
+            {
+              limited: {
+                eventSource: `grist-${this._deploymentType}`,
+                ...(this._deploymentType !== 'saas' ? {installationId: this._installationId} : {}),
+              },
+              full: {
+                userId: mreq.userId,
+                altSessionId: mreq.altSessionId,
+              },
+            },
+            req.body.metadata,
+          ));
+        } catch (e) {
+          this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e);
+          throw new ApiError(`Telemetry failed to log telemetry event ${event}`, 500);
+        }
+      }
+      return resp.status(200).send();
+    });
+  }
+
+  public getTelemetryLevel() {
+    return this._telemetryLevel;
+  }
+
+  private async _initialize() {
+    if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
+      this._telemetryLevel = TelemetryLevels.check(process.env.GRIST_TELEMETRY_LEVEL);
+      this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryLevel);
+    } else {
+      this._telemetryLevel = 'off';
+    }
+
+    const {id} = await this._gristServer.getActivations().current();
+    this._installationId = id;
+
+    for (const event of HomeDBTelemetryEvents.values) {
+      this._dbManager.on(event, async (metadata) => {
+        this.logEvent(event, metadata).catch(e =>
+          this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e));
+      });
+    }
+  }
+
+  private _checkTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadata) {
+    if (!this._checkEvent) {
+      throw new Error('Telemetry._checkEvent is undefined');
+    }
+
+    this._checkEvent(event, metadata);
+  }
+}

app/server/lib/Triggers.ts

@@ -630,7 +630,7 @@ export class DocTriggers {
         meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
         this._log("Sending batch of webhook events", meta);
         this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
-          numEvents: batch.length,
+          limited: {numEvents: meta.numEvents},
         });
         success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
         if (this._loopAbort.signal.aborted) {

app/server/lib/sendAppPage.ts

@@ -3,7 +3,7 @@ import {isAffirmative} from 'app/common/gutil';
 import {getTagManagerSnippet} from 'app/common/tagManager';
 import {Document} from 'app/common/UserAPI';
 import {SUPPORT_EMAIL} from 'app/gen-server/lib/HomeDBManager';
-import {isAnonymousUser, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {isAnonymousUser, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {GristServer} from 'app/server/lib/GristServer';
 import {getSupportedEngineChoices} from 'app/server/lib/serverUtils';
@@ -31,9 +31,16 @@ export interface ISendAppPageOptions {
   googleTagManager?: true | false | 'anon';
 }
 
-export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadConfig>,
+export interface MakeGristConfigOptons {
-                                baseDomain?: string, req?: express.Request
+  homeUrl: string|null;
-): GristLoadConfig {
+  extra: Partial<GristLoadConfig>;
+  baseDomain?: string;
+  req?: express.Request;
+  server?: GristServer|null;
+}
+
+export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
+  const {homeUrl, extra, baseDomain, req, server} = options;
   // .invalid is a TLD the IETF promises will never exist.
   const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
   const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") ||
@@ -69,6 +76,8 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
     featureFormulaAssistant: isAffirmative(process.env.GRIST_FORMULA_ASSISTANT),
     supportEmail: SUPPORT_EMAIL,
     userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
+    telemetry: server ? getTelemetryConfig(server) : undefined,
+    deploymentType: server?.getDeploymentType(),
     ...extra,
   };
 }
@@ -94,14 +103,18 @@ export function makeMessagePage(staticDir: string) {
  * placeholders replaced.
  */
 export function makeSendAppPage(opts: {
-  server: GristServer|null, staticDir: string, tag: string, testLogin?: boolean,
+  server: GristServer, staticDir: string, tag: string, testLogin?: boolean,
   baseDomain?: string
 }) {
   const {server, staticDir, tag, testLogin} = opts;
   return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
-    // .invalid is a TLD the IETF promises will never exist.
+      const config = makeGristConfig({
-    const config = makeGristConfig(server ? server.getHomeUrl(req) : null, options.config,
+        homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
-      opts.baseDomain, req);
+        extra: options.config,
+        baseDomain: opts.baseDomain,
+        req,
+        server,
+      });
 
     // We could cache file contents in memory, but the filesystem does caching too, and compared
     // to that, the performance gain is unlikely to be meaningful. So keep it simple here.
@@ -112,7 +125,7 @@ export function makeSendAppPage(opts: {
     const tagManagerSnippet = needTagManager ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) : '';
     const staticOrigin = process.env.APP_STATIC_URL || "";
     const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
-    const customHeadHtmlSnippet = server?.create.getExtraHeadHtml?.() ?? "";
+    const customHeadHtmlSnippet = server.create.getExtraHeadHtml?.() ?? "";
     const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
     // Preload all languages that will be used or are requested by client.
     const preloads = req.languages
@@ -127,7 +140,7 @@ export function makeSendAppPage(opts: {
       .replace("<!-- INSERT WARNING -->", warning)
       .replace("<!-- INSERT TITLE -->", getPageTitle(req, config))
       .replace("<!-- INSERT META -->", getPageMetadataHtmlSnippet(config))
-      .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server?.getGristConfig()))
+      .replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server.getGristConfig()))
       .replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet)
       .replace("<!-- INSERT LOCALE -->", preloads)
       .replace("<!-- INSERT CUSTOM -->", customHeadHtmlSnippet)
@@ -150,6 +163,13 @@ function getFeatures(): IFeature[] {
   return Features.checkAll(difference(enabledFeatures, disabledFeatures));
 }
 
+function getTelemetryConfig(server: GristServer) {
+  const telemetry = server.getTelemetry();
+  return {
+    telemetryLevel: telemetry.getTelemetryLevel(),
+  };
+}
+
 function configuredPageTitleSuffix() {
   const result = process.env.GRIST_PAGE_TITLE_SUFFIX;
   return result === "_blank" ? "" : result;

app/server/lib/serverUtils.ts

@@ -7,6 +7,7 @@ import uuidv4 from 'uuid/v4';
 import {AbortSignal} from 'node-abort-controller';
 
 import {EngineCode} from 'app/common/DocumentSettings';
+import {TelemetryMetadataByLevel} from 'app/common/Telemetry';
 import log from 'app/server/lib/log';
 import {OpenMode, SQLiteDB} from 'app/server/lib/SQLiteDB';
 import {getDocSessionAccessOrNull, getDocSessionUser, OptDocSession} from './DocSession';
@@ -166,14 +167,18 @@ export function getLogMetaFromDocSession(docSession: OptDocSession) {
 /**
  * Extract telemetry metadata from session.
  */
-export function getTelemetryMetaFromDocSession(docSession: OptDocSession) {
+export function getTelemetryMetaFromDocSession(docSession: OptDocSession): TelemetryMetadataByLevel {
   const client = docSession.client;
   const access = getDocSessionAccessOrNull(docSession);
   const user = getDocSessionUser(docSession);
   return {
-    access,
+    limited: {
-    ...(user ? {userId: user.id} : {}),
+      access,
-    ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+    },
+    full: {
+      ...(user ? {userId: user.id} : {}),
+      ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+    },
   };
 }
 

app/server/mergedServerMain.ts

@@ -119,19 +119,19 @@ export async function main(port: number, serverTypes: ServerType[],
     server.addHomeApi();
     server.addBillingApi();
     server.addNotifier();
+    server.addTelemetry();
     await server.addHousekeeper();
     await server.addLoginRoutes();
     server.addAccountPage();
     server.addBillingPages();
     server.addWelcomePaths();
     server.addLogEndpoint();
-    server.addTelemetryEndpoint();
     server.addGoogleAuthEndpoint();
   }
 
   if (includeDocs) {
     server.addJsonSupport();
-    server.addTelemetryEndpoint();
+    server.addTelemetry();
     await server.addDoc();
   }
 

stubs/app/client/ui/BillingButtons.ts

@@ -1,16 +0,0 @@
-import {AppModel} from 'app/client/models/AppModel';
-import {DomElementArg} from 'grainjs';
-
-export function buildUserMenuBillingItem(
-  _appModel: AppModel,
-  ..._args: DomElementArg[]
-) {
-  return null;
-}
-
-export function buildAppMenuBillingItem(
-  _appModel: AppModel,
-  ..._args: DomElementArg[]
-) {
-  return null;
-}

stubs/app/client/ui/WelcomeCoachingCall.ts

@@ -1 +0,0 @@
-export * from 'app/client/ui/WelcomeCoachingCallStub';

stubs/app/server/lib/TelemetryManager.ts

@@ -1,11 +0,0 @@
-import {TelemetryEventName} from 'app/common/Telemetry';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
-
-export class TelemetryManager {
-  constructor(_dbManager: HomeDBManager) {}
-
-  public logEvent(
-    _name: TelemetryEventName,
-    _metadata?: Record<string, any>
-  ) {}
-}

stubs/app/server/lib/create.ts

@@ -1,8 +1,10 @@
 import { checkMinIOExternalStorage,
          configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage';
 import { makeSimpleCreator } from 'app/server/lib/ICreate';
+import { Telemetry } from 'app/server/lib/Telemetry';
 
 export const create = makeSimpleCreator({
+  deploymentType: 'core',
   // This can and should be overridden by GRIST_SESSION_SECRET
   // (or generated randomly per install, like grist-omnibus does).
   sessionSecret: 'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh',
@@ -13,4 +15,7 @@ export const create = makeSimpleCreator({
       create: configureMinIOExternalStorage,
     },
   ],
+  telemetry: {
+    create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer),
+  }
 });

test/common/Telemetry.ts

@@ -0,0 +1,215 @@
+import {buildTelemetryEventChecker, filterMetadata, TelemetryEvent} from 'app/common/Telemetry';
+import {assert} from 'chai';
+
+describe('Telemetry', function() {
+  describe('buildTelemetryEventChecker', function() {
+    it('returns a function that checks telemetry data', function() {
+      assert.isFunction(buildTelemetryEventChecker('full'));
+    });
+
+    it('does not throw if event and metadata are valid', function() {
+      const checker = buildTelemetryEventChecker('full');
+      assert.doesNotThrow(() => checker('apiUsage', {
+        method: 'GET',
+        userId: 1,
+        userAgent: 'node-fetch/1.0',
+      }));
+      assert.doesNotThrow(() => checker('siteUsage', {
+        siteId: 1,
+        siteType: 'team',
+        inGoodStanding: true,
+        stripePlanId: 'stripePlanId',
+        numDocs: 1,
+        numWorkspaces: 1,
+        numMembers: 1,
+        lastActivity: new Date('2022-12-30T01:23:45'),
+      }));
+      assert.doesNotThrow(() => checker('watchedVideoTour', {
+        watchTimeSeconds: 30,
+        userId: 1,
+        altSessionId: 'altSessionId',
+      }));
+    });
+
+    it("does not throw when metadata is a subset of what's expected", function() {
+      const checker = buildTelemetryEventChecker('full');
+      assert.doesNotThrow(() => checker('documentUsage', {
+        docIdDigest: 'docIdDigest',
+        siteId: 1,
+        rowCount: 123,
+        attachmentTypes: ['pdf'],
+      }));
+    });
+
+    it('does not throw if all metadata is less than or equal to the expected telemetry level', function() {
+      const checker = buildTelemetryEventChecker('limited');
+      assert.doesNotThrow(() => checker('documentUsage', {
+        rowCount: 123,
+      }));
+      assert.doesNotThrow(() => checker('siteUsage', {
+        siteId: 1,
+        siteType: 'team',
+        inGoodStanding: true,
+        numDocs: 1,
+        numWorkspaces: 1,
+        numMembers: 1,
+        lastActivity: new Date('2022-12-30T01:23:45'),
+      }));
+      assert.doesNotThrow(() => checker('watchedVideoTour', {
+        watchTimeSeconds: 30,
+      }));
+    });
+
+    it('throws if event is invalid', function() {
+      const checker = buildTelemetryEventChecker('full');
+      assert.throws(
+        () => checker('invalidEvent' as TelemetryEvent, {}),
+        /Unknown telemetry event: invalidEvent/
+      );
+    });
+
+    it('throws if metadata is invalid', function() {
+      const checker = buildTelemetryEventChecker('full');
+      assert.throws(
+        () => checker('apiUsage', {invalidMetadata: '123'}),
+        /Unknown metadata for telemetry event apiUsage: invalidMetadata/
+      );
+    });
+
+    it('throws if metadata types do not match expected types', function() {
+      const checker = buildTelemetryEventChecker('full');
+      assert.throws(
+        () => checker('siteUsage', {siteId: '1'}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata siteId of event siteUsage expected a value of type number but received a value of type string/
+      );
+      assert.throws(
+        () => checker('siteUsage', {lastActivity: 1234567890}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata lastActivity of event siteUsage expected a value of type Date or string but received a value of type number/
+      );
+      assert.throws(
+        () => checker('siteUsage', {inGoodStanding: 'true'}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata inGoodStanding of event siteUsage expected a value of type boolean but received a value of type string/
+      );
+      assert.throws(
+        () => checker('siteUsage', {numDocs: '1'}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata numDocs of event siteUsage expected a value of type number but received a value of type string/
+      );
+      assert.throws(
+        () => checker('documentUsage', {attachmentTypes: '1,2,3'}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata attachmentTypes of event documentUsage expected a value of type array but received a value of type string/
+      );
+      assert.throws(
+        () => checker('documentUsage', {attachmentTypes: ['.txt', 1, true]}),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata attachmentTypes of event documentUsage expected a value of type string\[\] but received a value of type object\[\]/
+      );
+    });
+
+    it('throws if event requires an elevated telemetry level', function() {
+      const checker = buildTelemetryEventChecker('limited');
+      assert.throws(
+        () => checker('signupVerified', {}),
+        // eslint-disable-next-line max-len
+        /Telemetry event signupVerified requires a minimum telemetry level of 2 but the current level is 1/
+      );
+    });
+
+    it('throws if metadata requires an elevated telemetry level', function() {
+      const checker = buildTelemetryEventChecker('limited');
+      assert.throws(
+        () => checker('watchedVideoTour', {
+          watchTimeSeconds: 30,
+          userId: 1,
+          altSessionId: 'altSessionId',
+        }),
+        // eslint-disable-next-line max-len
+        /Telemetry metadata userId of event watchedVideoTour requires a minimum telemetry level of 2 but the current level is 1/
+      );
+    });
+  });
+
+  describe('filterMetadata', function() {
+    it('returns filtered and flattened metadata when maxLevel is "full"', function() {
+      const metadata = {
+        limited: {
+          foo: 'abc',
+        },
+        full: {
+          bar: '123',
+        },
+      };
+      assert.deepEqual(filterMetadata(metadata, 'full'), {
+        foo: 'abc',
+        bar: '123',
+      });
+    });
+
+    it('returns filtered and flattened metadata when maxLevel is "limited"', function() {
+      const metadata = {
+        limited: {
+          foo: 'abc',
+        },
+        full: {
+          bar: '123',
+        },
+      };
+      assert.deepEqual(filterMetadata(metadata, 'limited'), {
+        foo: 'abc',
+      });
+    });
+
+    it('returns undefined when maxLevel is "off"', function() {
+      assert.isUndefined(filterMetadata(undefined, 'off'));
+    });
+
+    it('returns an empty object when metadata is empty', function() {
+      assert.isEmpty(filterMetadata({}, 'full'));
+    });
+
+    it('returns undefined when metadata is undefined', function() {
+      assert.isUndefined(filterMetadata(undefined, 'full'));
+    });
+
+    it('does not mutate metadata', function() {
+      const metadata = {
+        limited: {
+          foo: 'abc',
+        },
+        full: {
+          bar: '123',
+        },
+      };
+      filterMetadata(metadata, 'limited');
+      assert.deepEqual(metadata, {
+        limited: {
+          foo: 'abc',
+        },
+        full: {
+          bar: '123',
+        },
+      });
+    });
+
+    it('excludes keys with nullish values', function() {
+      const metadata = {
+        limited: {
+          foo1: null,
+          foo2: 'abc',
+        },
+        full: {
+          bar1: undefined,
+          bar2: '123',
+        },
+      };
+      assert.deepEqual(filterMetadata(metadata, 'full'), {
+        foo2: 'abc',
+        bar2: '123',
+      });
+    });
+  });
+});

test/nbrowser/testServer.ts

@@ -205,7 +205,11 @@ export class TestServerMerged implements IMochaServer {
     }
     const state: IGristUrlState = { org: team };
     const baseDomain = parseSubdomain(new URL(this.getHost()).hostname).base;
-    const gristConfig = makeGristConfig(this.getHost(), {}, baseDomain);
+    const gristConfig = makeGristConfig({
+      homeUrl: this.getHost(),
+      extra: {},
+      baseDomain,
+    });
     const url = encodeUrl(gristConfig, state, new URL(this.getHost())).replace(/\/$/, "");
     return `${url}${relPath}`;
   }

test/server/lib/Authorizer.ts

@@ -34,7 +34,7 @@ async function activateServer(home: FlexServer, docManager: DocManager) {
   home.addJsonSupport();
   await home.addLandingPages();
   home.addHomeApi();
-  home.addTelemetryEndpoint();
+  home.addTelemetry();
   await home.addDoc();
   home.addApiErrorHandlers();
   serverUrl = home.getOwnUrl();

test/server/lib/Telemetry.ts

@@ -0,0 +1,309 @@
+import {GristDeploymentType} from 'app/common/gristUrls';
+import {TelemetryEvent, TelemetryLevel, TelemetryMetadata} from 'app/common/Telemetry';
+import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods';
+import {ITelemetry, Telemetry} from 'app/server/lib/Telemetry';
+import axios from 'axios';
+import {assert} from 'chai';
+import * as sinon from 'sinon';
+import {TestServer} from 'test/gen-server/apiUtils';
+import {configForUser} from 'test/gen-server/testUtils';
+
+const chimpy = configForUser('Chimpy');
+const anon = configForUser('Anonymous');
+
+describe('Telemetry', function() {
+  const deploymentTypesAndTelemetryLevels: [GristDeploymentType, TelemetryLevel][] = [
+    ['saas', 'off'],
+    ['saas', 'limited'],
+    ['saas', 'full'],
+    ['core', 'off'],
+    ['core', 'limited'],
+    ['core', 'full'],
+  ];
+
+  for (const [deploymentType, telemetryLevel] of deploymentTypesAndTelemetryLevels) {
+    describe(`in grist-${deploymentType} with a telemetry level of "${telemetryLevel}"`, function() {
+      let homeUrl: string;
+      let installationId: string;
+      let server: TestServer;
+      let telemetry: ITelemetry;
+
+      const sandbox = sinon.createSandbox();
+      const loggedEvents: [TelemetryEvent, ILogMeta][] = [];
+      const forwardedEvents: [TelemetryEvent, TelemetryMetadata | undefined][] = [];
+
+      before(async function() {
+        process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE = deploymentType;
+        process.env.GRIST_TELEMETRY_LEVEL = telemetryLevel;
+        server = new TestServer(this);
+        homeUrl = await server.start();
+        installationId = (await server.server.getActivations().current()).id;
+        sandbox
+          .stub(LogMethods.prototype, 'rawLog')
+          .callsFake((_level: string, _info: unknown, name: string, meta: ILogMeta) => {
+            loggedEvents.push([name as TelemetryEvent, meta]);
+          });
+        sandbox
+          .stub(Telemetry.prototype, 'forwardEvent')
+          .callsFake((event: TelemetryEvent, metadata?: TelemetryMetadata) => {
+            forwardedEvents.push([event, metadata]);
+          });
+        telemetry = server.server.getTelemetry();
+      });
+
+      after(async function() {
+        await server.stop();
+        sandbox.restore();
+        delete process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE;
+        delete process.env.GRIST_TELEMETRY_LEVEL;
+      });
+
+      it('returns the current telemetry level', async function() {
+        assert.equal(telemetry.getTelemetryLevel(), telemetryLevel);
+      });
+
+      if (telemetryLevel !== 'off') {
+        if (deploymentType === 'saas') {
+          it('logs telemetry events', async function() {
+            if (telemetryLevel === 'limited') {
+              await telemetry.logEvent('documentOpened', {
+                limited: {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                },
+              });
+              assert.deepEqual(loggedEvents[loggedEvents.length - 1], [
+                'documentOpened',
+                {
+                  eventName: 'documentOpened',
+                  eventSource: `grist-${deploymentType}`,
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                }
+              ]);
+            }
+
+            if (telemetryLevel === 'full') {
+              await telemetry.logEvent('documentOpened', {
+                limited: {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                },
+                full: {
+                  userId: 1,
+                },
+              });
+              assert.deepEqual(loggedEvents[loggedEvents.length - 1], [
+                'documentOpened',
+                {
+                  eventName: 'documentOpened',
+                  eventSource: `grist-${deploymentType}`,
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                  userId: 1,
+                }
+              ]);
+            }
+
+            assert.equal(loggedEvents.length, 1);
+            assert.isEmpty(forwardedEvents);
+          });
+        } else {
+          it('forwards telemetry events', async function() {
+            if (telemetryLevel === 'limited') {
+              await telemetry.logEvent('documentOpened', {
+                limited: {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                },
+              });
+              assert.deepEqual(forwardedEvents[forwardedEvents.length - 1], [
+                'documentOpened',
+                {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                }
+              ]);
+            }
+
+            if (telemetryLevel === 'full') {
+              await telemetry.logEvent('documentOpened', {
+                limited: {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                },
+                full: {
+                  userId: 1,
+                },
+              });
+              assert.deepEqual(forwardedEvents[forwardedEvents.length - 1], [
+                'documentOpened',
+                {
+                  docIdDigest: 'digest',
+                  isPublic: false,
+                  userId: 1,
+                }
+              ]);
+            }
+
+            assert.equal(forwardedEvents.length, 1);
+            assert.isEmpty(loggedEvents);
+          });
+        }
+      } else {
+        it('does not log telemetry events', async function() {
+          await telemetry.logEvent('documentOpened', {
+            limited: {
+              docIdDigest: 'digest',
+              isPublic: false,
+            },
+          });
+          assert.isEmpty(loggedEvents);
+          assert.isEmpty(forwardedEvents);
+        });
+      }
+
+      if (telemetryLevel !== 'off') {
+        it('throws an error when an event is invalid', async function() {
+          await assert.isRejected(
+            telemetry.logEvent('invalidEvent' as TelemetryEvent, {limited: {method: 'GET'}}),
+            /Unknown telemetry event: invalidEvent/
+          );
+        });
+
+        it("throws an error when an event's metadata is invalid", async function() {
+          await assert.isRejected(
+            telemetry.logEvent('documentOpened', {limited: {invalidMetadata: 'GET'}}),
+            /Unknown metadata for telemetry event documentOpened: invalidMetadata/
+          );
+        });
+
+        if (telemetryLevel === 'limited') {
+          it('throws an error when an event requires an elevated telemetry level', async function() {
+            await assert.isRejected(
+              telemetry.logEvent('signupVerified', {}),
+              /Telemetry event signupVerified requires a minimum telemetry level of 2 but the current level is 1/
+            );
+          });
+
+          it("throws an error when an event's metadata requires an elevated telemetry level", async function() {
+            await assert.isRejected(
+              telemetry.logEvent('documentOpened', {limited: {userId: 1}}),
+              // eslint-disable-next-line max-len
+              /Telemetry metadata userId of event documentOpened requires a minimum telemetry level of 2 but the current level is 1/
+            );
+          });
+        }
+      }
+
+      if (telemetryLevel !== 'off') {
+        if (deploymentType === 'saas') {
+          it('logs telemetry events sent to /api/telemetry', async function() {
+            await axios.post(`${homeUrl}/api/telemetry`, {
+              event: 'watchedVideoTour',
+              metadata: {
+                limited: {watchTimeSeconds: 30},
+              },
+            }, chimpy);
+            const [event, metadata] = loggedEvents[loggedEvents.length - 1];
+            assert.equal(event, 'watchedVideoTour');
+            if (telemetryLevel === 'limited') {
+              assert.deepEqual(metadata, {
+                eventName: 'watchedVideoTour',
+                eventSource: `grist-${deploymentType}`,
+                watchTimeSeconds: 30,
+              });
+            } else {
+              assert.containsAllKeys(metadata, [
+                'eventSource',
+                'watchTimeSeconds',
+                'userId',
+                'altSessionId',
+              ]);
+              assert.equal(metadata.watchTimeSeconds, 30);
+              assert.equal(metadata.userId, 1);
+            }
+
+            if (telemetryLevel === 'limited') {
+              assert.equal(loggedEvents.length, 2);
+            } else {
+              // The POST above also triggers an "apiUsage" event.
+              assert.equal(loggedEvents.length, 3);
+              assert.equal(loggedEvents[1][0], 'apiUsage');
+            }
+            assert.isEmpty(forwardedEvents);
+          });
+
+          if (telemetryLevel === 'limited') {
+            it('skips checks if event sent to /api/telemetry is from an external source', async function() {
+              await axios.post(`${homeUrl}/api/telemetry`, {
+                event: 'watchedVideoTour',
+                metadata: {
+                  eventSource: 'grist-core',
+                  watchTimeSeconds: 60,
+                  userId: 123,
+                  altSessionId: 'altSessionId',
+                },
+              }, anon);
+              const [event, metadata] = loggedEvents[loggedEvents.length - 1];
+              assert.equal(event, 'watchedVideoTour');
+              assert.containsAllKeys(metadata, [
+                'eventSource',
+                'watchTimeSeconds',
+                'userId',
+                'altSessionId',
+              ]);
+              assert.equal(metadata.watchTimeSeconds, 60);
+              assert.equal(metadata.userId, 123);
+              assert.equal(loggedEvents.length, 3);
+              assert.isEmpty(forwardedEvents);
+            });
+          }
+        } else {
+          it('forwards telemetry events sent to /api/telemetry', async function() {
+            await axios.post(`${homeUrl}/api/telemetry`, {
+              event: 'watchedVideoTour',
+              metadata: {
+                limited: {watchTimeSeconds: 30},
+              },
+            }, chimpy);
+            const [event, metadata] = forwardedEvents[forwardedEvents.length - 1];
+            assert.equal(event, 'watchedVideoTour');
+            if (telemetryLevel === 'limited') {
+              assert.deepEqual(metadata, {
+                eventSource: `grist-${deploymentType}`,
+                installationId,
+                watchTimeSeconds: 30,
+              });
+            } else {
+              assert.containsAllKeys(metadata, [
+                'eventSource',
+                'installationId',
+                'watchTimeSeconds',
+                'userId',
+                'altSessionId',
+              ]);
+              assert.equal(metadata!.watchTimeSeconds, 30);
+              assert.equal(metadata!.userId, 1);
+            }
+
+            if (telemetryLevel === 'limited') {
+              assert.equal(forwardedEvents.length, 2);
+            } else {
+              // The POST above also triggers an "apiUsage" event.
+              assert.equal(forwardedEvents.length, 3);
+              assert.equal(forwardedEvents[1][0], 'apiUsage');
+            }
+            assert.isEmpty(loggedEvents);
+          });
+        }
+      } else {
+        it('does not log telemetry events sent to /api/telemetry', async function() {
+          await telemetry.logEvent('apiUsage', {limited: {method: 'GET'}});
+          assert.isEmpty(loggedEvents);
+          assert.isEmpty(forwardedEvents);
+        });
+      }
+    });
+  }
+});