(core) Add telemetry

Test Plan: Server tests.

Reviewers: jarek

Differential Revision: https://phab.getgrist.com/D3818

app/client/lib/helpScout.ts

@@ -15,6 +15,7 @@
 
 // tslint:disable:unified-signatures
 
+import {logTelemetryEvent} from 'app/client/lib/telemetry';
 import {AppModel} from 'app/client/models/AppModel';
 import {reportWarning} from 'app/client/models/errors';
 import {IAppError} from 'app/client/models/NotifyModel';
@@ -54,6 +55,11 @@ interface ISessionData {
   [key: string]: string;
 }
 
+interface ICallbackAttributes {
+  id?: string;
+  query?: string;
+}
+
 /**
  * This provides the HelpScout Beacon API, taking care of initializing Beacon on first use.
  */
@@ -65,7 +71,8 @@ export function Beacon(method: 'navigate', route: string): void;
 export function Beacon(method: 'identify', userObj: IUserObj): void;
 export function Beacon(method: 'prefill', formObj: IFormObj): void;
 export function Beacon(method: 'config', configObj: object): void;
-export function Beacon(method: 'on'|'once', event: string, callback: () => void): void;
+export function Beacon(method: 'on'|'once', event: string,
+  callback: (attrs?: ICallbackAttributes) => void): void;
 export function Beacon(method: 'off', event: string, callback?: () => void): void;
 export function Beacon(method: 'session-data', data: ISessionData): void;
 export function Beacon(method: BeaconCmd): void;
@@ -187,6 +194,15 @@ function _beaconOpen(userObj: IUserObj|null, options: IBeaconOpenOptions) {
   if (!skipNav) {
     Beacon('navigate', route);
   }
+
+  Beacon('once', 'open', () => logTelemetryEvent('beaconOpen'));
+  Beacon('on', 'article-viewed', (article) => logTelemetryEvent('beaconArticleViewed', {
+    articleId: article!.id,
+  }));
+  Beacon('on', 'email-sent', () => logTelemetryEvent('beaconEmailSent'));
+  Beacon('on', 'search', (search) => logTelemetryEvent('beaconSearch', {
+    searchQuery: search!.query,
+  }));
 }
 
 function fixBeaconBaseHref() {

app/client/lib/telemetry.ts

@@ -0,0 +1,23 @@
+import {logError} from 'app/client/models/errors';
+import {TelemetryEventName} from 'app/common/Telemetry';
+import {fetchFromHome, pageHasHome} from 'app/common/urlUtils';
+
+export async function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<string, any>) {
+  if (!pageHasHome()) { return; }
+
+  await fetchFromHome('/api/telemetry', {
+    method: 'POST',
+    body: JSON.stringify({
+      name,
+      metadata,
+    }),
+    credentials: 'include',
+    headers: {
+      'Content-Type': 'application/json',
+      'X-Requested-With': 'XMLHttpRequest',
+    },
+  }).catch((e: Error) => {
+    console.warn(`Failed to log telemetry event ${name}`, e);
+    logError(e);
+  });
+}

app/client/models/errors.ts

@@ -59,7 +59,7 @@ export function reportMessage(msg: MessageType, options?: Partial<INotifyOptions
 export function reportWarning(msg: string, options?: Partial<INotifyOptions>) {
   options = {level: 'warning', ...options};
   log.warn(`${options.level}: `, msg);
-  _logError(msg);
+  logError(msg);
   return reportMessage(msg, options);
 }
 
@@ -84,7 +84,7 @@ export function reportError(err: Error|string): void {
     // This error can be emitted while a page is reloaded, and isn't worth reporting.
     return;
   }
-  _logError(err);
+  logError(err);
   if (_notifier && !_notifier.isDisposed()) {
     if (!isError(err)) {
       err = new Error(String(err));
@@ -175,7 +175,7 @@ export function setUpErrorHandling(doReportError = reportError, koUtil?: any) {
  * over-logging (regular errors such as access rights or account limits) and
  * under-logging (javascript errors during startup might never get reported).
  */
-function _logError(error: Error|string) {
+export function logError(error: Error|string) {
   if (!pageHasHome()) { return; }
   const docId = G.window.gristDocPageModel?.currentDocId?.get();
   fetchFromHome('/api/log', {

app/client/ui/OpenVideoTour.ts

@@ -1,5 +1,6 @@
 import * as commands from 'app/client/components/commands';
 import {makeT} from 'app/client/lib/localization';
+import {logTelemetryEvent} from 'app/client/lib/telemetry';
 import {getMainOrgUrl} from 'app/client/models/gristUrlState';
 import {cssLinkText, cssPageEntryMain, cssPageIcon, cssPageLink} from 'app/client/ui/LeftPanelCommon';
 import {YouTubePlayer} from 'app/client/ui/YouTubePlayer';
@@ -20,7 +21,26 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
  */
  export function openVideoTour(refElement: HTMLElement) {
   return modal(
-    (ctl) => {
+    (ctl, owner) => {
+      const youtubePlayer = YouTubePlayer.create(owner,
+        VIDEO_TOUR_YOUTUBE_EMBED_ID,
+        {
+          onPlayerReady: (player) => player.playVideo(),
+          height: '100%',
+          width: '100%',
+          origin: getMainOrgUrl(),
+        },
+        cssYouTubePlayer.cls(''),
+      );
+
+      owner.onDispose(async () => {
+        if (youtubePlayer.isLoading()) { return; }
+
+        await logTelemetryEvent('watchedVideoTour', {
+          watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime()),
+        });
+      });
+
       return [
         cssModal.cls(''),
         cssModalCloseButton(
@@ -28,18 +48,7 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
           dom.on('click', () => ctl.close()),
           testId('close'),
         ),
-        cssYouTubePlayerContainer(
-          dom.create(YouTubePlayer,
-            VIDEO_TOUR_YOUTUBE_EMBED_ID,
-            {
-              onPlayerReady: (player) => player.playVideo(),
-              height: '100%',
-              width: '100%',
-              origin: getMainOrgUrl(),
-            },
-            cssYouTubePlayer.cls(''),
-          ),
-        ),
+        cssYouTubePlayerContainer(youtubePlayer.buildDom()),
         testId('modal'),
       ];
     },

app/client/ui/YouTubePlayer.ts

@@ -10,6 +10,7 @@ export interface Player {
   mute(): void;
   unMute(): void;
   setVolume(volume: number): void;
+  getCurrentTime(): number;
 }
 
 export interface PlayerOptions {
@@ -80,6 +81,10 @@ export class YouTubePlayer extends Disposable {
     }
   }
 
+  public isLoading() {
+    return this._isLoading();
+  }
+
   public isLoaded() {
     return waitObs(this._isLoading, (val) => !val);
   }
@@ -92,6 +97,10 @@ export class YouTubePlayer extends Disposable {
     this._player.setVolume(volume);
   }
 
+  public getCurrentTime(): number {
+    return this._player.getCurrentTime();
+  }
+
   public buildDom() {
     return dom('div', {id: this._playerId}, ...this._domArgs);
   }

app/common/Telemetry.ts

@@ -0,0 +1,21 @@
+export const TelemetryTemplateSignupCookieName = 'gr_template_signup_trk';
+
+export const TelemetryEventNames = [
+  'apiUsage',
+  'beaconOpen',
+  'beaconArticleViewed',
+  'beaconEmailSent',
+  'beaconSearch',
+  'documentForked',
+  'documentOpened',
+  'documentUsage',
+  'sendingWebhooks',
+  'signupVerified',
+  'siteMembership',
+  'siteUsage',
+  'tutorialProgressChanged',
+  'tutorialRestarted',
+  'watchedVideoTour',
+] as const;
+
+export type TelemetryEventName = typeof TelemetryEventNames[number];

app/common/UserAPI.ts

@@ -125,6 +125,7 @@ export interface DocumentOptions {
 
 export interface TutorialMetadata {
   lastSlideIndex?: number;
+  numSlides?: number;
 }
 
 export interface DocumentProperties extends CommonProperties {

app/common/gristUrls.ts

@@ -79,6 +79,7 @@ export const commonUrls = {
   efcrHelp: 'https://www.nioxus.info/eFCR-Help',
   basicTutorial: 'https://templates.getgrist.com/woXtXUBmiN5T/Grist-Basics',
   basicTutorialImage: 'https://www.getgrist.com/wp-content/uploads/2021/08/lightweight-crm.png',
+  gristLabsCustomWidgets: 'https://gristlabs.github.io/grist-widget/',
 };
 
 /**

app/common/schema.ts

@@ -4,7 +4,7 @@ import { GristObjCode } from "app/plugin/GristData";
 
 // tslint:disable:object-literal-key-quotes
 
-export const SCHEMA_VERSION = 36;
+export const SCHEMA_VERSION = 37;
 
 export const schema = {
 
@@ -148,6 +148,7 @@ export const schema = {
     fileName            : "Text",
     fileType            : "Text",
     fileSize            : "Int",
+    fileExt             : "Text",
     imageHeight         : "Int",
     imageWidth          : "Int",
     timeDeleted         : "DateTime",
@@ -354,6 +355,7 @@ export interface SchemaTypes {
     fileName: string;
     fileType: string;
     fileSize: number;
+    fileExt: string;
     imageHeight: number;
     imageWidth: number;
     timeDeleted: number;

app/gen-server/entity/Document.ts

@@ -2,7 +2,8 @@ import {ApiError} from 'app/common/ApiError';
 import {DocumentUsage} from 'app/common/DocUsage';
 import {Role} from 'app/common/roles';
 import {DocumentOptions, DocumentProperties, documentPropertyKeys,
-        DocumentType, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+        DocumentType, NEW_DOCUMENT_CODE, TutorialMetadata} from "app/common/UserAPI";
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {nativeValues} from 'app/gen-server/lib/values';
 import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
 import {AclRuleDoc} from "./AclRule";
@@ -90,7 +91,7 @@ export class Document extends Resource {
     return super.checkProperties(props, documentPropertyKeys);
   }
 
-  public updateFromProperties(props: Partial<DocumentProperties>) {
+  public updateFromProperties(props: Partial<DocumentProperties>, dbManager?: HomeDBManager) {
     super.updateFromProperties(props);
     if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
     if (props.urlId !== undefined) {
@@ -131,6 +132,12 @@ export class Document extends Resource {
             if (props.options.tutorial.lastSlideIndex !== undefined) {
               this.options.tutorial.lastSlideIndex = props.options.tutorial.lastSlideIndex;
             }
+            if (props.options.tutorial.numSlides !== undefined) {
+              this.options.tutorial.numSlides = props.options.tutorial.numSlides;
+              if (dbManager && props.options?.tutorial?.lastSlideIndex !== undefined) {
+                this._emitTutorialProgressChangeEvent(dbManager, this.options.tutorial);
+              }
+            }
           }
         }
         // Normalize so that null equates with absence.
@@ -146,6 +153,24 @@ export class Document extends Resource {
       }
     }
   }
+
+  private _emitTutorialProgressChangeEvent(
+    dbManager: HomeDBManager,
+    tutorialMetadata: TutorialMetadata
+  ) {
+    const lastSlideIndex = tutorialMetadata.lastSlideIndex;
+    const numSlides = tutorialMetadata.numSlides;
+    const percentComplete = lastSlideIndex !== undefined && numSlides !== undefined
+      ? Math.floor((lastSlideIndex / numSlides) * 100)
+      : undefined;
+    dbManager?.emit('tutorialProgressChange', {
+      tutorialForkId: this.id,
+      tutorialTrunkId: this.trunkId,
+      lastSlideIndex,
+      numSlides,
+      percentComplete,
+    });
+  }
 }
 
 // Check that icon points to an expected location.  This will definitely

app/gen-server/lib/HomeDBManager.ts

@@ -89,6 +89,14 @@ export const NotifierEvents = StringUnion(
 
 export type NotifierEvent = typeof NotifierEvents.type;
 
+export const TelemetryEvents = StringUnion(
+  'tutorialProgressChange',
+);
+
+export type TelemetryEvent = typeof TelemetryEvents.type;
+
+export type Event = NotifierEvent | TelemetryEvent;
+
 // Nominal email address of a user who can view anything (for thumbnails).
 export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com';
 
@@ -276,6 +284,7 @@ export class HomeDBManager extends EventEmitter {
   // In restricted mode, documents should be read-only.
   private _restrictedMode: boolean = false;
 
+
   /**
    * Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers',
    * 'guests', and 'members') are created by default on every new entity (Organization,
@@ -315,7 +324,7 @@ export class HomeDBManager extends EventEmitter {
     orgOnly: true
   }];
 
-  public emit(event: NotifierEvent, ...args: any[]): boolean {
+  public emit(event: Event, ...args: any[]): boolean {
     return super.emit(event, ...args);
   }
 
@@ -1944,7 +1953,7 @@ export class HomeDBManager extends EventEmitter {
       // Update the name and save.
       const doc: Document = queryResult.data;
       doc.checkProperties(props);
-      doc.updateFromProperties(props);
+      doc.updateFromProperties(props, this);
       if (forkId) {
         await manager.save(doc);
         return {status: 200};

app/gen-server/lib/Housekeeper.ts

@@ -1,6 +1,7 @@
 import { ApiError } from 'app/common/ApiError';
 import { buildUrlId } from 'app/common/gristUrls';
 import { Document } from 'app/gen-server/entity/Document';
+import { Organization } from 'app/gen-server/entity/Organization';
 import { Workspace } from 'app/gen-server/entity/Workspace';
 import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
 import { fromNow } from 'app/gen-server/sqlUtils';
@@ -14,6 +15,7 @@ import { optStringParam, stringParam } from 'app/server/lib/requestUtils';
 import * as express from 'express';
 import fetch from 'node-fetch';
 import * as Fetch from 'node-fetch';
+import { EntityManager } from 'typeorm';
 
 const HOUSEKEEPER_PERIOD_MS = 1 * 60 * 60 * 1000;   // operate every 1 hour
 const AGE_THRESHOLD_OFFSET = '-30 days';            // should be an interval known by postgres + sqlite
@@ -23,6 +25,7 @@ const AGE_THRESHOLD_OFFSET = '-30 days';            // should be an interval kno
  *
  *  - deleting old soft-deleted documents
  *  - deleting old soft-deleted workspaces
+ *  - logging metrics
  *
  * Call start(), keep the object around, and call stop() when shutting down.
  *
@@ -42,7 +45,9 @@ export class Housekeeper {
    */
   public async start() {
     await this.stop();
-    this._interval = setInterval(() => this.deleteTrashExclusively().catch(log.warn.bind(log)), HOUSEKEEPER_PERIOD_MS);
+    this._interval = setInterval(() => {
+      this.doHousekeepingExclusively().catch(log.warn.bind(log));
+    }, HOUSEKEEPER_PERIOD_MS);
   }
 
   /**
@@ -56,16 +61,18 @@ export class Housekeeper {
   }
 
   /**
-   * Deletes old trash if no other server is working on it or worked on it recently.
+   * Deletes old trash and logs metrics if no other server is working on it or worked on it
+   * recently.
    */
-  public async deleteTrashExclusively(): Promise<boolean> {
-    const electionKey = await this._electionStore.getElection('trash', HOUSEKEEPER_PERIOD_MS / 2.0);
+  public async doHousekeepingExclusively(): Promise<boolean> {
+    const electionKey = await this._electionStore.getElection('housekeeping', HOUSEKEEPER_PERIOD_MS / 2.0);
     if (!electionKey) {
-      log.info('Skipping deleteTrash since another server is working on it or worked on it recently');
+      log.info('Skipping housekeeping since another server is working on it or worked on it recently');
       return false;
     }
     this._electionKey = electionKey;
     await this.deleteTrash();
+    await this.logMetrics();
     return true;
   }
 
@@ -145,6 +152,39 @@ export class Housekeeper {
     }
   }
 
+  /**
+   * Logs metrics regardless of what other servers may be doing.
+   */
+  public async logMetrics() {
+    await this._dbManager.connection.transaction('READ UNCOMMITTED', async (manager) => {
+      const telemetryManager = this._server.getTelemetryManager();
+      const usageSummaries = await this._getOrgUsageSummaries(manager);
+      for (const summary of usageSummaries) {
+        telemetryManager?.logEvent('siteUsage', {
+          siteId: summary.site_id,
+          siteType: summary.site_type,
+          inGoodStanding: summary.in_good_standing,
+          stripePlanId: summary.stripe_plan_id,
+          numDocs: summary.num_docs,
+          numWorkspaces: summary.num_workspaces,
+          numMembers: summary.num_members,
+          lastActivity: summary.last_activity,
+        });
+      }
+
+      const membershipSummaries = await this._getOrgMembershipSummaries(manager);
+      for (const summary of membershipSummaries) {
+        telemetryManager?.logEvent('siteMembership', {
+          siteId: summary.site_id,
+          siteType: summary.site_type,
+          numOwners: summary.num_owners,
+          numEditors: summary.num_editors,
+          numViewers: summary.num_viewers,
+        });
+      }
+    });
+  }
+
   public addEndpoints(app: express.Application) {
     // Allow support user to perform housekeeping tasks for a specific
     // document.  The tasks necessarily bypass user access controls.
@@ -208,7 +248,7 @@ export class Housekeeper {
    */
   public async testClearExclusivity(): Promise<void> {
     if (this._electionKey) {
-      await this._electionStore.removeElection('trash', this._electionKey);
+      await this._electionStore.removeElection('housekeeping', this._electionKey);
       this._electionKey = undefined;
     }
   }
@@ -249,6 +289,52 @@ export class Housekeeper {
     return forks;
   }
 
+  private async _getOrgUsageSummaries(manager: EntityManager) {
+    const orgs = await manager.createQueryBuilder()
+      .select('orgs.id', 'site_id')
+      .addSelect('products.name', 'site_type')
+      .addSelect('billing_accounts.in_good_standing', 'in_good_standing')
+      .addSelect('billing_accounts.stripe_plan_id', 'stripe_plan_id')
+      .addSelect('COUNT(DISTINCT docs.id)', 'num_docs')
+      .addSelect('COUNT(DISTINCT workspaces.id)', 'num_workspaces')
+      .addSelect('COUNT(DISTINCT org_member_users.id)', 'num_members')
+      .addSelect('MAX(docs.updated_at)', 'last_activity')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.workspaces', 'workspaces')
+      .leftJoin('workspaces.docs', 'docs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .leftJoin('billing_accounts.product', 'products')
+      .leftJoin('orgs.aclRules', 'acl_rules')
+      .leftJoin('acl_rules.group', 'org_groups')
+      .leftJoin('org_groups.memberUsers', 'org_member_users')
+      .where('org_member_users.id IS NOT NULL')
+      .groupBy('orgs.id')
+      .addGroupBy('products.id')
+      .addGroupBy('billing_accounts.id')
+      .getRawMany();
+    return orgs;
+  }
+
+  private async _getOrgMembershipSummaries(manager: EntityManager) {
+    const orgs = await manager.createQueryBuilder()
+      .select('orgs.id', 'site_id')
+      .addSelect('products.name', 'site_type')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'owners' THEN 1 ELSE 0 END)", 'num_owners')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'editors' THEN 1 ELSE 0 END)", 'num_editors')
+      .addSelect("SUM(CASE WHEN org_groups.name = 'viewers' THEN 1 ELSE 0 END)", 'num_viewers')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .leftJoin('billing_accounts.product', 'products')
+      .leftJoin('orgs.aclRules', 'acl_rules')
+      .leftJoin('acl_rules.group', 'org_groups')
+      .leftJoin('org_groups.memberUsers', 'org_member_users')
+      .where('org_member_users.id IS NOT NULL')
+      .groupBy('orgs.id')
+      .addGroupBy('products.id')
+      .getRawMany();
+    return orgs;
+  }
+
   /**
    * TypeORM isn't very adept at handling date representation for
    * comparisons, so we construct the threshold date in SQL so that we

app/server/generateInitialDocSql.ts

@@ -31,7 +31,10 @@ export async function main(baseName: string) {
     if (await fse.pathExists(fname)) {
       await fse.remove(fname);
     }
-    const docManager = new DocManager(storageManager, pluginManager, null as any, {create} as any);
+    const docManager = new DocManager(storageManager, pluginManager, null as any, {
+      create,
+      getTelemetryManager: () => undefined,
+    } as any);
     const activeDoc = new ActiveDoc(docManager, baseName);
     const session = makeExceptionalDocSession('nascent');
     await activeDoc.createEmptyDocWithDataEngine(session);

app/server/lib/ActiveDoc.ts

@@ -65,19 +65,22 @@ import {
 import {normalizeEmail} from 'app/common/emails';
 import {Product} from 'app/common/Features';
 import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
-import {parseUrlId} from 'app/common/gristUrls';
+import {isHiddenCol} from 'app/common/gristTypes';
+import {commonUrls, parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
 import {schema, SCHEMA_VERSION} from 'app/common/schema';
 import {MetaRowRecord, SingleCell} from 'app/common/TableData';
+import {TelemetryEventName} from 'app/common/Telemetry';
 import {UIRowId} from 'app/common/UIRowId';
 import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
-import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
+import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
 import {convertFromColumn} from 'app/common/ValueConverter';
 import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
 import {parseUserAction} from 'app/common/ValueParser';
+import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from 'app/gen-server/entity/Document';
 import {ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
@@ -117,6 +120,7 @@ import {DocPluginManager} from './DocPluginManager';
 import {
   DocSession,
   getDocSessionAccess,
+  getDocSessionAltSessionId,
   getDocSessionUser,
   getDocSessionUserId,
   makeExceptionalDocSession,
@@ -157,6 +161,9 @@ const UPDATE_CURRENT_TIME_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 100
 // Measure and broadcast data size every 5 minutes
 const UPDATE_DATA_SIZE_DELAY = {delayMs: 5 * 60 * 1000, varianceMs: 30 * 1000};
 
+// Log document metrics every hour
+const LOG_DOCUMENT_METRICS_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 1000};
+
 // A hook for dependency injection.
 export const Deps = {ACTIVEDOC_TIMEOUT};
 
@@ -215,6 +222,7 @@ export class ActiveDoc extends EventEmitter {
   private _fullyLoaded: boolean = false;  // Becomes true once all columns are loaded/computed.
   private _lastMemoryMeasurement: number = 0;    // Timestamp when memory was last measured.
   private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
+  private _doc: Document|undefined;
   private _docUsage: DocumentUsage|null = null;
   private _product?: Product;
   private _gracePeriodStart: Date|null = null;
@@ -260,6 +268,12 @@ export class ActiveDoc extends EventEmitter {
       UPDATE_DATA_SIZE_DELAY,
       {onError: (e) => this._log.error(null, 'failed to update data size', e)},
     ),
+    // Log document metrics every hour.
+    new Interval(
+      () => this._logDocMetrics(makeExceptionalDocSession('system'), 'interval'),
+      LOG_DOCUMENT_METRICS_DELAY,
+      {onError: (e) => this._log.error(null, 'failed to log document metrics', e)},
+    ),
   ];
 
   constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
@@ -268,7 +282,8 @@ export class ActiveDoc extends EventEmitter {
     this._isForkOrSnapshot = Boolean(forkId || snapshotId);
     if (_options?.safeMode) { this._recoveryMode = true; }
     if (_options?.doc) {
-      const {gracePeriodStart, workspace, usage} = _options.doc;
+      this._doc = _options.doc;
+      const {gracePeriodStart, workspace, usage} = this._doc;
       const billingAccount = workspace.org.billingAccount;
       this._product = billingAccount?.product;
       this._gracePeriodStart = gracePeriodStart;
@@ -1348,6 +1363,17 @@ export class ActiveDoc extends EventEmitter {
       }
 
       await dbManager.forkDoc(userId, doc, forkIds.forkId);
+
+      // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
+      const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
+      this.logTelemetryEvent(docSession, 'documentForked', {
+        forkId: forkIds.forkId,
+        forkDocId: forkIds.docId,
+        forkUrlId: forkIds.urlId,
+        trunkId: doc.trunkId,
+        isTemplate,
+        lastActivity: doc.updatedAt,
+      });
     } finally {
       await permitStore.removePermit(permitKey);
     }
@@ -1720,6 +1746,17 @@ export class ActiveDoc extends EventEmitter {
     return this._triggers.summary();
   }
 
+  public logTelemetryEvent(
+    docSession: OptDocSession | null,
+    eventName: TelemetryEventName,
+    metadata?: Record<string, any>
+  ) {
+    this._docManager.gristServer.getTelemetryManager()?.logEvent(eventName, {
+      ...this._getTelemetryMeta(docSession),
+      ...metadata,
+    });
+  }
+
   /**
    * Loads an open document from DocStorage.  Returns a list of the tables it contains.
    */
@@ -1883,6 +1920,7 @@ export class ActiveDoc extends EventEmitter {
       }
 
       this._syncDocUsageToDatabase(true);
+      this._logDocMetrics(docSession, 'docClose');
 
       try {
         await this._docManager.storageManager.closeDocument(this.docName);
@@ -2101,6 +2139,7 @@ export class ActiveDoc extends EventEmitter {
       // We used to set fileType, but it's not easily available for native types. Since it's
       // also entirely unused, we just skip it until it becomes relevant.
       fileSize: fileData.size,
+      fileExt: fileData.ext,
       imageHeight: dimensions.height,
       imageWidth: dimensions.width,
       timeUploaded: Date.now()
@@ -2238,6 +2277,147 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
+  private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
+    this.logTelemetryEvent(docSession, 'documentUsage', {
+      triggeredBy,
+      access: this._doc?.access,
+      isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
+      rowCount: this._docUsage?.rowCount?.total,
+      dataSizeBytes: this._docUsage?.dataSizeBytes,
+      attachmentsSize: this._docUsage?.attachmentsSizeBytes,
+      ...this._getAccessRuleMetrics(),
+      ...this._getAttachmentMetrics(),
+      ...this._getChartMetrics(),
+      ...this._getWidgetMetrics(),
+      ...this._getColumnMetrics(),
+      ...this._getTableMetrics(),
+      ...this._getCustomWidgetMetrics(),
+    });
+  }
+
+  private _getAccessRuleMetrics() {
+    const accessRules = this.docData?.getMetaTable('_grist_ACLRules');
+    const numAccessRules = accessRules?.numRecords() ?? 0;
+    const numUserAttributes = accessRules?.getRecords().filter(r => r.userAttributes).length ?? 0;
+
+    return {
+      numAccessRules,
+      numUserAttributes,
+    };
+  }
+
+  private _getAttachmentMetrics() {
+    const attachments = this.docData?.getMetaTable('_grist_Attachments');
+    const numAttachments = attachments?.numRecords() ?? 0;
+    const attachmentTypes = attachments?.getRecords()
+      .map(r => r.fileExt?.slice(1) ?? null)
+      .filter(ext => ext !== null);
+
+    return {
+      numAttachments,
+      attachmentTypes,
+    };
+  }
+
+  private _getChartMetrics() {
+    const viewSections = this.docData?.getMetaTable('_grist_Views_section');
+    const viewSectionRecords = viewSections?.getRecords() ?? [];
+    const chartRecords = viewSectionRecords?.filter(r => r.parentKey === 'chart') ?? [];
+    const chartTypes = chartRecords.map(r => r.chartType || 'bar');
+    const numCharts = chartRecords.length;
+    const numLinkedCharts = chartRecords.filter(r => r.linkSrcSectionRef).length;
+
+    return {
+      numCharts,
+      chartTypes,
+      numLinkedCharts,
+    };
+  }
+
+  private _getWidgetMetrics() {
+    const viewSections = this.docData?.getMetaTable('_grist_Views_section');
+    const viewSectionRecords = viewSections?.getRecords() ?? [];
+    const numLinkedWidgets = viewSectionRecords.filter(r => r.linkSrcSectionRef).length;
+
+    return {
+      numLinkedWidgets,
+    };
+  }
+
+  private _getColumnMetrics() {
+    const columns = this.docData?.getMetaTable('_grist_Tables_column');
+    const columnRecords = columns?.getRecords().filter(r => !isHiddenCol(r.colId)) ?? [];
+    const numColumns = columnRecords.length;
+    const numColumnsWithConditionalFormatting = columnRecords.filter(r => r.rules).length;
+    const numFormulaColumns = columnRecords.filter(r => r.isFormula && r.formula).length;
+    const numTriggerFormulaColumns = columnRecords.filter(r => !r.isFormula && r.formula).length;
+
+    const tables = this.docData?.getMetaTable('_grist_Tables');
+    const tableRecords = tables?.getRecords().filter(r =>
+      r.tableId && !r.tableId.startsWith('GristHidden_')) ?? [];
+    const summaryTables = tableRecords.filter(r => r.summarySourceTable);
+    const summaryTableIds = new Set([...summaryTables.map(t => t.id)]);
+    const numSummaryFormulaColumns = columnRecords.filter(r =>
+      r.isFormula && summaryTableIds.has(r.parentId)).length;
+
+    const viewSectionFields = this.docData?.getMetaTable('_grist_Views_section_field');
+    const viewSectionFieldRecords = viewSectionFields?.getRecords() ?? [];
+    const numFieldsWithConditionalFormatting = viewSectionFieldRecords.filter(r => r.rules).length;
+
+    return {
+      numColumns,
+      numColumnsWithConditionalFormatting,
+      numFormulaColumns,
+      numTriggerFormulaColumns,
+      numSummaryFormulaColumns,
+      numFieldsWithConditionalFormatting,
+    };
+  }
+
+  private _getTableMetrics() {
+    const tables = this.docData?.getMetaTable('_grist_Tables');
+    const tableRecords = tables?.getRecords().filter(r =>
+      r.tableId && !r.tableId.startsWith('GristHidden_')) ?? [];
+    const numTables = tableRecords.length;
+    const numOnDemandTables = tableRecords.filter(r => r.onDemand).length;
+
+    const viewSections = this.docData?.getMetaTable('_grist_Views_section');
+    const viewSectionRecords = viewSections?.getRecords() ?? [];
+    const numTablesWithConditionalFormatting = viewSectionRecords.filter(r => r.rules).length;
+
+    const summaryTables = tableRecords.filter(r => r.summarySourceTable);
+    const numSummaryTables = summaryTables.length;
+
+    return {
+      numTables,
+      numOnDemandTables,
+      numTablesWithConditionalFormatting,
+      numSummaryTables,
+    };
+  }
+
+  private _getCustomWidgetMetrics() {
+    const viewSections = this.docData?.getMetaTable('_grist_Views_section');
+    const viewSectionRecords = viewSections?.getRecords() ?? [];
+    const customWidgetUrls: string[] = [];
+    for (const r of viewSectionRecords) {
+      const {customView} = safeJsonParse(r.options, {});
+      if (!customView) { continue; }
+
+      const {url} = safeJsonParse(customView, {});
+      if (!url) { continue; }
+
+      const isGristUrl = url.startsWith(commonUrls.gristLabsCustomWidgets);
+      customWidgetUrls.push(isGristUrl ? url : 'externalURL');
+    }
+    const numCustomWidgets = customWidgetUrls.length;
+
+    return {
+      numCustomWidgets,
+      customWidgetUrls,
+    };
+  }
+
   private async _fetchQueryFromDB(query: ServerQuery, onDemand: boolean): Promise<TableDataAction> {
     // Expand query to compute formulas (or include placeholders for them).
     const expandedQuery = expandQuery(query, this.docData!, onDemand);
@@ -2279,7 +2459,10 @@ export class ActiveDoc extends EventEmitter {
     if (this._docUsage?.attachmentsSizeBytes === undefined) {
       promises.push(this._updateAttachmentsSize(options));
     }
-    if (promises.length === 0) { return; }
+    if (promises.length === 0) {
+      this._logDocMetrics(docSession, 'docOpen');
+      return;
+    }
 
     try {
       await Promise.all(promises);
@@ -2288,6 +2471,20 @@ export class ActiveDoc extends EventEmitter {
     } catch (e) {
       this._log.warn(docSession, 'failed to initialize doc usage', e);
     }
+
+    this._logDocMetrics(docSession, 'docOpen');
+  }
+
+  private _getTelemetryMeta(docSession: OptDocSession|null) {
+    return {
+      ...(docSession ? {
+          ...getLogMetaFromDocSession(docSession),
+          altSessionId: getDocSessionAltSessionId(docSession),
+        } : {}),
+      docId: this._docName,
+      siteId: this._doc?.workspace.org.id,
+      siteType: this._product?.name,
+    };
   }
 
   /**

app/server/lib/AppEndpoint.ts

@@ -7,10 +7,12 @@ import * as express from 'express';
 import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
 
 import {ApiError} from 'app/common/ApiError';
-import {getSlugIfNeeded, parseSubdomainStrictly} from 'app/common/gristUrls';
+import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
 import {LocalPlugin} from "app/common/plugin";
+import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
 import {Document as APIDocument} from 'app/common/UserAPI';
+import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
@@ -18,6 +20,7 @@ import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
 import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
+import {getCookieDomain} from 'app/server/lib/gristSessions';
 import {getAssignmentId} from 'app/server/lib/idUtils';
 import log from 'app/server/lib/log';
 import {adaptServerUrl, addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
@@ -299,6 +302,42 @@ export function attachAppEndpoint(options: AttachOptions): void {
       body = await workerInfo.resp.json();
     }
 
+    const isPublic = ((doc as unknown) as APIDocument).public ?? false;
+    const isSnapshot = parseUrlId(urlId).snapshotId;
+    // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
+    const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
+    if (isPublic || isTemplate) {
+      gristServer.getTelemetryManager()?.logEvent('documentOpened', {
+        docId,
+        siteId: doc.workspace.org.id,
+        siteType: doc.workspace.org.billingAccount.product.name,
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+        access: doc.access,
+        isPublic,
+        isSnapshot,
+        isTemplate,
+        lastUpdated: doc.updatedAt,
+      });
+    }
+
+    if (isTemplate) {
+      // Keep track of the last template a user visited in the last hour.
+      // If a sign-up occurs within that time period, we'll know which
+      // template, if any, was viewed most recently.
+      const value = {
+        isAnonymous: isAnonymousUser(mreq),
+        templateId: docId,
+      };
+      res.cookie(TelemetryTemplateSignupCookieName, JSON.stringify(value), {
+        maxAge: 1000 * 60 * 60,
+        httpOnly: true,
+        path: '/',
+        domain: getCookieDomain(req),
+        sameSite: 'lax',
+      });
+    }
+
     await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
                                  googleTagManager: 'anon', config: {
       assignmentId: docId,

app/server/lib/Authorizer.ts

@@ -160,6 +160,8 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
   // about this case.
   let authDone: boolean = false;
 
+  let hasApiKey: boolean = false;
+
   // Support providing an access token via an `auth` query parameter.
   // This is useful for letting the browser load assets like image
   // attachments.
@@ -191,6 +193,7 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
       mreq.user = user;
       mreq.userId = user.id;
       mreq.userIsAuthorized = true;
+      hasApiKey = true;
     }
   }
 
@@ -392,6 +395,12 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
     altSessionId: mreq.altSessionId,
   };
   log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
+  if (hasApiKey) {
+    options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
+      ...meta,
+      userAgent: req.headers['user-agent'],
+    });
+  }
 
   return next();
 }

app/server/lib/DocApi.ts

@@ -836,6 +836,7 @@ export class DocWorkerApi {
     // This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it
     // starts with to become muted.
     this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => {
+      const docSession = docSessionFromRequest(req);
       const activeDoc = await this._getActiveDoc(req);
       const options: DocReplacementOptions = {};
       if (req.body.sourceDocId) {
@@ -881,12 +882,17 @@ export class DocWorkerApi {
               manager
             );
           });
+          const {forkId} = parseUrlId(scope.urlId);
+          activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
+            tutorialForkId: forkId,
+            tutorialForkUrlId: scope.urlId,
+            tutorialTrunkId,
+          });
         }
       }
       if (req.body.snapshotId) {
         options.snapshotId = String(req.body.snapshotId);
       }
-      const docSession = docSessionFromRequest(req);
       await activeDoc.replace(docSession, options);
       res.json(null);
     }));

app/server/lib/FlexServer.ts

@@ -6,6 +6,7 @@ import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPath
 import {getOrgUrlInfo} from 'app/common/gristUrls';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {tbind} from 'app/common/tbind';
+import {TelemetryEventName, TelemetryEventNames} from 'app/common/Telemetry';
 import * as version from 'app/common/version';
 import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer';
 import {Document} from "app/gen-server/entity/Document";
@@ -55,6 +56,7 @@ import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
+import {TelemetryManager} from 'app/server/lib/TelemetryManager';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
@@ -127,6 +129,7 @@ export class FlexServer implements GristServer {
   private _sessions: Sessions;
   private _sessionStore: SessionStore;
   private _storageManager: IDocStorageManager;
+  private _telemetryManager: TelemetryManager|undefined;
   private _docWorkerMap: IDocWorkerMap;
   private _widgetRepository: IWidgetRepository;
   private _notifier: INotifier;
@@ -338,6 +341,10 @@ export class FlexServer implements GristServer {
     return this._storageManager;
   }
 
+  public getTelemetryManager(): TelemetryManager|undefined {
+    return this._telemetryManager;
+  }
+
   public getWidgetRepository(): IWidgetRepository {
     if (!this._widgetRepository) { throw new Error('no widget repository available'); }
     return this._widgetRepository;
@@ -663,7 +670,8 @@ export class FlexServer implements GristServer {
    */
   public addLogEndpoint() {
     if (this._check('log-endpoint', 'json', 'api-mw')) { return; }
-    this.app.post('/api/log', expressWrap(async (req, resp) => {
+
+    this.app.post('/api/log', async (req, resp) => {
       const mreq = req as RequestWithLogin;
       log.rawWarn('client error', {
         event: req.body.event,
@@ -676,7 +684,26 @@ export class FlexServer implements GristServer {
         altSessionId: mreq.altSessionId,
       });
       return resp.status(200).send();
-    }));
+    });
+  }
+
+  public addTelemetryEndpoint() {
+    if (this._check('telemetry-endpoint', 'json', 'api-mw', 'homedb')) { return; }
+
+    this._telemetryManager = new TelemetryManager(this._dbManager);
+
+    this.app.post('/api/telemetry', async (req, resp) => {
+      const mreq = req as RequestWithLogin;
+      const name = stringParam(req.body.name, 'name', TelemetryEventNames);
+      this._telemetryManager?.logEvent(name as TelemetryEventName, {
+        userId: mreq.userId,
+        email: mreq.user?.loginEmail,
+        altSessionId: mreq.altSessionId,
+        site: mreq.org,
+        ...req.body.metadata,
+      });
+      return resp.status(200).send();
+    });
   }
 
   public async close() {
@@ -1076,7 +1103,8 @@ export class FlexServer implements GristServer {
 
   // Add document-related endpoints and related support.
   public async addDoc() {
-    this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ? null : 'homedb', 'api-mw', 'map');
+    this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ?
+      null : 'homedb', 'api-mw', 'map', 'telemetry-endpoint');
     // add handlers for cleanup, if we are in charge of the doc manager.
     if (!this._docManager) { this.addCleanup(); }
     await this.loadConfig();

app/server/lib/GristServer.ts

@@ -16,6 +16,7 @@ import { IPermitStore } from 'app/server/lib/Permit';
 import { ISendAppPageOptions } from 'app/server/lib/sendAppPage';
 import { fromCallback } from 'app/server/lib/serverUtils';
 import { Sessions } from 'app/server/lib/Sessions';
+import { TelemetryManager } from 'app/server/lib/TelemetryManager';
 import * as express from 'express';
 import { IncomingMessage } from 'http';
 
@@ -42,6 +43,7 @@ export interface GristServer {
   getHosts(): Hosts;
   getHomeDBManager(): HomeDBManager;
   getStorageManager(): IDocStorageManager;
+  getTelemetryManager(): TelemetryManager|undefined;
   getNotifier(): INotifier;
   getDocTemplate(): Promise<DocTemplate>;
   getTag(): string;
@@ -118,6 +120,7 @@ export function createDummyGristServer(): GristServer {
     getHosts() { throw new Error('no hosts'); },
     getHomeDBManager() { throw new Error('no db'); },
     getStorageManager() { throw new Error('no storage manager'); },
+    getTelemetryManager() { return undefined; },
     getNotifier() { throw new Error('no notifier'); },
     getDocTemplate() { throw new Error('no doc template'); },
     getTag() { return 'tag'; },

app/server/lib/Triggers.ts

@@ -647,6 +647,7 @@ export class DocTriggers {
         await this._stats.logStatus(id, 'sending');
         meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
         this._log("Sending batch of webhook events", meta);
+        this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', meta);
         success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
         if (this._loopAbort.signal.aborted) {
           continue;

app/server/lib/initialDocSql.ts

@@ -6,7 +6,7 @@ export const GRIST_DOC_SQL = `
 PRAGMA foreign_keys=OFF;
 BEGIN TRANSACTION;
 CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
-INSERT INTO _grist_DocInfo VALUES(1,'','','',36,'','');
+INSERT INTO _grist_DocInfo VALUES(1,'','','',37,'','');
 CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0);
 CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
 CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS "_grist_Views_section" (id INTEGER PRIMARY KEY, "tabl
 CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colRef" INTEGER DEFAULT 0, "width" INTEGER DEFAULT 0, "widgetOptions" TEXT DEFAULT '', "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '', "rules" TEXT DEFAULT NULL);
 CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
 CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
-CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
+CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
 CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
 CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
 INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');
@@ -43,7 +43,7 @@ export const GRIST_DOC_WITH_TABLE1_SQL = `
 PRAGMA foreign_keys=OFF;
 BEGIN TRANSACTION;
 CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
-INSERT INTO _grist_DocInfo VALUES(1,'','','',36,'','');
+INSERT INTO _grist_DocInfo VALUES(1,'','','',37,'','');
 CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0);
 INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0,2);
 CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
@@ -74,7 +74,7 @@ INSERT INTO _grist_Views_section_field VALUES(5,2,5,3,0,'',0,0,'',NULL);
 INSERT INTO _grist_Views_section_field VALUES(6,2,6,4,0,'',0,0,'',NULL);
 CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
 CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
-CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
+CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
 CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
 CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
 INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');

app/server/lib/requestUtils.ts

@@ -260,7 +260,7 @@ export function optStringParam(p: any): string|undefined {
   return undefined;
 }
 
-export function stringParam(p: any, name: string, allowed?: string[]): string {
+export function stringParam(p: any, name: string, allowed?: readonly string[]): string {
   if (typeof p !== 'string') {
     throw new ApiError(`${name} parameter should be a string: ${p}`, 400);
   }

app/server/mergedServerMain.ts

@@ -125,11 +125,13 @@ export async function main(port: number, serverTypes: ServerType[],
     server.addBillingPages();
     server.addWelcomePaths();
     server.addLogEndpoint();
+    server.addTelemetryEndpoint();
     server.addGoogleAuthEndpoint();
   }
 
   if (includeDocs) {
     server.addJsonSupport();
+    server.addTelemetryEndpoint();
     await server.addDoc();
   }
 

sandbox/grist/migrations.py

@@ -1196,3 +1196,11 @@ def migration36(tdset):
   Add description to column
   """
   return tdset.apply_doc_actions([add_column('_grist_Tables_column', 'description', 'Text')])
+
+
+@migration(schema_version=37)
+def migration37(tdset):
+  """
+  Add fileExt column to _grist_Attachments.
+  """
+  return tdset.apply_doc_actions([add_column('_grist_Attachments', 'fileExt', 'Text')])

sandbox/grist/schema.py

@@ -15,7 +15,7 @@ import six
 
 import actions
 
-SCHEMA_VERSION = 36
+SCHEMA_VERSION = 37
 
 def make_column(col_id, col_type, formula='', isFormula=False):
   return {
@@ -236,6 +236,7 @@ def schema_create_actions():
       make_column("fileName",     "Text"), # User defined file name
       make_column("fileType",     "Text"), # A string indicating the MIME type of the data
       make_column("fileSize",     "Int"),  # The size in bytes
+      make_column("fileExt",      "Text"), # The file extension
       make_column("imageHeight",  "Int"),  # height in pixels
       make_column("imageWidth",   "Int"),  # width in pixels
       make_column("timeDeleted",  "DateTime"),

stubs/app/server/lib/TelemetryManager.ts

@@ -0,0 +1,11 @@
+import {TelemetryEventName} from 'app/common/Telemetry';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+
+export class TelemetryManager {
+  constructor(_dbManager: HomeDBManager) {}
+
+  public logEvent(
+    _name: TelemetryEventName,
+    _metadata?: Record<string, any>
+  ) {}
+}

test/server/lib/Authorizer.ts

@@ -34,6 +34,7 @@ async function activateServer(home: FlexServer, docManager: DocManager) {
   home.addJsonSupport();
   await home.addLandingPages();
   home.addHomeApi();
+  home.addTelemetryEndpoint();
   await home.addDoc();
   home.addApiErrorHandlers();
   serverUrl = home.getOwnUrl();