(core) Tweak telemetry

Summary: Adjusts the level of telemetry collected from Grist SaaS.

Test Plan: Tested manually.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3899

app/server/lib/ActiveDoc.ts

@@ -68,6 +68,7 @@ import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccess
 import {isHiddenCol} from 'app/common/gristTypes';
 import {commonUrls, parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
+import {hashId} from 'app/common/hashingUtils';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {Interval} from 'app/common/Interval';
 import * as roles from 'app/common/roles';
@@ -131,10 +132,12 @@ import {createAttachmentsIndex, DocStorage, REMOVE_UNUSED_ATTACHMENTS_DELAY} fro
 import {expandQuery} from './ExpandedQuery';
 import {GranularAccess, GranularAccessForBundle} from './GranularAccess';
 import {OnDemandActions} from './OnDemandActions';
-import {getLogMetaFromDocSession, getPubSubPrefix, timeoutReached} from './serverUtils';
+import {getLogMetaFromDocSession, getPubSubPrefix, getTelemetryMetaFromDocSession,
+        timeoutReached} from './serverUtils';
 import {findOrAddAllEnvelope, Sharing} from './Sharing';
 import cloneDeep = require('lodash/cloneDeep');
 import flatten = require('lodash/flatten');
+import merge = require('lodash/merge');
 import pick = require('lodash/pick');
 import remove = require('lodash/remove');
 import sum = require('lodash/sum');
@@ -1389,10 +1392,9 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
       // TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
       const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
       this.logTelemetryEvent(docSession, 'documentForked', {
-        forkId: forkIds.forkId,
-        forkDocId: forkIds.docId,
-        forkUrlId: forkIds.urlId,
-        trunkId: doc.trunkId,
+        forkIdDigest: hashId(forkIds.forkId),
+        forkDocIdDigest: hashId(forkIds.docId),
+        trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
         isTemplate,
         lastActivity: doc.updatedAt,
       });
@@ -2314,7 +2316,6 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
   private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
     this.logTelemetryEvent(docSession, 'documentUsage', {
       triggeredBy,
-      access: this._doc?.access,
       isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
       rowCount: this._docUsage?.rowCount?.total,
       dataSizeBytes: this._docUsage?.dataSizeBytes,
@@ -2434,22 +2435,22 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
   private _getCustomWidgetMetrics() {
     const viewSections = this.docData?.getMetaTable('_grist_Views_section');
     const viewSectionRecords = viewSections?.getRecords() ?? [];
-    const customWidgetUrls: string[] = [];
+    const customWidgetIds: string[] = [];
     for (const r of viewSectionRecords) {
       const {customView} = safeJsonParse(r.options, {});
       if (!customView) { continue; }
 
-      const {url} = safeJsonParse(customView, {});
+      const {pluginId, url} = safeJsonParse(customView, {});
       if (!url) { continue; }
 
-      const isGristUrl = url.startsWith(commonUrls.gristLabsCustomWidgets);
-      customWidgetUrls.push(isGristUrl ? url : 'externalURL');
+      const isGristLabsWidget = url.startsWith(commonUrls.gristLabsCustomWidgets);
+      customWidgetIds.push(isGristLabsWidget ? pluginId : 'externalId');
     }
-    const numCustomWidgets = customWidgetUrls.length;
+    const numCustomWidgets = customWidgetIds.length;
 
     return {
       numCustomWidgets,
-      customWidgetUrls,
+      customWidgetIds,
     };
   }
 
@@ -2511,15 +2512,16 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
   }
 
   private _getTelemetryMeta(docSession: OptDocSession|null) {
-    return {
-      ...(docSession ? {
-          ...getLogMetaFromDocSession(docSession),
-          altSessionId: getDocSessionAltSessionId(docSession),
-        } : {}),
-      docId: this._docName,
-      siteId: this._doc?.workspace.org.id,
-      siteType: this._product?.name,
-    };
+    const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined;
+    return merge(
+      docSession ? getTelemetryMetaFromDocSession(docSession) : {},
+      altSessionId ? {altSessionId} : undefined,
+      {
+        docIdDigest: hashId(this._docName),
+        siteId: this._doc?.workspace.org.id,
+        siteType: this._product?.name,
+      },
+    );
   }
 
   /**

app/server/lib/AppEndpoint.ts

@@ -9,6 +9,7 @@ import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
 import {ApiError} from 'app/common/ApiError';
 import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
+import {hashId} from 'app/common/hashingUtils';
 import {LocalPlugin} from "app/common/plugin";
 import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
 import {Document as APIDocument} from 'app/common/UserAPI';
@@ -308,7 +309,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
     const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
     if (isPublic || isTemplate) {
       gristServer.getTelemetryManager()?.logEvent('documentOpened', {
-        docId,
+        docIdDigest: hashId(docId),
         siteId: doc.workspace.org.id,
         siteType: doc.workspace.org.billingAccount.product.name,
         userId: mreq.userId,

app/server/lib/Authorizer.ts

@@ -397,7 +397,8 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
   log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
   if (hasApiKey) {
     options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
-      ...meta,
+      method: mreq.method,
+      userId: mreq.userId,
       userAgent: req.headers['user-agent'],
     });
   }

app/server/lib/Client.ts

@@ -433,6 +433,16 @@ export class Client {
     return meta;
   }
 
+  public getFullTelemetryMeta() {
+    const meta: Record<string, any> = {};
+    // We assume the _userId has already been cached, which will be true always (for all practical
+    // purposes) because it's set when the Authorizer checks this client.
+    if (this._userId) { meta.userId = this._userId; }
+    const altSessionId = this.getAltSessionId();
+    if (altSessionId) { meta.altSessionId = altSessionId; }
+    return meta;
+  }
+
   private async _refreshUser(dbManager: HomeDBManager) {
     if (this._profile) {
       const user = await this._fetchUser(dbManager);

app/server/lib/DocApi.ts

@@ -11,6 +11,7 @@ import {
 import {isRaisedException} from "app/common/gristTypes";
 import {buildUrlId, parseUrlId} from "app/common/gristUrls";
 import {isAffirmative} from "app/common/gutil";
+import {hashId} from "app/common/hashingUtils";
 import {SchemaTypes} from "app/common/schema";
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
@@ -915,9 +916,8 @@ export class DocWorkerApi {
           });
           const {forkId} = parseUrlId(scope.urlId);
           activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
-            tutorialForkId: forkId,
-            tutorialForkUrlId: scope.urlId,
-            tutorialTrunkId,
+            tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
+            tutorialTrunkIdDigest: hashId(tutorialTrunkId),
           });
         }
       }

app/server/lib/FlexServer.ts

@@ -697,9 +697,7 @@ export class FlexServer implements GristServer {
       const name = stringParam(req.body.name, 'name', TelemetryEventNames);
       this._telemetryManager?.logEvent(name as TelemetryEventName, {
         userId: mreq.userId,
-        email: mreq.user?.loginEmail,
         altSessionId: mreq.altSessionId,
-        site: mreq.org,
         ...req.body.metadata,
       });
       return resp.status(200).send();

app/server/lib/Triggers.ts

@@ -619,7 +619,9 @@ export class DocTriggers {
         await this._stats.logStatus(id, 'sending');
         meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
         this._log("Sending batch of webhook events", meta);
-        this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', meta);
+        this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
+          numEvents: batch.length,
+        });
         success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
         if (this._loopAbort.signal.aborted) {
           continue;

app/server/lib/serverUtils.ts

@@ -163,6 +163,20 @@ export function getLogMetaFromDocSession(docSession: OptDocSession) {
   };
 }
 
+/**
+ * Extract telemetry metadata from session.
+ */
+export function getTelemetryMetaFromDocSession(docSession: OptDocSession) {
+  const client = docSession.client;
+  const access = getDocSessionAccessOrNull(docSession);
+  const user = getDocSessionUser(docSession);
+  return {
+    access,
+    ...(user ? {userId: user.id} : {}),
+    ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+  };
+}
+
 /**
  * Only offer choices of engine on experimental deployments (staging/dev).
  */