(core) Add more telemetry events

Summary: Adds new telemetry events and a flag for whether an event originated from a team site.

Test Plan: Manual.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: jarek, dsagal

Differential Revision: https://phab.getgrist.com/D4105

app/common/Telemetry.ts

@@ -745,6 +745,7 @@ export const TelemetryContracts: TelemetryContracts = {
     },
   },
   signupFirstVisit: {
+    category: 'ProductVisits',
     description: 'Triggered when a new user first opens the Grist app',
     minimumTelemetryLevel: Level.full,
     retentionPeriod: 'indefinitely',
@@ -1058,6 +1059,86 @@ export const TelemetryContracts: TelemetryContracts = {
       },
     },
   },
+  invitedMember: {
+    category: 'TeamSite',
+    description: 'Triggered when users are added to a team site.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      count: {
+        description: 'The number of users added.',
+        dataType: 'number',
+      },
+      siteId: {
+        description: 'The id of the site.',
+        dataType: 'number',
+      },
+    },
+  },
+  uninvitedMember: {
+    category: 'TeamSite',
+    description: 'Triggered when users are removed from a team site.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      count: {
+        description: 'The number of users removed.',
+        dataType: 'number',
+      },
+      siteId: {
+        description: 'The id of the site.',
+        dataType: 'number',
+      },
+    },
+  },
+  invitedDocUser: {
+    category: 'DocumentUsage',
+    description: 'Triggered when users are added to a document.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      access: {
+        description: 'The access level granted to the added users.',
+        dataType: 'string',
+      },
+      count: {
+        description: 'The number of users added.',
+        dataType: 'number',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+    },
+  },
+  madeDocPublic: {
+    category: 'DocumentUsage',
+    description: 'Triggered when public access to a document is enabled.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      access: {
+        description: 'The access level granted to public users.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+    },
+  },
+  madeDocPrivate: {
+    category: 'DocumentUsage',
+    description: 'Triggered when public access to a document is disabled.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+    },
+  },
   openedTemplate: {
     category: 'TemplateUsage',
     description: 'Triggered when a template is opened.',
@@ -1098,6 +1179,26 @@ export const TelemetryContracts: TelemetryContracts = {
       },
     },
   },
+  copiedTemplate: {
+    category: 'TemplateUsage',
+    description: 'Triggered when a copy of a template is saved.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      templateId: {
+        description: 'The document id of the template.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
   subscribedToPlan: {
     category: 'SubscriptionPlan',
     description: 'Triggered on subscription to a plan.',
@@ -1138,7 +1239,7 @@ export const TelemetryContracts: TelemetryContracts = {
     metadataContracts: {
       workspaceId: {
         description: 'The id of the workspace.',
-        dataType: 'string',
+        dataType: 'number',
       },
       userId: {
         description: 'The id of the user that triggered this event.',
@@ -1162,6 +1263,139 @@ export const TelemetryContracts: TelemetryContracts = {
       },
     },
   },
+  visitedPage: {
+    category: 'ProductVisits',
+    description: 'Triggered when a page is loaded.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id. Only included on visits to doc pages.',
+        dataType: 'string',
+      },
+      url: {
+        description: 'The URL of the visited page. Link keys, doc ids, and other identifiers ' +
+          'are excluded from the URL.',
+        dataType: 'string',
+      },
+      path: {
+        description: 'The path of the visited page (e.g. "app.html").',
+        dataType: 'string',
+      },
+      userAgent: {
+        description: 'The User-Agent HTTP request header.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  openedDoc: {
+    category: 'DocumentUsage',
+    description: 'Triggered when a document is opened.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  'createdDoc-Empty': {
+    category: 'DocumentUsage',
+    description: 'Triggered when a new empty document is created.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  'createdDoc-FileImport': {
+    category: 'DocumentUsage',
+    description: 'Triggered when a document is created via file import.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  'createdDoc-CopyTemplate': {
+    category: 'DocumentUsage',
+    description: 'Triggered when a document is created by saving a copy of a template.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  'createdDoc-CopyDoc': {
+    category: 'DocumentUsage',
+    description: 'Triggered when a document is created by saving a copy of a document.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
 };
 
 type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
@@ -1199,12 +1433,24 @@ export const TelemetryEvents = StringUnion(
   'deletedAccount',
   'createdSite',
   'deletedSite',
+  'invitedMember',
+  'uninvitedMember',
+  'invitedDocUser',
+  'madeDocPublic',
+  'madeDocPrivate',
   'openedTemplate',
   'openedTemplateTour',
+  'copiedTemplate',
   'subscribedToPlan',
   'cancelledPlan',
   'createdWorkspace',
   'deletedWorkspace',
+  'visitedPage',
+  'openedDoc',
+  'createdDoc-Empty',
+  'createdDoc-FileImport',
+  'createdDoc-CopyTemplate',
+  'createdDoc-CopyDoc',
 );
 export type TelemetryEvent = typeof TelemetryEvents.type;
 
@@ -1216,7 +1462,8 @@ type TelemetryEventCategory =
   | 'Welcome'
   | 'SubscriptionPlan'
   | 'DocumentUsage'
-  | 'TeamSite';
+  | 'TeamSite'
+  | 'ProductVisits';
 
 interface TelemetryEventContract {
   description: string;

app/gen-server/ApiServer.ts

@@ -6,7 +6,8 @@ import {Request} from 'express';
 
 import {ApiError} from 'app/common/ApiError';
 import {FullUser} from 'app/common/LoginSessionAPI';
-import {OrganizationProperties} from 'app/common/UserAPI';
+import {BasicRole} from 'app/common/roles';
+import {OrganizationProperties, PermissionDelta} from 'app/common/UserAPI';
 import {User} from 'app/gen-server/entity/User';
 import {BillingOptions, HomeDBManager, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
 import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
@@ -235,7 +236,7 @@ export class ApiServer {
         const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
         this._gristServer.getTelemetry().logEvent(mreq, 'deletedWorkspace', {
           full: {
-            workspaceId: wsId,
+            workspaceId: query.data,
             userId: mreq.userId,
           },
         });
@@ -261,9 +262,10 @@ export class ApiServer {
       const mreq = req as RequestWithLogin;
       const wsId = integerParam(req.params.wid, 'wid');
       const query = await this._dbManager.addDocument(getScope(req), wsId, req.body);
+      const docId = query.data!;
       this._gristServer.getTelemetry().logEvent(mreq, 'documentCreated', {
         limited: {
-          docIdDigest: query.data!,
+          docIdDigest: docId,
           sourceDocIdDigest: undefined,
           isImport: false,
           fileType: undefined,
@@ -274,6 +276,13 @@ export class ApiServer {
           altSessionId: mreq.altSessionId,
         },
       });
+      this._gristServer.getTelemetry().logEvent(mreq, 'createdDoc-Empty', {
+        full: {
+          docIdDigest: docId,
+          userId: mreq.userId,
+          altSessionId: mreq.altSessionId,
+        },
+      });
       return sendReply(req, res, query);
     }));
 
@@ -345,6 +354,7 @@ export class ApiServer {
     this._app.patch('/api/docs/:did/access', expressWrap(async (req, res) => {
       const delta = req.body.delta;
       const query = await this._dbManager.updateDocPermissions(getDocScope(req), delta);
+      this._logInvitedDocUserTelemetryEvents(req as RequestWithLogin, delta);
       return sendReply(req, res, query);
     }));
 
@@ -613,6 +623,47 @@ export class ApiServer {
     const extendedScope = addPermit(scope, this._dbManager.getSupportUserId(), {org});
     return await op(extendedScope);
   }
+
+  private _logInvitedDocUserTelemetryEvents(mreq: RequestWithLogin, delta: PermissionDelta) {
+    if (!delta.users) { return; }
+
+    const numInvitedUsersByAccess: Record<BasicRole, number> = {
+      'viewers': 0,
+      'editors': 0,
+      'owners': 0,
+    };
+    for (const [email, access] of Object.entries(delta.users)) {
+      if (email === 'everyone@getgrist.com') { continue; }
+      if (access === null || access === 'members') { continue; }
+
+      numInvitedUsersByAccess[access] += 1;
+    }
+    for (const [access, count] of Object.entries(numInvitedUsersByAccess)) {
+      if (count === 0) { continue; }
+
+      this._gristServer.getTelemetry().logEvent(mreq, 'invitedDocUser', {
+        full: {
+          access,
+          count,
+          userId: mreq.userId,
+        },
+      });
+    }
+
+    const publicAccess = delta.users['everyone@getgrist.com'];
+    if (publicAccess !== undefined) {
+      this._gristServer.getTelemetry().logEvent(
+        mreq,
+        publicAccess ? 'madeDocPublic' : 'madeDocPrivate',
+        {
+          full: {
+            ...(publicAccess ? {access: publicAccess} : {}),
+            userId: mreq.userId,
+          },
+        }
+      );
+    }
+  }
 }
 
 /**

app/server/lib/DocApi.ts

@@ -1185,6 +1185,11 @@ export class DocWorkerApi {
           },
         },
       });
+      this._logCreatedFileImportDocTelemetryEvent(req, {
+        full: {
+          docIdDigest: result.id,
+        },
+      });
       res.json(result);
     }));
 
@@ -1319,6 +1324,11 @@ export class DocWorkerApi {
           },
         });
         docId = result.id;
+        this._logCreatedFileImportDocTelemetryEvent(req, {
+          full: {
+            docIdDigest: docId,
+          },
+        });
       } else if (workspaceId !== undefined) {
         docId = await this._createNewSavedDoc(req, {
           workspaceId: workspaceId,
@@ -1376,6 +1386,30 @@ export class DocWorkerApi {
         },
       },
     });
+
+    const sourceDocument = await this._dbManager.getRawDocById(sourceDocumentId);
+    const isTemplateCopy = sourceDocument.type === 'template';
+    if (isTemplateCopy) {
+      this._grist.getTelemetry().logEvent(mreq, 'copiedTemplate', {
+        full: {
+          templateId: parseUrlId(sourceDocument.urlId || sourceDocument.id).trunkId,
+          userId: mreq.userId,
+          altSessionId: mreq.altSessionId,
+        },
+      });
+    }
+    this._grist.getTelemetry().logEvent(
+      mreq,
+      `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`,
+      {
+        full: {
+          docIdDigest: result.id,
+          userId: mreq.userId,
+          altSessionId: mreq.altSessionId,
+        },
+      }
+    );
+
     return result.id;
   }
 
@@ -1387,19 +1421,25 @@ export class DocWorkerApi {
     const {status, data, errMessage} = await this._dbManager.addDocument(getScope(req), workspaceId, {
       name: documentName ?? 'Untitled document',
     });
+    const docId = data!;
     if (status !== 200) {
       throw new ApiError(errMessage || 'unable to create document', status);
     }
     this._logDocumentCreatedTelemetryEvent(req, {
       limited: {
-        docIdDigest: data!,
+        docIdDigest: docId,
         sourceDocIdDigest: undefined,
         isImport: false,
         fileType: undefined,
         isSaved: true,
       },
     });
-    return data!;
+    this._logCreatedEmptyDocTelemetryEvent(req, {
+      full: {
+        docIdDigest: docId,
+      },
+    });
+    return docId;
   }
 
   private async _createNewUnsavedDoc(req: Request, options: {
@@ -1431,6 +1471,11 @@ export class DocWorkerApi {
         isSaved: false,
       },
     });
+    this._logCreatedEmptyDocTelemetryEvent(req, {
+      full: {
+        docIdDigest: docId,
+      },
+    });
     return docId;
   }
 
@@ -1444,6 +1489,28 @@ export class DocWorkerApi {
     }, metadata));
   }
 
+  private _logCreatedEmptyDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
+    this._logCreatedDocTelemetryEvent(req, 'createdDoc-Empty', metadata);
+  }
+
+  private _logCreatedFileImportDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
+    this._logCreatedDocTelemetryEvent(req, 'createdDoc-FileImport', metadata);
+  }
+
+  private _logCreatedDocTelemetryEvent(
+    req: Request,
+    event: 'createdDoc-Empty' | 'createdDoc-FileImport',
+    metadata: TelemetryMetadataByLevel,
+  ) {
+    const mreq = req as RequestWithLogin;
+    this._grist.getTelemetry().logEvent(mreq, event, _.merge({
+      full: {
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+      },
+    }, metadata));
+  }
+
   /**
    * Check for read access to the given document, and return its
    * canonical docId.  Throws error if read access not available.

app/server/lib/DocManager.ts

@@ -301,13 +301,14 @@ export class DocManager extends EventEmitter {
                        openMode: OpenDocMode = 'default',
                        linkParameters: Record<string, string> = {}): Promise<OpenLocalDocResult> {
     let auth: Authorizer;
+    let userId: number | undefined;
     const dbManager = this._homeDbManager;
     if (!isSingleUserMode()) {
       if (!dbManager) { throw new Error("HomeDbManager not available"); }
       // Sets up authorization of the document.
       const org = client.getOrg();
       if (!org) { throw new Error('Documents can only be opened in the context of a specific organization'); }
-      const userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId();
+      userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId();
       const userRef = await client.getUserRef(dbManager);
 
       // We use docId in the key, and disallow urlId, so we can be sure that we are looking at the
@@ -397,6 +398,14 @@ export class DocManager extends EventEmitter {
         this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));
       }
 
+      this.gristServer.getTelemetry().logEvent(docSession, 'openedDoc', {
+        full: {
+          docIdDigest: docId,
+          userId,
+          altSessionId: client.getAltSessionId(),
+        },
+      });
+
       return {activeDoc, result};
     });
   }

app/server/lib/Telemetry.ts

@@ -252,13 +252,23 @@ export class Telemetry implements ITelemetry {
     metadata?: TelemetryMetadata
   ) {
     let isInternalUser: boolean | undefined;
+    let isTeamSite: boolean | undefined;
     if (requestOrSession) {
-      const email = ('get' in requestOrSession)
+      let email: string | undefined;
-        ? requestOrSession.user?.loginEmail
+      let org: string | undefined;
-        : getDocSessionUser(requestOrSession)?.email;
+      if ('get' in requestOrSession) {
+        email = requestOrSession.user?.loginEmail;
+        org = requestOrSession.org;
+      } else {
+        email = getDocSessionUser(requestOrSession)?.email;
+        org = requestOrSession.client?.getOrg() ?? requestOrSession.req?.org;
+      }
       if (email) {
         isInternalUser = email !== 'anon@getgrist.com' && email.endsWith('@getgrist.com');
       }
+      if (org && !process.env.GRIST_SINGLE_ORG) {
+        isTeamSite = !this._dbManager.isMergedOrg(org);
+      }
     }
     const {category: eventCategory} = TelemetryContracts[event];
     this._telemetryLogger.rawLog('info', getEventType(event), event, {
@@ -268,6 +278,7 @@ export class Telemetry implements ITelemetry {
       eventSource: `grist-${this._deploymentType}`,
       installationId: this._activation!.id,
       ...(isInternalUser !== undefined ? {isInternalUser} : undefined),
+      ...(isTeamSite !== undefined ? {isTeamSite} : undefined),
     });
   }
 

app/server/lib/sendAppPage.ts

@@ -116,13 +116,13 @@ export function makeSendAppPage(opts: {
 }) {
   const {server, staticDir, tag, testLogin} = opts;
   return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
-      const config = makeGristConfig({
+    const config = makeGristConfig({
-        homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
+      homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
-        extra: options.config,
+      extra: options.config,
-        baseDomain: opts.baseDomain,
+      baseDomain: opts.baseDomain,
-        req,
+      req,
-        server,
+      server,
-      });
+    });
 
     // We could cache file contents in memory, but the filesystem does caching too, and compared
     // to that, the performance gain is unlikely to be meaningful. So keep it simple here.
@@ -156,10 +156,46 @@ export function makeSendAppPage(opts: {
         "<!-- INSERT CONFIG -->",
         `<script>window.gristConfig = ${jsesc(config, {isScriptContext: true, json: true})};</script>`
       );
+    logVisitedPageTelemetryEvent(req as RequestWithLogin, {
+      server,
+      pagePath: options.path,
+      docId: config.assignmentId,
+    });
     resp.status(options.status).type('html').send(content);
   };
 }
 
+interface LogVisitedPageEventOptions {
+  server: GristServer;
+  pagePath: string;
+  docId?: string;
+}
+
+function logVisitedPageTelemetryEvent(req: RequestWithLogin, options: LogVisitedPageEventOptions) {
+  const {server, pagePath, docId} = options;
+
+  // Construct a fake URL and append the utm_* parameters from the original URL.
+  // We avoid using the original URL here because it may contain sensitive identifiers,
+  // such as link key parameters and site/doc ids.
+  const url = new URL('fake', server.getMergedOrgUrl(req));
+  for (const [key, value] of Object.entries(req.query)) {
+    if (key.startsWith('utm_')) {
+      url.searchParams.set(key, String(value));
+    }
+  }
+
+  server.getTelemetry().logEvent(req, 'visitedPage', {
+    full: {
+      docIdDigest: docId,
+      url: url.toString(),
+      path: pagePath,
+      userAgent: req.headers['user-agent'],
+      userId: req.userId,
+      altSessionId: req.altSessionId,
+    },
+  });
+}
+
 function shouldSupportAnon() {
   // Enable UI for anonymous access if a flag is explicitly set in the environment
   return process.env.GRIST_SUPPORT_ANON === "true";