gristlabs_grist-core/app/server/lib/Telemetry.ts
Paul Fitzpatrick c5e750abc6 (core) add a cli command to view telemetry settings
Summary:
This adds a `yarn cli settings telemetry [--json] [--all]` command
that allows telemetry settings to be inspected. It is useful for
keeping documentation about telemetry up to date.

Test Plan:
manual (a bit cheeky; justified on basis of breakage
not being very important yet, this is essentially an internal
feature)

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D3917
2023-06-12 09:58:38 -04:00

227 lines
7.5 KiB
TypeScript

import {ApiError} from 'app/common/ApiError';
import {
buildTelemetryEventChecker,
filterMetadata,
removeNullishKeys,
TelemetryEvent,
TelemetryEventChecker,
TelemetryEvents,
TelemetryLevel,
TelemetryLevels,
TelemetryMetadata,
TelemetryMetadataByLevel,
} from 'app/common/Telemetry';
import {HomeDBManager, HomeDBTelemetryEvents} from 'app/gen-server/lib/HomeDBManager';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {GristServer} from 'app/server/lib/GristServer';
import {LogMethods} from 'app/server/lib/LogMethods';
import {stringParam} from 'app/server/lib/requestUtils';
import * as express from 'express';
import merge = require('lodash/merge');
export interface ITelemetry {
logEvent(name: TelemetryEvent, metadata?: TelemetryMetadataByLevel): Promise<void>;
addEndpoints(app: express.Express): void;
getTelemetryLevel(): TelemetryLevel;
}
const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25;
/**
* Manages telemetry for Grist.
*/
export class Telemetry implements ITelemetry {
private _telemetryLevel: TelemetryLevel;
private _deploymentType = this._gristServer.getDeploymentType();
private _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
private _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
'https://telemetry.getgrist.com/api/telemetry';
private _numPendingForwardEventRequests = 0;
private _installationId: string | undefined;
private _logger = new LogMethods('Telemetry ', () => ({}));
private _telemetryLogger = new LogMethods('Telemetry ', () => ({
eventType: 'telemetry',
}));
private _checkEvent: TelemetryEventChecker | undefined;
constructor(private _dbManager: HomeDBManager, private _gristServer: GristServer) {
this._initialize().catch((e) => {
this._logger.error(undefined, 'failed to initialize', e);
});
}
/**
* Logs a telemetry `event` and its `metadata`.
*
* Depending on the deployment type, this will either forward the
* data to an endpoint (set via GRIST_TELEMETRY_URL) or log it
* directly. In hosted Grist, telemetry is logged directly, and
* subsequently sent to an OpenSearch instance via CloudWatch. In
* other deployment types, telemetry is forwarded to an endpoint
* of hosted Grist, which then handles logging to OpenSearch.
*
* Note that `metadata` is grouped by telemetry level, with only the
* groups meeting the current telemetry level being included in
* what's logged. If the current telemetry level is `off`, nothing
* will be logged. Otherwise, `metadata` will be filtered according
* to the current telemetry level, keeping only the groups that are
* less than or equal to the current level.
*
* Additionally, runtime checks are also performed to verify that the
* event and metadata being passed in are being logged appropriately
* for the configured telemetry level. If any checks fail, an error
* is thrown.
*
* Example:
*
* The following will only log the `rowCount` if the telemetry level is set
* to `limited`, and will log both the `method` and `userId` if the telemetry
* level is set to `full`:
*
* ```
* logEvent('documentUsage', {
* limited: {
* rowCount: 123,
* },
* full: {
* userId: 1586,
* },
* });
* ```
*/
public async logEvent(
event: TelemetryEvent,
metadata?: TelemetryMetadataByLevel
) {
if (this._telemetryLevel === 'off') { return; }
metadata = filterMetadata(metadata, this._telemetryLevel);
this._checkTelemetryEvent(event, metadata);
if (this._shouldForwardTelemetryEvents) {
await this._forwardEvent(event, metadata);
} else {
this._telemetryLogger.rawLog('info', null, event, {
eventName: event,
eventSource: `grist-${this._deploymentType}`,
...metadata,
});
}
}
public addEndpoints(app: express.Application) {
/**
* Logs telemetry events and their metadata.
*
* Clients of this endpoint may be external Grist instances, so the behavior
* varies based on the presence of an `eventSource` key in the event metadata.
*
* If an `eventSource` key is present, the telemetry event will be logged
* directly, as the request originated from an external source; runtime checks
* of telemetry data are skipped since they should have already occured at the
* source. Otherwise, the event will only be logged after passing various
* checks.
*/
app.post('/api/telemetry', async (req, resp) => {
const mreq = req as RequestWithLogin;
const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
if ('eventSource' in req.body.metadata) {
this._telemetryLogger.rawLog('info', null, event, {
eventName: event,
...(removeNullishKeys(req.body.metadata)),
});
} else {
try {
await this.logEvent(event as TelemetryEvent, merge(
{
limited: {
eventSource: `grist-${this._deploymentType}`,
...(this._deploymentType !== 'saas' ? {installationId: this._installationId} : {}),
},
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
},
req.body.metadata,
));
} catch (e) {
this._logger.error(undefined, `failed to log telemetry event ${event}`, e);
throw new ApiError(`Telemetry failed to log telemetry event ${event}`, 500);
}
}
return resp.status(200).send();
});
}
public getTelemetryLevel() {
return this._telemetryLevel;
}
private async _initialize() {
this._telemetryLevel = getTelemetryLevel();
if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryLevel);
}
const {id} = await this._gristServer.getActivations().current();
this._installationId = id;
for (const event of HomeDBTelemetryEvents.values) {
this._dbManager.on(event, async (metadata) => {
this.logEvent(event, metadata).catch(e =>
this._logger.error(undefined, `failed to log telemetry event ${event}`, e));
});
}
}
private _checkTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadata) {
if (!this._checkEvent) {
throw new Error('Telemetry._checkEvent is undefined');
}
this._checkEvent(event, metadata);
}
private async _forwardEvent(
event: TelemetryEvent,
metadata?: TelemetryMetadata
) {
if (this._numPendingForwardEventRequests === MAX_PENDING_FORWARD_EVENT_REQUESTS) {
this._logger.warn(undefined, 'exceeded the maximum number of pending forwardEvent calls '
+ `(${MAX_PENDING_FORWARD_EVENT_REQUESTS}). Skipping forwarding of event ${event}.`);
return;
}
try {
this._numPendingForwardEventRequests += 1;
await this._postJsonPayload(JSON.stringify({event, metadata}));
} catch (e) {
this._logger.error(undefined, `failed to forward telemetry event ${event}`, e);
} finally {
this._numPendingForwardEventRequests -= 1;
}
}
private async _postJsonPayload(payload: string) {
await fetch(this._forwardTelemetryEventsUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: payload,
});
}
}
export function getTelemetryLevel(): TelemetryLevel {
if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
return TelemetryLevels.check(process.env.GRIST_TELEMETRY_LEVEL);
} else {
return 'off';
}
}