Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) Add optional telemetry to grist-core

Browse Source 
Summary:
Adds support for optional telemetry to grist-core.

A new environment variable, GRIST_TELEMETRY_LEVEL, controls the level of telemetry collected.

Test Plan: Server and unit tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: dsagal, anaisconce

Differential Revision: https://phab.getgrist.com/D3880
This commit is contained in:
George Gevoian 2023-06-06 13:08:50 -04:00
parent 0d082c9cfc
commit 10f5f0cb37
38 changed files with 2177 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -285,6 +285,7 @@ GRIST_SINGLE_ORG | set to an org "domain" to pin client to that org
GRIST_HELP_CENTER | set the help center link ref
GRIST_SUPPORT_ANON | if set to 'true', show UI for anonymous access (not shown by default)
GRIST_SUPPORT_EMAIL | if set, give a user with the specified email support powers. The main extra power is the ability to share sites, workspaces, and docs with all users in a listed way.
GRIST_TELEMETRY_LEVEL | the telemetry level. Can be set to: `off` (default), `limited`, or `full`.
GRIST_THROTTLE_CPU | if set, CPU throttling is enabled
GRIST_USER_ROOT | an extra path to look for plugins in.
GRIST_UI_FEATURES | comma-separated list of UI features to enable. Allowed names of parts: `helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,tutorials`. If a part also exists in GRIST_HIDE_UI_ELEMENTS, it won't be enabled.

4
app/client/lib/helpScout.ts
View File

@ -197,11 +197,11 @@ function _beaconOpen(userObj: IUserObj|null, options: IBeaconOpenOptions) {
Beacon('once', 'open', () => logTelemetryEvent('beaconOpen'));
Beacon('on', 'article-viewed', (article) => logTelemetryEvent('beaconArticleViewed', {
articleId: article!.id,
full: {articleId: article!.id},
}));
Beacon('on', 'email-sent', () => logTelemetryEvent('beaconEmailSent'));
Beacon('on', 'search', (search) => logTelemetryEvent('beaconSearch', {
searchQuery: search!.query,
full: {searchQuery: search!.query},
}));
}

16
app/client/lib/telemetry.ts
View File

@ -1,14 +1,20 @@
import {logError} from 'app/client/models/errors';
import {TelemetryEventName} from 'app/common/Telemetry';
import {fetchFromHome, pageHasHome} from 'app/common/urlUtils';
import {Level, TelemetryContracts, TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
import {fetchFromHome, getGristConfig, pageHasHome} from 'app/common/urlUtils';
export function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<string, any>) {
export function logTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadataByLevel) {
if (!pageHasHome()) { return; }
const {telemetry} = getGristConfig();
if (!telemetry) { return; }
const {telemetryLevel} = telemetry;
if (Level[telemetryLevel] < TelemetryContracts[event].minimumTelemetryLevel) { return; }
fetchFromHome('/api/telemetry', {
method: 'POST',
body: JSON.stringify({
name,
event,
metadata,
}),
credentials: 'include',
@ -17,7 +23,7 @@ export function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<st
'X-Requested-With': 'XMLHttpRequest',
},
}).catch((e: Error) => {
console.warn(`Failed to log telemetry event ${name}`, e);
console.warn(`Failed to log telemetry event ${event}`, e);
logError(e);
});
}

2
app/client/models/errors.ts
View File

@ -215,6 +215,6 @@ export function logError(error: Error|string) {
}).catch(e => {
// There ... isn't much we can do about this.
// tslint:disable-next-line:no-console
console.warn('Failed to log event', event);
console.warn('Failed to log event', e);
});
}

32
app/client/ui/AccountWidget.ts
View File

@ -1,7 +1,6 @@
import {AppModel} from 'app/client/models/AppModel';
import {DocPageModel} from 'app/client/models/DocPageModel';
import {getLoginOrSignupUrl, getLoginUrl, getLogoutUrl, urlState} from 'app/client/models/gristUrlState';
import {buildUserMenuBillingItem} from 'app/client/ui/BillingButtons';
import {manageTeamUsers} from 'app/client/ui/OpenUserManager';
import {createUserImage} from 'app/client/ui/UserImage';
import * as viewport from 'app/client/ui/viewport';
@ -16,6 +15,7 @@ import {Disposable, dom, DomElementArg, styled} from 'grainjs';
import {cssMenuItem} from 'popweasel';
import {maybeAddSiteSwitcherSection} from 'app/client/ui/SiteSwitcher';
import {makeT} from 'app/client/lib/localization';
import {getGristConfig} from 'app/common/urlUtils';
const t = makeT('AccountWidget');
@ -98,7 +98,8 @@ export class AccountWidget extends Disposable {
// Don't show on doc pages, or for personal orgs.
null),
buildUserMenuBillingItem(this._appModel),
this._maybeBuildBillingPageMenuItem(),
this._maybeBuildActivationPageMenuItem(),
mobileModeToggle,
@ -141,6 +142,33 @@ export class AccountWidget extends Disposable {
}
this._appModel.topAppModel.initialize();
}
private _maybeBuildBillingPageMenuItem() {
const {deploymentType} = getGristConfig();
if (deploymentType !== 'saas') { return null; }
const {currentValidUser, currentOrg, isTeamSite} = this._appModel;
const isBillingManager = Boolean(currentOrg && currentOrg.billingAccount &&
(currentOrg.billingAccount.isManager || currentValidUser?.isSupport));
return isTeamSite ?
// For links, disabling with just a class is hard; easier to just not make it a link.
// TODO weasel menus should support disabling menuItemLink.
(isBillingManager ?
menuItemLink(urlState().setLinkUrl({billing: 'billing'}), 'Billing Account') :
menuItem(() => null, 'Billing Account', dom.cls('disabled', true))
) :
menuItem(() => this._appModel.showUpgradeModal(), 'Upgrade Plan');
}
private _maybeBuildActivationPageMenuItem() {
const {activation, deploymentType} = getGristConfig();
if (deploymentType !== 'enterprise' || !activation?.isManager) {
return null;
}
return menuItemLink('Activation', urlState().setLinkUrl({activation: 'activation'}));
}
}
const cssAccountWidget = styled('div', `

39
app/client/ui/AppHeader.ts
View File

@ -1,5 +1,4 @@
import {getWelcomeHomeUrl, urlState} from 'app/client/models/gristUrlState';
import {buildAppMenuBillingItem} from 'app/client/ui/BillingButtons';
import {getTheme} from 'app/client/ui/CustomThemes';
import {cssLeftPane} from 'app/client/ui/PagePanels';
import {colors, testId, theme, vars} from 'app/client/ui2018/cssVars';
@ -14,6 +13,7 @@ import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
import {maybeAddSiteSwitcherSection} from 'app/client/ui/SiteSwitcher';
import {BindableValue, Disposable, dom, DomContents, styled} from 'grainjs';
import {makeT} from 'app/client/lib/localization';
import {getGristConfig} from 'app/common/urlUtils';
const t = makeT('AppHeader');
@ -71,7 +71,8 @@ export class AppHeader extends Disposable {
// Don't show on doc pages, or for personal orgs.
null),
buildAppMenuBillingItem(this._appModel, testId('orgmenu-billing')),
this._maybeBuildBillingPageMenuItem(),
this._maybeBuildActivationPageMenuItem(),
maybeAddSiteSwitcherSection(this._appModel),
], { placement: 'bottom-start' }),
@ -88,6 +89,40 @@ export class AppHeader extends Disposable {
return {href: getWelcomeHomeUrl()};
}
}
private _maybeBuildBillingPageMenuItem() {
const {deploymentType} = getGristConfig();
if (deploymentType !== 'saas') { return null; }
const {currentOrg} = this._appModel;
const isBillingManager = this._appModel.isBillingManager() || this._appModel.isSupport();
return currentOrg && !currentOrg.owner ?
// For links, disabling with just a class is hard; easier to just not make it a link.
// TODO weasel menus should support disabling menuItemLink.
(isBillingManager
? menuItemLink(
urlState().setLinkUrl({billing: 'billing'}),
'Billing Account',
testId('orgmenu-billing'),
)
: menuItem(
() => null,
'Billing Account',
dom.cls('disabled', true),
testId('orgmenu-billing'),
)
) :
null;
}
private _maybeBuildActivationPageMenuItem() {
const {activation, deploymentType} = getGristConfig();
if (deploymentType !== 'enterprise' || !activation?.isManager) {
return null;
}
return menuItemLink('Activation', urlState().setLinkUrl({activation: 'activation'}));
}
}
export function productPill(org: Organization|null, options: {large?: boolean} = {}): DomContents {

2
app/client/ui/OpenVideoTour.ts
View File

@ -37,7 +37,7 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
if (youtubePlayer.isLoading()) { return; }
logTelemetryEvent('watchedVideoTour', {
watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime()),
limited: {watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime())},
});
});

162
app/client/ui/WelcomeCoachingCall.ts Normal file
View File

@ -0,0 +1,162 @@
import {AppModel} from 'app/client/models/AppModel';
import {bigBasicButton, bigPrimaryButtonLink} from 'app/client/ui2018/buttons';
import {testId, theme, vars} from 'app/client/ui2018/cssVars';
import {cardPopup, cssPopupBody, cssPopupButtons, cssPopupCloseButton,
cssPopupTitle} from 'app/client/ui2018/popups';
import {icon} from 'app/client/ui2018/icons';
import {getGristConfig} from 'app/common/urlUtils';
import {dom, styled} from 'grainjs';
const FREE_COACHING_CALL_URL = 'https://calendly.com/grist-team/grist-free-coaching-call';
export function shouldShowWelcomeCoachingCall(appModel: AppModel) {
const {deploymentType} = getGristConfig();
if (deploymentType !== 'saas') { return false; }
const {behavioralPromptsManager, dismissedWelcomePopups} = appModel;
// Defer showing coaching call until Add New tip is dismissed.
const hasSeenAddNewTip = behavioralPromptsManager.hasSeenTip('addNew');
const shouldShowTips = behavioralPromptsManager.shouldShowTips();
if (!hasSeenAddNewTip && shouldShowTips) { return false; }
const popup = dismissedWelcomePopups.get().find(p => p.id === 'coachingCall');
return (
// Only show if the user is an owner.
appModel.isOwner() && (
// And preferences for the popup haven't been saved before.
popup === undefined ||
// Or the popup has been shown before, and it's time to shown it again.
popup.nextAppearanceAt !== null && popup.nextAppearanceAt <= Date.now()
)
);
}
/**
* Shows a popup with an offer for a free coaching call.
*/
export function showWelcomeCoachingCall(triggerElement: Element, appModel: AppModel) {
const {dismissedWelcomePopups} = appModel;
cardPopup(triggerElement, (ctl) => {
const dismissPopup = (scheduleNextAppearance?: boolean) => {
const dismissedPopups = dismissedWelcomePopups.get();
const newDismissedPopups = [...dismissedPopups];
const coachingPopup = newDismissedPopups.find(p => p.id === 'coachingCall');
if (!coachingPopup) {
newDismissedPopups.push({
id: 'coachingCall',
lastDismissedAt: Date.now(),
timesDismissed: 1,
nextAppearanceAt: scheduleNextAppearance
? new Date().setDate(new Date().getDate() + 7)
: null,
});
} else {
Object.assign(coachingPopup, {
lastDismissedAt: Date.now(),
timesDismissed: coachingPopup.timesDismissed + 1,
nextAppearanceAt: scheduleNextAppearance && coachingPopup.timesDismissed + 1 <= 1
? new Date().setDate(new Date().getDate() + 7)
: null,
});
}
dismissedWelcomePopups.set(newDismissedPopups);
ctl.close();
};
// TODO: i18n
return [
cssPopup.cls(''),
cssPopupHeader(
cssLogoAndName(
cssLogo(),
cssName('Grist'),
),
cssPopupCloseButton(
cssCloseIcon('CrossBig'),
dom.on('click', () => dismissPopup(true)),
testId('popup-close-button'),
),
),
cssPopupTitle('Free Coaching Call', testId('popup-title')),
cssPopupBody(
cssBody(
dom('div',
'Schedule your ', cssBoldText('free coaching call'), ' with a member of our team.'
),
dom('div',
"On the call, we'll take the time to understand your needs and "
+ 'tailor the call to you. We can show you the Grist basics, or start '
+ 'working with your data right away to build the dashboards you need.'
),
),
testId('popup-body'),
),
cssPopupButtons(
bigPrimaryButtonLink(
'Schedule Call',
dom.on('click', () => dismissPopup(false)),
{
href: FREE_COACHING_CALL_URL,
target: '_blank',
},
testId('popup-primary-button'),
),
bigBasicButton(
'Maybe Later',
dom.on('click', () => dismissPopup(true)),
testId('popup-basic-button'),
),
),
testId('coaching-call'),
];
});
}
const cssBody = styled('div', `
display: flex;
flex-direction: column;
row-gap: 16px;
`);
const cssBoldText = styled('span', `
font-weight: 600;
`);
const cssCloseIcon = styled(icon, `
padding: 12px;
`);
const cssName = styled('div', `
color: ${theme.popupCloseButtonFg};
font-size: ${vars.largeFontSize};
font-weight: 600;
`);
const cssLogo = styled('div', `
flex: none;
height: 32px;
width: 32px;
background-image: var(--icon-GristLogo);
background-size: ${vars.logoSize};
background-repeat: no-repeat;
background-position: center;
`);
const cssLogoAndName = styled('div', `
display: flex;
align-items: center;
gap: 4px;
`);
const cssPopup = styled('div', `
display: flex;
flex-direction: column;
`);
const cssPopupHeader = styled('div', `
display: flex;
justify-content: space-between;
margin-bottom: 16px;
`);

9
app/client/ui/WelcomeCoachingCallStub.ts
View File

@ -1,9 +0,0 @@
import {AppModel} from 'app/client/models/AppModel';
export function shouldShowWelcomeCoachingCall(_app: AppModel) {
return false;
}
export function showWelcomeCoachingCall(_triggerElement: Element, _app: AppModel) {
}

934
app/common/Telemetry.ts
View File

@ -1,6 +1,796 @@
export const TelemetryTemplateSignupCookieName = 'gr_template_signup_trk';
import {StringUnion} from 'app/common/StringUnion';
import pickBy = require('lodash/pickBy');
export const TelemetryEventNames = [
/**
* Telemetry levels, in increasing order of data collected.
*/
export enum Level {
off = 0,
limited = 1,
full = 2,
}
/**
* A set of contracts that all telemetry events must follow prior to being
* logged.
*
* Currently, this includes meeting minimum telemetry levels for events
* and their metadata, and passing in the correct data type for the value of
* each metadata property.
*
* The `minimumTelemetryLevel` defined at the event level will also be applied
* to all metadata properties of an event, and can be overridden at the metadata
* level.
*/
export const TelemetryContracts: TelemetryContracts = {
/**
* Triggered when an HTTP request with an API key is made.
*/
apiUsage: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* The HTTP request method (e.g. GET, POST, PUT).
*/
method: {
dataType: 'string',
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
/**
* The User-Agent HTTP request header.
*/
userAgent: {
dataType: 'string',
},
},
},
/**
* Triggered when HelpScout Beacon is opened.
*/
beaconOpen: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
},
},
},
/**
* Triggered when an article is opened in HelpScout Beacon.
*/
beaconArticleViewed: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* The id of the article.
*/
articleId: {
dataType: 'string',
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
},
},
},
/**
* Triggered when an email is sent in HelpScout Beacon.
*/
beaconEmailSent: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
},
},
},
/**
* Triggered when a search is made in HelpScout Beacon.
*/
beaconSearch: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* The search query.
*/
searchQuery: {
dataType: 'string',
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
},
},
},
/**
* Triggered when a document is forked.
*/
documentForked: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* A hash of the doc id.
*/
docIdDigest: {
dataType: 'string',
},
/**
* The id of the site containing the forked document.
*/
siteId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* The type of the site.
*/
siteType: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* A hash of the fork id.
*/
forkIdDigest: {
dataType: 'string',
},
/**
* A hash of the full id of the fork, including the trunk id and fork id.
*/
forkDocIdDigest: {
dataType: 'string',
},
/**
* A hash of the trunk id.
*/
trunkIdDigest: {
dataType: 'string',
},
/**
* Whether the trunk is a template.
*/
isTemplate: {
dataType: 'boolean',
},
/**
* Timestamp of the last update to the trunk document.
*/
lastActivity: {
dataType: 'date',
},
},
},
/**
* Triggered when a public document or template is opened.
*/
documentOpened: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* A hash of the doc id.
*/
docIdDigest: {
dataType: 'string',
},
/**
* The site id.
*/
siteId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The document access level of the user that triggered this event.
*/
access: {
dataType: 'boolean',
},
/**
* Whether the document is public.
*/
isPublic: {
dataType: 'boolean',
},
/**
* Whether a snapshot was opened.
*/
isSnapshot: {
dataType: 'boolean',
},
/**
* Whether the document is a template.
*/
isTemplate: {
dataType: 'boolean',
},
/**
* Timestamp of when the document was last updated.
*/
lastUpdated: {
dataType: 'date',
},
},
},
/**
* Triggered on doc open and close, as well as hourly while a document is open.
*/
documentUsage: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* A hash of the doc id.
*/
docIdDigest: {
dataType: 'string',
},
/**
* The site id.
*/
siteId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* What caused this event to trigger.
*
* May be either "docOpen", "interval", or "docClose".
*/
triggeredBy: {
dataType: 'string',
},
/**
* Whether the document is public.
*/
isPublic: {
dataType: 'boolean',
},
/**
* The number of rows in the document.
*/
rowCount: {
dataType: 'number',
},
/**
* The total size of all data in the document, excluding attachments.
*/
dataSizeBytes: {
dataType: 'number',
},
/**
* The total size of all attachments in the document.
*/
attachmentsSize: {
dataType: 'number',
},
/**
* The number of access rules in the document.
*/
numAccessRules: {
dataType: 'number',
},
/**
* The number of user attributes in the document.
*/
numUserAttributes: {
dataType: 'number',
},
/**
* The number of attachments in the document.
*/
numAttachments: {
dataType: 'number',
},
/**
* A list of unique file extensions compiled from all of the document's attachments.
*/
attachmentTypes: {
dataType: 'string[]',
},
/**
* The number of charts in the document.
*/
numCharts: {
dataType: 'number',
},
/**
* A list of chart types of every chart in the document.
*/
chartTypes: {
dataType: 'string[]',
},
/**
* The number of linked charts in the document.
*/
numLinkedCharts: {
dataType: 'number',
},
/**
* The number of linked widgets in the document.
*/
numLinkedWidgets: {
dataType: 'number',
},
/**
* The number of columns in the document.
*/
numColumns: {
dataType: 'number',
},
/**
* The number of columns with conditional formatting in the document.
*/
numColumnsWithConditionalFormatting: {
dataType: 'number',
},
/**
* The number of formula columns in the document.
*/
numFormulaColumns: {
dataType: 'number',
},
/**
* The number of trigger formula columns in the document.
*/
numTriggerFormulaColumns: {
dataType: 'number',
},
/**
* The number of summary formula columns in the document.
*/
numSummaryFormulaColumns: {
dataType: 'number',
},
/**
* The number of fields with conditional formatting in the document.
*/
numFieldsWithConditionalFormatting: {
dataType: 'number',
},
/**
* The number of tables in the document.
*/
numTables: {
dataType: 'number',
},
/**
* The number of on-demand tables in the document.
*/
numOnDemandTables: {
dataType: 'number',
},
/**
* The number of tables with conditional formatting in the document.
*/
numTablesWithConditionalFormatting: {
dataType: 'number',
},
/**
* The number of summary tables in the document.
*/
numSummaryTables: {
dataType: 'number',
},
/**
* The number of custom widgets in the document.
*/
numCustomWidgets: {
dataType: 'number',
},
/**
* A list of plugin ids for every custom widget in the document.
*
* The ids of widgets not created by Grist Labs are replaced with "externalId".
*/
customWidgetIds: {
dataType: 'string[]',
},
},
},
/**
* Triggered every 5 seconds.
*/
processMonitor: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/** Size of JS heap in use, in MiB. */
heapUsedMB: {
dataType: 'number',
},
/** Total heap size, in MiB, allocated for JS by V8. */
heapTotalMB: {
dataType: 'number',
},
/** Fraction (typically between 0 and 1) of CPU usage. Includes all threads, so may exceed 1. */
cpuAverage: {
dataType: 'number',
},
/** Interval (in milliseconds) over which `cpuAverage` is reported. */
intervalMs: {
dataType: 'number',
},
},
},
/**
* Triggered when sending webhooks.
*/
sendingWebhooks: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* The number of events in the batch of webhooks being sent.
*/
numEvents: {
dataType: 'number',
},
/**
* A hash of the doc id.
*/
docIdDigest: {
dataType: 'string',
},
/**
* The site id.
*/
siteId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
},
},
/**
* Triggered after a user successfully verifies their account during sign-up.
*
* Not triggered in grist-core.
*/
signupVerified: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* Whether the user viewed any templates before signing up.
*/
isAnonymousTemplateSignup: {
dataType: 'boolean',
},
/**
* The doc id of the template the user last viewed before signing up, if any.
*/
templateId: {
dataType: 'string',
},
},
},
/**
* Triggered daily.
*/
siteMembership: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* The site id.
*/
siteId: {
dataType: 'number',
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
},
/**
* The number of users with an owner role in this site.
*/
numOwners: {
dataType: 'number',
},
/**
* The number of users with an editor role in this site.
*/
numEditors: {
dataType: 'number',
},
/**
* The number of users with a viewer role in this site.
*/
numViewers: {
dataType: 'number',
},
},
},
/**
* Triggered daily.
*/
siteUsage: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* The site id.
*/
siteId: {
dataType: 'number',
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
},
/**
* Whether the site's subscription is in good standing.
*/
inGoodStanding: {
dataType: 'boolean',
},
/**
* The Stripe Plan id associated with this site.
*/
stripePlanId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
/**
* The number of docs in this site.
*/
numDocs: {
dataType: 'number',
},
/**
* The number of workspaces in this site.
*/
numWorkspaces: {
dataType: 'number',
},
/**
* The number of site members.
*/
numMembers: {
dataType: 'number',
},
/**
* A timestamp of the most recent update made to a site document.
*/
lastActivity: {
dataType: 'date',
},
},
},
/**
* Triggered on changes to tutorial progress.
*/
tutorialProgressChanged: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* A hash of the tutorial fork id.
*/
tutorialForkIdDigest: {
dataType: 'string',
},
/**
* A hash of the tutorial trunk id.
*/
tutorialTrunkIdDigest: {
dataType: 'string',
},
/**
* The 0-based index of the last tutorial slide the user had open.
*/
lastSlideIndex: {
dataType: 'number',
},
/**
* The total number of slides in the tutorial.
*/
numSlides: {
dataType: 'number',
},
/**
* Percentage of tutorial completion.
*/
percentComplete: {
dataType: 'number',
},
},
},
/**
* Triggered when a tutorial is restarted.
*/
tutorialRestarted: {
minimumTelemetryLevel: Level.full,
metadataContracts: {
/**
* A hash of the tutorial fork id.
*/
tutorialForkIdDigest: {
dataType: 'string',
},
/**
* A hash of the tutorial trunk id.
*/
tutorialTrunkIdDigest: {
dataType: 'string',
},
/**
* A hash of the doc id.
*/
docIdDigest: {
dataType: 'string',
},
/**
* The site id.
*/
siteId: {
dataType: 'number',
},
/**
* The site type.
*/
siteType: {
dataType: 'string',
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
},
},
},
/**
* Triggered when the video tour is closed.
*/
watchedVideoTour: {
minimumTelemetryLevel: Level.limited,
metadataContracts: {
/**
* The number of seconds elapsed in the video player.
*/
watchTimeSeconds: {
dataType: 'number',
},
/**
* The id of the user that triggered this event.
*/
userId: {
dataType: 'number',
minimumTelemetryLevel: Level.full,
},
/**
* A random, session-based identifier for the user that triggered this event.
*/
altSessionId: {
dataType: 'string',
minimumTelemetryLevel: Level.full,
},
},
},
};
type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
export const TelemetryEvents = StringUnion(
'apiUsage',
'beaconOpen',
'beaconArticleViewed',
@ -17,6 +807,142 @@ export const TelemetryEventNames = [
'tutorialProgressChanged',
'tutorialRestarted',
'watchedVideoTour',
] as const;
);
export type TelemetryEvent = typeof TelemetryEvents.type;
export type TelemetryEventName = typeof TelemetryEventNames[number];
interface TelemetryEventContract {
minimumTelemetryLevel: Level;
metadataContracts?: Record<string, MetadataContract>;
}
interface MetadataContract {
dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date';
minimumTelemetryLevel?: Level;
}
export type TelemetryMetadataByLevel = Partial<Record<EnabledTelemetryLevel, TelemetryMetadata>>;
export type EnabledTelemetryLevel = Exclude<TelemetryLevel, 'off'>;
export const TelemetryLevels = StringUnion('off', 'limited', 'full');
export type TelemetryLevel = typeof TelemetryLevels.type;
export type TelemetryMetadata = Record<string, any>;
/**
* The name of a cookie that's set whenever a template is opened.
*
* The cookie remembers the last template that was opened, which is then read during
* sign-up to track which templates were viewed before sign-up.
*/
export const TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME = 'gr_template_signup_trk';
// A set of metadata keys that are always allowed when logging.
const ALLOWED_METADATA_KEYS = new Set(['eventSource', 'installationId']);
/**
* Returns a function that accepts a telemetry event and metadata, and performs various
* checks on it based on a set of contracts and the `telemetryLevel`.
*
* The function throws if any checks fail.
*/
export function buildTelemetryEventChecker(telemetryLevel: TelemetryLevel) {
const currentTelemetryLevel = Level[telemetryLevel];
return (event: TelemetryEvent, metadata?: TelemetryMetadata) => {
const eventContract = TelemetryContracts[event];
if (!eventContract) {
throw new Error(`Unknown telemetry event: ${event}`);
}
const eventMinimumTelemetryLevel = eventContract.minimumTelemetryLevel;
if (currentTelemetryLevel < eventMinimumTelemetryLevel) {
throw new Error(
`Telemetry event ${event} requires a minimum telemetry level of ${eventMinimumTelemetryLevel} ` +
`but the current level is ${currentTelemetryLevel}`
);
}
for (const [key, value] of Object.entries(metadata ?? {})) {
if (ALLOWED_METADATA_KEYS.has(key)) { continue; }
const metadataContract = eventContract.metadataContracts?.[key];
if (!metadataContract) {
throw new Error(`Unknown metadata for telemetry event ${event}: ${key}`);
}
const metadataMinimumTelemetryLevel = metadataContract.minimumTelemetryLevel;
if (metadataMinimumTelemetryLevel && currentTelemetryLevel < metadataMinimumTelemetryLevel) {
throw new Error(
`Telemetry metadata ${key} of event ${event} requires a minimum telemetry level of ` +
`${metadataMinimumTelemetryLevel} but the current level is ${currentTelemetryLevel}`
);
}
const {dataType} = metadataContract;
if (dataType.endsWith('[]')) {
if (!Array.isArray(value)) {
throw new Error(
`Telemetry metadata ${key} of event ${event} expected a value of type array ` +
`but received a value of type ${typeof value}`
);
}
const elementDataType = dataType.slice(0, -2);
if (value.some(element => typeof element !== elementDataType)) {
throw new Error(
`Telemetry metadata ${key} of event ${event} expected a value of type ${elementDataType}[] ` +
`but received a value of type ${typeof value}[]`
);
}
} else if (dataType === 'date') {
if (!(value instanceof Date) && typeof value !== 'string') {
throw new Error(
`Telemetry metadata ${key} of event ${event} expected a value of type Date or string ` +
`but received a value of type ${typeof value}`
);
}
} else if (dataType !== typeof value) {
throw new Error(
`Telemetry metadata ${key} of event ${event} expected a value of type ${dataType} ` +
`but received a value of type ${typeof value}`
);
}
}
};
}
export type TelemetryEventChecker = (event: TelemetryEvent, metadata?: TelemetryMetadata) => void;
/**
* Returns a new, filtered metadata object.
*
* Metadata in groups that don't meet `telemetryLevel` are removed from the
* returned object, and the returned object is flattened.
*
* Returns undefined if `metadata` is undefined.
*/
export function filterMetadata(
metadata: TelemetryMetadataByLevel | undefined,
telemetryLevel: TelemetryLevel
): TelemetryMetadata | undefined {
if (!metadata) { return; }
let filteredMetadata = {};
for (const level of ['limited', 'full'] as const) {
if (Level[telemetryLevel] < Level[level]) { break; }
filteredMetadata = {...filteredMetadata, ...metadata[level]};
}
filteredMetadata = removeNullishKeys(filteredMetadata);
return removeNullishKeys(filteredMetadata);
}
/**
* Returns a copy of `object` with all null and undefined keys removed.
*/
export function removeNullishKeys(object: Record<string, any>) {
return pickBy(object, value => value !== null && value !== undefined);
}

14
app/common/gristUrls.ts
View File

@ -4,6 +4,7 @@ import {EngineCode} from 'app/common/DocumentSettings';
import {encodeQueryParams, isAffirmative} from 'app/common/gutil';
import {LocalPlugin} from 'app/common/plugin';
import {StringUnion} from 'app/common/StringUnion';
import {TelemetryLevel} from 'app/common/Telemetry';
import {UIRowId} from 'app/common/UIRowId';
import {getGristConfig} from 'app/common/urlUtils';
import {Document} from 'app/common/UserAPI';
@ -627,6 +628,12 @@ export interface GristLoadConfig {
// Current user locale, read from the user options;
userLocale?: string;
// Telemetry config.
telemetry?: TelemetryConfig;
// The Grist deployment type (e.g. core, enterprise).
deploymentType?: GristDeploymentType;
}
export const Features = StringUnion(
@ -648,6 +655,13 @@ export function getPageTitleSuffix(config?: GristLoadConfig) {
return config?.pageTitleSuffix ?? " - Grist";
}
export interface TelemetryConfig {
telemetryLevel: TelemetryLevel;
}
export const GristDeploymentTypes = StringUnion('saas', 'core', 'enterprise', 'electron', 'static');
export type GristDeploymentType = typeof GristDeploymentTypes.type;
/**
* For a packaged version of Grist that requires activation, this
* summarizes the current state. Not applicable to grist-core.

2
app/common/orgNameUtils.ts
View File

@ -16,7 +16,7 @@ const BLACKLISTED_SUBDOMAINS = new Set([
'docs', 'api', 'static',
'ftp', 'imap', 'pop', 'smtp', 'mail', 'git', 'blog', 'wiki', 'support', 'kb', 'help',
'admin', 'store', 'dev', 'beta',
'community', 'try', 'wpx',
'community', 'try', 'wpx', 'telemetry',
// a few random tech brands
'google', 'apple', 'microsoft', 'ms', 'facebook', 'fb', 'twitter', 'youtube', 'yt',

14
app/gen-server/entity/Document.ts
View File

@ -164,12 +164,14 @@ export class Document extends Resource {
const percentComplete = lastSlideIndex !== undefined && numSlides !== undefined
? Math.floor((lastSlideIndex / numSlides) * 100)
: undefined;
dbManager?.emit('tutorialProgressChange', {
tutorialForkIdDigest: hashId(this.id),
tutorialTrunkIdDigest: this.trunkId ? hashId(this.trunkId) : undefined,
lastSlideIndex,
numSlides,
percentComplete,
dbManager?.emit('tutorialProgressChanged', {
full: {
tutorialForkIdDigest: hashId(this.id),
tutorialTrunkIdDigest: this.trunkId ? hashId(this.trunkId) : undefined,
lastSlideIndex,
numSlides,
percentComplete,
},
});
}
}

8
app/gen-server/lib/HomeDBManager.ts
View File

@ -89,13 +89,13 @@ export const NotifierEvents = StringUnion(
export type NotifierEvent = typeof NotifierEvents.type;
export const TelemetryEvents = StringUnion(
'tutorialProgressChange',
export const HomeDBTelemetryEvents = StringUnion(
'tutorialProgressChanged',
);
export type TelemetryEvent = typeof TelemetryEvents.type;
export type HomeDBTelemetryEvent = typeof HomeDBTelemetryEvents.type;
export type Event = NotifierEvent | TelemetryEvent;
export type Event = NotifierEvent | HomeDBTelemetryEvent;
// Nominal email address of a user who can view anything (for thumbnails).
export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com';

44
app/gen-server/lib/Housekeeper.ts
View File

@ -37,6 +37,7 @@ export class Housekeeper {
private _deleteTrashinterval?: NodeJS.Timeout;
private _logMetricsInterval?: NodeJS.Timeout;
private _electionKey?: string;
private _telemetry = this._server.getTelemetry();
public constructor(private _dbManager: HomeDBManager, private _server: GristServer,
private _permitStore: IPermitStore, private _electionStore: IElectionStore) {
@ -174,30 +175,37 @@ export class Housekeeper {
*/
public async logMetrics() {
await this._dbManager.connection.transaction('READ UNCOMMITTED', async (manager) => {
const telemetryManager = this._server.getTelemetryManager();
const usageSummaries = await this._getOrgUsageSummaries(manager);
for (const summary of usageSummaries) {
telemetryManager?.logEvent('siteUsage', {
siteId: summary.site_id,
siteType: summary.site_type,
inGoodStanding: Boolean(summary.in_good_standing),
stripePlanId: summary.stripe_plan_id,
numDocs: Number(summary.num_docs),
numWorkspaces: Number(summary.num_workspaces),
numMembers: Number(summary.num_members),
lastActivity: summary.last_activity,
});
this._telemetry.logEvent('siteUsage', {
limited: {
siteId: summary.site_id,
siteType: summary.site_type,
inGoodStanding: Boolean(summary.in_good_standing),
numDocs: Number(summary.num_docs),
numWorkspaces: Number(summary.num_workspaces),
numMembers: Number(summary.num_members),
lastActivity: summary.last_activity,
},
full: {
stripePlanId: summary.stripe_plan_id,
},
})
.catch(e => log.error('failed to log telemetry event siteUsage', e));
}
const membershipSummaries = await this._getOrgMembershipSummaries(manager);
for (const summary of membershipSummaries) {
telemetryManager?.logEvent('siteMembership', {
siteId: summary.site_id,
siteType: summary.site_type,
numOwners: Number(summary.num_owners),
numEditors: Number(summary.num_editors),
numViewers: Number(summary.num_viewers),
});
this._telemetry.logEvent('siteMembership', {
limited: {
siteId: summary.site_id,
siteType: summary.site_type,
numOwners: Number(summary.num_owners),
numEditors: Number(summary.num_editors),
numViewers: Number(summary.num_viewers),
},
})
.catch(e => log.error('failed to log telemetry event siteMembership', e));
}
});
}

3
app/server/generateInitialDocSql.ts
View File

@ -3,6 +3,7 @@ import { create } from 'app/server/lib/create';
import { DocManager } from 'app/server/lib/DocManager';
import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
import { DocStorageManager } from 'app/server/lib/DocStorageManager';
import { createDummyTelemetry } from 'app/server/lib/GristServer';
import { PluginManager } from 'app/server/lib/PluginManager';
import * as childProcess from 'child_process';
@ -33,7 +34,7 @@ export async function main(baseName: string) {
}
const docManager = new DocManager(storageManager, pluginManager, null as any, {
create,
getTelemetryManager: () => undefined,
getTelemetry() { return createDummyTelemetry(); },
} as any);
const activeDoc = new ActiveDoc(docManager, baseName);
const session = makeExceptionalDocSession('nascent');

71
app/server/lib/ActiveDoc.ts
View File

@ -74,7 +74,7 @@ import {Interval} from 'app/common/Interval';
import * as roles from 'app/common/roles';
import {schema, SCHEMA_VERSION} from 'app/common/schema';
import {MetaRowRecord, SingleCell} from 'app/common/TableData';
import {TelemetryEventName} from 'app/common/Telemetry';
import {TelemetryEvent, TelemetryMetadataByLevel} from 'app/common/Telemetry';
import {UIRowId} from 'app/common/UIRowId';
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
@ -1395,11 +1395,13 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
// TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
this.logTelemetryEvent(docSession, 'documentForked', {
forkIdDigest: hashId(forkIds.forkId),
forkDocIdDigest: hashId(forkIds.docId),
trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
isTemplate,
lastActivity: doc.updatedAt,
limited: {
forkIdDigest: hashId(forkIds.forkId),
forkDocIdDigest: hashId(forkIds.docId),
trunkIdDigest: doc.trunkId ? hashId(doc.trunkId) : undefined,
isTemplate,
lastActivity: doc.updatedAt,
},
});
} finally {
await permitStore.removePermit(permitKey);
@ -1789,13 +1791,14 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
public logTelemetryEvent(
docSession: OptDocSession | null,
eventName: TelemetryEventName,
metadata?: Record<string, any>
event: TelemetryEvent,
metadata?: TelemetryMetadataByLevel
) {
this._docManager.gristServer.getTelemetryManager()?.logEvent(eventName, {
...this._getTelemetryMeta(docSession),
...metadata,
});
this._docManager.gristServer.getTelemetry().logEvent(event, merge(
this._getTelemetryMeta(docSession),
metadata,
))
.catch(e => this._log.error(docSession, `failed to log telemetry event ${event}`, e));
}
/**
@ -2332,18 +2335,20 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
this.logTelemetryEvent(docSession, 'documentUsage', {
triggeredBy,
isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
rowCount: this._docUsage?.rowCount?.total,
dataSizeBytes: this._docUsage?.dataSizeBytes,
attachmentsSize: this._docUsage?.attachmentsSizeBytes,
...this._getAccessRuleMetrics(),
...this._getAttachmentMetrics(),
...this._getChartMetrics(),
...this._getWidgetMetrics(),
...this._getColumnMetrics(),
...this._getTableMetrics(),
...this._getCustomWidgetMetrics(),
limited: {
triggeredBy,
isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
rowCount: this._docUsage?.rowCount?.total,
dataSizeBytes: this._docUsage?.dataSizeBytes,
attachmentsSize: this._docUsage?.attachmentsSizeBytes,
...this._getAccessRuleMetrics(),
...this._getAttachmentMetrics(),
...this._getChartMetrics(),
...this._getWidgetMetrics(),
...this._getColumnMetrics(),
...this._getTableMetrics(),
...this._getCustomWidgetMetrics(),
},
});
}
@ -2365,10 +2370,10 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
// Exclude the leading ".", if any.
.map(r => r.fileExt?.trim()?.slice(1))
.filter(ext => Boolean(ext));
const uniqueAttachmentTypes = [...new Set(attachmentTypes ?? [])];
return {
numAttachments,
attachmentTypes,
attachmentTypes: uniqueAttachmentTypes,
};
}
@ -2528,15 +2533,19 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
this._logDocMetrics(docSession, 'docOpen');
}
private _getTelemetryMeta(docSession: OptDocSession|null) {
private _getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel {
const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined;
return merge(
docSession ? getTelemetryMetaFromDocSession(docSession) : {},
altSessionId ? {altSessionId} : undefined,
altSessionId ? {altSessionId} : {},
{
docIdDigest: hashId(this._docName),
siteId: this._doc?.workspace.org.id,
siteType: this._product?.name,
limited: {
docIdDigest: hashId(this._docName),
},
full: {
siteId: this._doc?.workspace.org.id,
siteType: this._product?.name,
},
},
);
}

33
app/server/lib/AppEndpoint.ts
View File

@ -11,7 +11,7 @@ import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gr
import {removeTrailingSlash} from 'app/common/gutil';
import {hashId} from 'app/common/hashingUtils';
import {LocalPlugin} from "app/common/plugin";
import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
import {Document as APIDocument} from 'app/common/UserAPI';
import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
import {Document} from "app/gen-server/entity/Document";
@ -308,18 +308,23 @@ export function attachAppEndpoint(options: AttachOptions): void {
// TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
if (isPublic || isTemplate) {
gristServer.getTelemetryManager()?.logEvent('documentOpened', {
docIdDigest: hashId(docId),
siteId: doc.workspace.org.id,
siteType: doc.workspace.org.billingAccount.product.name,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
access: doc.access,
isPublic,
isSnapshot,
isTemplate,
lastUpdated: doc.updatedAt,
});
gristServer.getTelemetry().logEvent('documentOpened', {
limited: {
docIdDigest: hashId(docId),
access: doc.access,
isPublic,
isSnapshot,
isTemplate,
lastUpdated: doc.updatedAt,
},
full: {
siteId: doc.workspace.org.id,
siteType: doc.workspace.org.billingAccount.product.name,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
})
.catch(e => log.error('failed to log telemetry event documentOpened', e));
}
if (isTemplate) {
@ -330,7 +335,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
isAnonymous: isAnonymousUser(mreq),
templateId: docId,
};
res.cookie(TelemetryTemplateSignupCookieName, JSON.stringify(value), {
res.cookie(TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME, JSON.stringify(value), {
maxAge: 1000 * 60 * 60,
httpOnly: true,
path: '/',

13
app/server/lib/Authorizer.ts
View File

@ -396,11 +396,14 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
};
log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
if (hasApiKey) {
options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
method: mreq.method,
userId: mreq.userId,
userAgent: req.headers['user-agent'],
});
options.gristServer.getTelemetry().logEvent('apiUsage', {
full: {
method: mreq.method,
userId: mreq.userId,
userAgent: mreq.headers['user-agent'],
},
})
.catch(e => log.error('failed to log telemetry event apiUsage', e));
}
return next();

5
app/server/lib/Client.ts
View File

@ -4,6 +4,7 @@ import {delay} from 'app/common/delay';
import {CommClientConnect, CommMessage, CommResponse, CommResponseError} from 'app/common/CommTypes';
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {TelemetryMetadata} from 'app/common/Telemetry';
import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI';
import {User} from 'app/gen-server/entity/User';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
@ -433,8 +434,8 @@ export class Client {
return meta;
}
public getFullTelemetryMeta() {
const meta: Record<string, any> = {};
public getFullTelemetryMeta(): TelemetryMetadata {
const meta: TelemetryMetadata = {};
// We assume the _userId has already been cached, which will be true always (for all practical
// purposes) because it's set when the Authorizer checks this client.
if (this._userId) { meta.userId = this._userId; }

6
app/server/lib/DocApi.ts
View File

@ -916,8 +916,10 @@ export class DocWorkerApi {
});
const {forkId} = parseUrlId(scope.urlId);
activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
tutorialTrunkIdDigest: hashId(tutorialTrunkId),
full: {
tutorialForkIdDigest: forkId ? hashId(forkId) : undefined,
tutorialTrunkIdDigest: hashId(tutorialTrunkId),
},
});
}
}

65
app/server/lib/FlexServer.ts
View File

@ -1,12 +1,12 @@
import {ApiError} from 'app/common/ApiError';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPathOnly,
parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
sanitizePathTail} from 'app/common/gristUrls';
import {getOrgUrlInfo} from 'app/common/gristUrls';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {tbind} from 'app/common/tbind';
import {TelemetryEventName, TelemetryEventNames} from 'app/common/Telemetry';
import * as version from 'app/common/version';
import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer';
import {Document} from "app/gen-server/entity/Document";
@ -57,7 +57,7 @@ import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
import {Sessions} from 'app/server/lib/Sessions';
import * as shutdown from 'app/server/lib/shutdown';
import {TagChecker} from 'app/server/lib/TagChecker';
import {TelemetryManager} from 'app/server/lib/TelemetryManager';
import {ITelemetry} from 'app/server/lib/Telemetry';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {addUploadRoute} from 'app/server/lib/uploads';
@ -117,7 +117,9 @@ export class FlexServer implements GristServer {
public electronServerMethods: ElectronServerMethods;
public readonly docsRoot: string;
public readonly i18Instance: i18n;
private _activations: Activations;
private _comm: Comm;
private _deploymentType: GristDeploymentType;
private _dbManager: HomeDBManager;
private _defaultBaseDomain: string|undefined;
private _pluginUrl: string|undefined;
@ -130,7 +132,7 @@ export class FlexServer implements GristServer {
private _sessions: Sessions;
private _sessionStore: SessionStore;
private _storageManager: IDocStorageManager;
private _telemetryManager: TelemetryManager|undefined;
private _telemetry: ITelemetry;
private _processMonitorStop?: () => void; // Callback to stop the ProcessMonitor
private _docWorkerMap: IDocWorkerMap;
private _widgetRepository: IWidgetRepository;
@ -199,6 +201,11 @@ export class FlexServer implements GristServer {
this.docsRoot = fse.realpathSync(docsRoot);
this.info.push(['docsRoot', this.docsRoot]);
this._deploymentType = this.create.deploymentType();
if (process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE) {
this._deploymentType = GristDeploymentTypes.check(process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE);
}
const homeUrl = process.env.APP_HOME_URL;
// The "base domain" is only a thing if orgs are encoded as a subdomain.
if (process.env.GRIST_ORG_IN_PATH === 'true' || process.env.GRIST_SINGLE_ORG) {
@ -328,11 +335,20 @@ export class FlexServer implements GristServer {
return this._comm;
}
public getDeploymentType(): GristDeploymentType {
return this._deploymentType;
}
public getHosts(): Hosts {
if (!this._hosts) { throw new Error('no hosts available'); }
return this._hosts;
}
public getActivations(): Activations {
if (!this._activations) { throw new Error('no activations available'); }
return this._activations;
}
public getHomeDBManager(): HomeDBManager {
if (!this._dbManager) { throw new Error('no home db available'); }
return this._dbManager;
@ -343,8 +359,9 @@ export class FlexServer implements GristServer {
return this._storageManager;
}
public getTelemetryManager(): TelemetryManager|undefined {
return this._telemetryManager;
public getTelemetry(): ITelemetry {
if (!this._telemetry) { throw new Error('no telemetry available'); }
return this._telemetry;
}
public getWidgetRepository(): IWidgetRepository {
@ -553,8 +570,8 @@ export class FlexServer implements GristServer {
// Report which database we are using, without sensitive credentials.
this.info.push(['database', getDatabaseUrl(this._dbManager.connection.options, false)]);
// If the installation appears to be new, give it an id and a creation date.
const activations = new Activations(this._dbManager);
await activations.current();
this._activations = new Activations(this._dbManager);
await this._activations.current();
}
public addDocWorkerMap() {
@ -689,24 +706,14 @@ export class FlexServer implements GristServer {
});
}
public addTelemetryEndpoint() {
if (this._check('telemetry-endpoint', 'json', 'api-mw', 'homedb')) { return; }
public addTelemetry() {
if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }
this._telemetryManager = new TelemetryManager(this._dbManager);
this._telemetry = this.create.Telemetry(this._dbManager, this);
this._telemetry.addEndpoints(this.app);
// Start up a monitor for memory and cpu usage.
this._processMonitorStop = ProcessMonitor.start(this._telemetryManager);
this.app.post('/api/telemetry', async (req, resp) => {
const mreq = req as RequestWithLogin;
const name = stringParam(req.body.name, 'name', TelemetryEventNames);
this._telemetryManager?.logEvent(name as TelemetryEventName, {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
...req.body.metadata,
});
return resp.status(200).send();
});
this._processMonitorStop = ProcessMonitor.start(this._telemetry);
}
public async close() {
@ -828,7 +835,7 @@ export class FlexServer implements GristServer {
// Initialize _sendAppPage helper.
this._sendAppPage = makeSendAppPage({
server: isSingleUserMode() ? null : this,
server: this,
staticDir: getAppPathTo(this.appRoot, 'static'),
tag: this.tag,
testLogin: allowTestLogin(),
@ -1108,7 +1115,7 @@ export class FlexServer implements GristServer {
// Add document-related endpoints and related support.
public async addDoc() {
this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ?
null : 'homedb', 'api-mw', 'map', 'telemetry-endpoint');
null : 'homedb', 'api-mw', 'map', 'telemetry');
// add handlers for cleanup, if we are in charge of the doc manager.
if (!this._docManager) { this.addCleanup(); }
await this.loadConfig();
@ -1368,7 +1375,11 @@ export class FlexServer implements GristServer {
}
public getGristConfig(): GristLoadConfig {
return makeGristConfig(this.getDefaultHomeUrl(), {}, this._defaultBaseDomain);
return makeGristConfig({
homeUrl: this.getDefaultHomeUrl(),
extra: {},
baseDomain: this._defaultBaseDomain,
});
}
/**

21
app/server/lib/GristServer.ts
View File

@ -1,8 +1,9 @@
import { GristLoadConfig } from 'app/common/gristUrls';
import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
import { FullUser, UserProfile } from 'app/common/UserAPI';
import { Document } from 'app/gen-server/entity/Document';
import { Organization } from 'app/gen-server/entity/Organization';
import { Workspace } from 'app/gen-server/entity/Workspace';
import { Activations } from 'app/gen-server/lib/Activations';
import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
import { IAccessTokens } from 'app/server/lib/AccessTokens';
import { RequestWithLogin } from 'app/server/lib/Authorizer';
@ -16,7 +17,7 @@ import { IPermitStore } from 'app/server/lib/Permit';
import { ISendAppPageOptions } from 'app/server/lib/sendAppPage';
import { fromCallback } from 'app/server/lib/serverUtils';
import { Sessions } from 'app/server/lib/Sessions';
import { TelemetryManager } from 'app/server/lib/TelemetryManager';
import { ITelemetry } from 'app/server/lib/Telemetry';
import * as express from 'express';
import { IncomingMessage } from 'http';
@ -40,10 +41,12 @@ export interface GristServer {
getExternalPermitStore(): IPermitStore;
getSessions(): Sessions;
getComm(): Comm;
getDeploymentType(): GristDeploymentType;
getHosts(): Hosts;
getActivations(): Activations;
getHomeDBManager(): HomeDBManager;
getStorageManager(): IDocStorageManager;
getTelemetryManager(): TelemetryManager|undefined;
getTelemetry(): ITelemetry;
getNotifier(): INotifier;
getDocTemplate(): Promise<DocTemplate>;
getTag(): string;
@ -117,10 +120,12 @@ export function createDummyGristServer(): GristServer {
getResourceUrl() { return Promise.resolve(''); },
getSessions() { throw new Error('no sessions'); },
getComm() { throw new Error('no comms'); },
getDeploymentType() { return 'core'; },
getHosts() { throw new Error('no hosts'); },
getActivations() { throw new Error('no activations'); },
getHomeDBManager() { throw new Error('no db'); },
getStorageManager() { throw new Error('no storage manager'); },
getTelemetryManager() { return undefined; },
getTelemetry() { return createDummyTelemetry(); },
getNotifier() { throw new Error('no notifier'); },
getDocTemplate() { throw new Error('no doc template'); },
getTag() { return 'tag'; },
@ -128,3 +133,11 @@ export function createDummyGristServer(): GristServer {
getAccessTokens() { throw new Error('no access tokens'); },
};
}
export function createDummyTelemetry(): ITelemetry {
return {
logEvent() { return Promise.resolve(); },
addEndpoints() { /* do nothing */ },
getTelemetryLevel() { return 'off'; },
};
}

18
app/server/lib/ICreate.ts
View File

@ -1,19 +1,22 @@
import {GristDeploymentType} from 'app/common/gristUrls';
import {getThemeBackgroundSnippet} from 'app/common/Themes';
import {Document} from 'app/gen-server/entity/Document';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {ExternalStorage} from 'app/server/lib/ExternalStorage';
import {GristServer} from 'app/server/lib/GristServer';
import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
import {IBilling} from 'app/server/lib/IBilling';
import {INotifier} from 'app/server/lib/INotifier';
import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox';
import {IShell} from 'app/server/lib/IShell';
import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
import {SqliteVariant} from 'app/server/lib/SqliteCommon';
import {ITelemetry} from 'app/server/lib/Telemetry';
export interface ICreate {
Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
Shell?(): IShell; // relevant to electron version of Grist only.
// Create a space to store files externally, for storing either:
@ -25,6 +28,7 @@ export interface ICreate {
NSandbox(options: ISandboxCreationOptions): ISandbox;
deploymentType(): GristDeploymentType;
sessionSecret(): string;
// Check configuration of the app early enough to show on startup.
configure?(): Promise<void>;
@ -57,19 +61,26 @@ export interface ICreateBillingOptions {
create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined;
}
export interface ICreateTelemetryOptions {
create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
}
export function makeSimpleCreator(opts: {
deploymentType: GristDeploymentType,
sessionSecret?: string,
storage?: ICreateStorageOptions[],
billing?: ICreateBillingOptions,
notifier?: ICreateNotifierOptions,
telemetry?: ICreateTelemetryOptions,
sandboxFlavor?: string,
shell?: IShell,
getExtraHeadHtml?: () => string,
getSqliteVariant?: () => SqliteVariant,
getSandboxVariants?: () => Record<string, SpawnFn>,
}): ICreate {
const {sessionSecret, storage, notifier, billing} = opts;
const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts;
return {
deploymentType() { return deploymentType; },
Billing(dbManager, gristConfig) {
return billing?.create(dbManager, gristConfig) ?? {
addEndpoints() { /* do nothing */ },
@ -93,6 +104,9 @@ export function makeSimpleCreator(opts: {
}
return undefined;
},
Telemetry(dbManager, gristConfig) {
return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();
},
NSandbox(options) {
return createSandbox(opts.sandboxFlavor || 'unsandboxed', options);
},

26
app/server/lib/ProcessMonitor.ts
View File

@ -1,4 +1,5 @@
import { TelemetryManager } from 'app/server/lib/TelemetryManager';
import log from 'app/server/lib/log';
import { ITelemetry } from 'app/server/lib/Telemetry';
const MONITOR_PERIOD_MS = 5_000; // take a look at memory usage this often
const MEMORY_DELTA_FRACTION = 0.1; // fraction by which usage should change to get reported
@ -16,7 +17,7 @@ let _lastReportedCpuAverage: number = 0;
* Monitor process memory (heap) and CPU usage, reporting as telemetry on an interval, and more
* often when usage ticks up or down by a big enough delta.
*
* There is a single global process monitor, reporting to the telemetryManager passed into the
* There is a single global process monitor, reporting to the `telemetry` object passed into the
* first call to start().
*
* Returns a function that stops the monitor, or null if there was already a process monitor
@ -30,12 +31,12 @@ let _lastReportedCpuAverage: number = 0;
* - intervalMs: Interval (in milliseconds) over which cpuAverage is reported. Being much
* higher than MONITOR_PERIOD_MS is a sign of being CPU bound for that long.
*/
export function start(telemetryManager: TelemetryManager): (() => void) | undefined {
export function start(telemetry: ITelemetry): (() => void) | undefined {
if (!_timer) {
// Initialize variables needed for accurate first-tick measurement.
_lastTickTime = Date.now();
_lastCpuUsage = process.cpuUsage();
_timer = setInterval(() => monitor(telemetryManager), MONITOR_PERIOD_MS);
_timer = setInterval(() => monitor(telemetry), MONITOR_PERIOD_MS);
return function stop() {
clearInterval(_timer);
@ -44,7 +45,7 @@ export function start(telemetryManager: TelemetryManager): (() => void) | undefi
}
}
function monitor(telemetryManager: TelemetryManager) {
function monitor(telemetry: ITelemetry) {
const memoryUsage = process.memoryUsage();
const heapUsed = memoryUsage.heapUsed;
const cpuUsage = process.cpuUsage();
@ -66,12 +67,15 @@ function monitor(telemetryManager: TelemetryManager) {
Math.abs(heapUsed - _lastReportedHeapUsed) > _lastReportedHeapUsed * MEMORY_DELTA_FRACTION ||
Math.abs(cpuAverage - _lastReportedCpuAverage) > CPU_DELTA_FRACTION
) {
telemetryManager.logEvent('processMonitor', {
heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
cpuAverage: Math.round(cpuAverage * 100) / 100,
intervalMs,
});
telemetry.logEvent('processMonitor', {
full: {
heapUsedMB: Math.round(memoryUsage.heapUsed/1024/1024),
heapTotalMB: Math.round(memoryUsage.heapTotal/1024/1024),
cpuAverage: Math.round(cpuAverage * 100) / 100,
intervalMs,
},
})
.catch(e => log.error('failed to log telemetry event processMonitor', e));
_lastReportedHeapUsed = heapUsed;
_lastReportedCpuAverage = cpuAverage;
_lastReportTime = now;

210
app/server/lib/Telemetry.ts Normal file
View File

@ -0,0 +1,210 @@
import {ApiError} from 'app/common/ApiError';
import {
buildTelemetryEventChecker,
filterMetadata,
removeNullishKeys,
TelemetryEvent,
TelemetryEventChecker,
TelemetryEvents,
TelemetryLevel,
TelemetryLevels,
TelemetryMetadata,
TelemetryMetadataByLevel,
} from 'app/common/Telemetry';
import {HomeDBManager, HomeDBTelemetryEvents} from 'app/gen-server/lib/HomeDBManager';
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import {GristServer} from 'app/server/lib/GristServer';
import {LogMethods} from 'app/server/lib/LogMethods';
import {stringParam} from 'app/server/lib/requestUtils';
import * as express from 'express';
import merge = require('lodash/merge');
export interface ITelemetry {
logEvent(name: TelemetryEvent, metadata?: TelemetryMetadataByLevel): Promise<void>;
addEndpoints(app: express.Express): void;
getTelemetryLevel(): TelemetryLevel;
}
/**
* Manages telemetry for Grist.
*/
export class Telemetry implements ITelemetry {
private _telemetryLevel: TelemetryLevel;
private _deploymentType = this._gristServer.getDeploymentType();
private _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
private _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
'https://telemetry.getgrist.com/api/telemetry';
private _installationId: string | undefined;
private _errorLogger = new LogMethods('Telemetry ', () => ({}));
private _telemetryLogger = new LogMethods('Telemetry ', () => ({
eventType: 'telemetry',
}));
private _checkEvent: TelemetryEventChecker | undefined;
constructor(private _dbManager: HomeDBManager, private _gristServer: GristServer) {
this._initialize().catch((e) => {
this._errorLogger.error(undefined, 'failed to initialize', e);
});
}
/**
* Logs a telemetry `event` and its `metadata`.
*
* Depending on the deployment type, this will either forward the
* data to an endpoint (set via GRIST_TELEMETRY_URL) or log it
* directly. In hosted Grist, telemetry is logged directly, and
* subsequently sent to an OpenSearch instance via CloudWatch. In
* other deployment types, telemetry is forwarded to an endpoint
* of hosted Grist, which then handles logging to OpenSearch.
*
* Note that `metadata` is grouped by telemetry level, with only the
* groups meeting the current telemetry level being included in
* what's logged. If the current telemetry level is `off`, nothing
* will be logged. Otherwise, `metadata` will be filtered according
* to the current telemetry level, keeping only the groups that are
* less than or equal to the current level.
*
* Additionally, runtime checks are also performed to verify that the
* event and metadata being passed in are being logged appropriately
* for the configured telemetry level. If any checks fail, an error
* is thrown.
*
* Example:
*
* The following will only log the `rowCount` if the telemetry level is set
* to `limited`, and will log both the `method` and `userId` if the telemetry
* level is set to `full`:
*
* ```
* logEvent('documentUsage', {
* limited: {
* rowCount: 123,
* },
* full: {
* userId: 1586,
* },
* });
* ```
*/
public async logEvent(
event: TelemetryEvent,
metadata?: TelemetryMetadataByLevel
) {
if (this._telemetryLevel === 'off') { return; }
metadata = filterMetadata(metadata, this._telemetryLevel);
this._checkTelemetryEvent(event, metadata);
if (this._shouldForwardTelemetryEvents) {
await this.forwardEvent(event, metadata);
} else {
this._telemetryLogger.rawLog('info', null, event, {
eventName: event,
eventSource: `grist-${this._deploymentType}`,
...metadata,
});
}
}
/**
* Forwards a telemetry event and its metadata to another server.
*/
public async forwardEvent(
event: TelemetryEvent,
metadata?: TelemetryMetadata
) {
try {
await fetch(this._forwardTelemetryEventsUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
event,
metadata,
}),
});
} catch (e) {
this._errorLogger.error(undefined, `failed to forward telemetry event ${event}`, e);
}
}
public addEndpoints(app: express.Application) {
/**
* Logs telemetry events and their metadata.
*
* Clients of this endpoint may be external Grist instances, so the behavior
* varies based on the presence of an `eventSource` key in the event metadata.
*
* If an `eventSource` key is present, the telemetry event will be logged
* directly, as the request originated from an external source; runtime checks
* of telemetry data are skipped since they should have already occured at the
* source. Otherwise, the event will only be logged after passing various
* checks.
*/
app.post('/api/telemetry', async (req, resp) => {
const mreq = req as RequestWithLogin;
const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
if ('eventSource' in req.body.metadata) {
this._telemetryLogger.rawLog('info', null, event, {
eventName: event,
...(removeNullishKeys(req.body.metadata)),
});
} else {
try {
await this.logEvent(event as TelemetryEvent, merge(
{
limited: {
eventSource: `grist-${this._deploymentType}`,
...(this._deploymentType !== 'saas' ? {installationId: this._installationId} : {}),
},
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
},
req.body.metadata,
));
} catch (e) {
this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e);
throw new ApiError(`Telemetry failed to log telemetry event ${event}`, 500);
}
}
return resp.status(200).send();
});
}
public getTelemetryLevel() {
return this._telemetryLevel;
}
private async _initialize() {
if (process.env.GRIST_TELEMETRY_LEVEL !== undefined) {
this._telemetryLevel = TelemetryLevels.check(process.env.GRIST_TELEMETRY_LEVEL);
this._checkTelemetryEvent = buildTelemetryEventChecker(this._telemetryLevel);
} else {
this._telemetryLevel = 'off';
}
const {id} = await this._gristServer.getActivations().current();
this._installationId = id;
for (const event of HomeDBTelemetryEvents.values) {
this._dbManager.on(event, async (metadata) => {
this.logEvent(event, metadata).catch(e =>
this._errorLogger.error(undefined, `failed to log telemetry event ${event}`, e));
});
}
}
private _checkTelemetryEvent(event: TelemetryEvent, metadata?: TelemetryMetadata) {
if (!this._checkEvent) {
throw new Error('Telemetry._checkEvent is undefined');
}
this._checkEvent(event, metadata);
}
}

2
app/server/lib/Triggers.ts
View File

@ -630,7 +630,7 @@ export class DocTriggers {
meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
this._log("Sending batch of webhook events", meta);
this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
numEvents: batch.length,
limited: {numEvents: meta.numEvents},
});
success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
if (this._loopAbort.signal.aborted) {

40
app/server/lib/sendAppPage.ts
View File

@ -3,7 +3,7 @@ import {isAffirmative} from 'app/common/gutil';
import {getTagManagerSnippet} from 'app/common/tagManager';
import {Document} from 'app/common/UserAPI';
import {SUPPORT_EMAIL} from 'app/gen-server/lib/HomeDBManager';
import {isAnonymousUser, RequestWithLogin} from 'app/server/lib/Authorizer';
import {isAnonymousUser, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {GristServer} from 'app/server/lib/GristServer';
import {getSupportedEngineChoices} from 'app/server/lib/serverUtils';
@ -31,9 +31,16 @@ export interface ISendAppPageOptions {
googleTagManager?: true | false | 'anon';
}
export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadConfig>,
baseDomain?: string, req?: express.Request
): GristLoadConfig {
export interface MakeGristConfigOptons {
homeUrl: string|null;
extra: Partial<GristLoadConfig>;
baseDomain?: string;
req?: express.Request;
server?: GristServer|null;
}
export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
const {homeUrl, extra, baseDomain, req, server} = options;
// .invalid is a TLD the IETF promises will never exist.
const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") ||
@ -69,6 +76,8 @@ export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadCo
featureFormulaAssistant: isAffirmative(process.env.GRIST_FORMULA_ASSISTANT),
supportEmail: SUPPORT_EMAIL,
userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
telemetry: server ? getTelemetryConfig(server) : undefined,
deploymentType: server?.getDeploymentType(),
...extra,
};
}
@ -94,14 +103,18 @@ export function makeMessagePage(staticDir: string) {
* placeholders replaced.
*/
export function makeSendAppPage(opts: {
server: GristServer|null, staticDir: string, tag: string, testLogin?: boolean,
server: GristServer, staticDir: string, tag: string, testLogin?: boolean,
baseDomain?: string
}) {
const {server, staticDir, tag, testLogin} = opts;
return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
// .invalid is a TLD the IETF promises will never exist.
const config = makeGristConfig(server ? server.getHomeUrl(req) : null, options.config,
opts.baseDomain, req);
const config = makeGristConfig({
homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
extra: options.config,
baseDomain: opts.baseDomain,
req,
server,
});
// We could cache file contents in memory, but the filesystem does caching too, and compared
// to that, the performance gain is unlikely to be meaningful. So keep it simple here.
@ -112,7 +125,7 @@ export function makeSendAppPage(opts: {
const tagManagerSnippet = needTagManager ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) : '';
const staticOrigin = process.env.APP_STATIC_URL || "";
const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
const customHeadHtmlSnippet = server?.create.getExtraHeadHtml?.() ?? "";
const customHeadHtmlSnippet = server.create.getExtraHeadHtml?.() ?? "";
const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
// Preload all languages that will be used or are requested by client.
const preloads = req.languages
@ -127,7 +140,7 @@ export function makeSendAppPage(opts: {
.replace("<!-- INSERT WARNING -->", warning)
.replace("<!-- INSERT TITLE -->", getPageTitle(req, config))
.replace("<!-- INSERT META -->", getPageMetadataHtmlSnippet(config))
.replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server?.getGristConfig()))
.replace("<!-- INSERT TITLE SUFFIX -->", getPageTitleSuffix(server.getGristConfig()))
.replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet)
.replace("<!-- INSERT LOCALE -->", preloads)
.replace("<!-- INSERT CUSTOM -->", customHeadHtmlSnippet)
@ -150,6 +163,13 @@ function getFeatures(): IFeature[] {
return Features.checkAll(difference(enabledFeatures, disabledFeatures));
}
function getTelemetryConfig(server: GristServer) {
const telemetry = server.getTelemetry();
return {
telemetryLevel: telemetry.getTelemetryLevel(),
};
}
function configuredPageTitleSuffix() {
const result = process.env.GRIST_PAGE_TITLE_SUFFIX;
return result === "_blank" ? "" : result;

13
app/server/lib/serverUtils.ts
View File

@ -7,6 +7,7 @@ import uuidv4 from 'uuid/v4';
import {AbortSignal} from 'node-abort-controller';
import {EngineCode} from 'app/common/DocumentSettings';
import {TelemetryMetadataByLevel} from 'app/common/Telemetry';
import log from 'app/server/lib/log';
import {OpenMode, SQLiteDB} from 'app/server/lib/SQLiteDB';
import {getDocSessionAccessOrNull, getDocSessionUser, OptDocSession} from './DocSession';
@ -166,14 +167,18 @@ export function getLogMetaFromDocSession(docSession: OptDocSession) {
/**
* Extract telemetry metadata from session.
*/
export function getTelemetryMetaFromDocSession(docSession: OptDocSession) {
export function getTelemetryMetaFromDocSession(docSession: OptDocSession): TelemetryMetadataByLevel {
const client = docSession.client;
const access = getDocSessionAccessOrNull(docSession);
const user = getDocSessionUser(docSession);
return {
access,
...(user ? {userId: user.id} : {}),
...(client ? client.getFullTelemetryMeta() : {}), // Client if present will repeat and add to user info.
limited: {
access,
},
full: {
...(user ? {userId: user.id} : {}),
...(client ? client.getFullTelemetryMeta() : {}), // Client if present will repeat and add to user info.
},
};
}

4
app/server/mergedServerMain.ts
View File

@ -119,19 +119,19 @@ export async function main(port: number, serverTypes: ServerType[],
server.addHomeApi();
server.addBillingApi();
server.addNotifier();
server.addTelemetry();
await server.addHousekeeper();
await server.addLoginRoutes();
server.addAccountPage();
server.addBillingPages();
server.addWelcomePaths();
server.addLogEndpoint();
server.addTelemetryEndpoint();
server.addGoogleAuthEndpoint();
}
if (includeDocs) {
server.addJsonSupport();
server.addTelemetryEndpoint();
server.addTelemetry();
await server.addDoc();
}

16
stubs/app/client/ui/BillingButtons.ts
View File

@ -1,16 +0,0 @@
import {AppModel} from 'app/client/models/AppModel';
import {DomElementArg} from 'grainjs';
export function buildUserMenuBillingItem(
_appModel: AppModel,
..._args: DomElementArg[]
) {
return null;
}
export function buildAppMenuBillingItem(
_appModel: AppModel,
..._args: DomElementArg[]
) {
return null;
}

1
stubs/app/client/ui/WelcomeCoachingCall.ts
View File

@ -1 +0,0 @@
export * from 'app/client/ui/WelcomeCoachingCallStub';

11
stubs/app/server/lib/TelemetryManager.ts
View File

@ -1,11 +0,0 @@
import {TelemetryEventName} from 'app/common/Telemetry';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
export class TelemetryManager {
constructor(_dbManager: HomeDBManager) {}
public logEvent(
_name: TelemetryEventName,
_metadata?: Record<string, any>
) {}
}

5
stubs/app/server/lib/create.ts
View File

@ -1,8 +1,10 @@
import { checkMinIOExternalStorage,
configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage';
import { makeSimpleCreator } from 'app/server/lib/ICreate';
import { Telemetry } from 'app/server/lib/Telemetry';
export const create = makeSimpleCreator({
deploymentType: 'core',
// This can and should be overridden by GRIST_SESSION_SECRET
// (or generated randomly per install, like grist-omnibus does).
sessionSecret: 'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh',
@ -13,4 +15,7 @@ export const create = makeSimpleCreator({
create: configureMinIOExternalStorage,
},
],
telemetry: {
create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer),
}
});

215
test/common/Telemetry.ts Normal file
View File

@ -0,0 +1,215 @@
import {buildTelemetryEventChecker, filterMetadata, TelemetryEvent} from 'app/common/Telemetry';
import {assert} from 'chai';
describe('Telemetry', function() {
describe('buildTelemetryEventChecker', function() {
it('returns a function that checks telemetry data', function() {
assert.isFunction(buildTelemetryEventChecker('full'));
});
it('does not throw if event and metadata are valid', function() {
const checker = buildTelemetryEventChecker('full');
assert.doesNotThrow(() => checker('apiUsage', {
method: 'GET',
userId: 1,
userAgent: 'node-fetch/1.0',
}));
assert.doesNotThrow(() => checker('siteUsage', {
siteId: 1,
siteType: 'team',
inGoodStanding: true,
stripePlanId: 'stripePlanId',
numDocs: 1,
numWorkspaces: 1,
numMembers: 1,
lastActivity: new Date('2022-12-30T01:23:45'),
}));
assert.doesNotThrow(() => checker('watchedVideoTour', {
watchTimeSeconds: 30,
userId: 1,
altSessionId: 'altSessionId',
}));
});
it("does not throw when metadata is a subset of what's expected", function() {
const checker = buildTelemetryEventChecker('full');
assert.doesNotThrow(() => checker('documentUsage', {
docIdDigest: 'docIdDigest',
siteId: 1,
rowCount: 123,
attachmentTypes: ['pdf'],
}));
});
it('does not throw if all metadata is less than or equal to the expected telemetry level', function() {
const checker = buildTelemetryEventChecker('limited');
assert.doesNotThrow(() => checker('documentUsage', {
rowCount: 123,
}));
assert.doesNotThrow(() => checker('siteUsage', {
siteId: 1,
siteType: 'team',
inGoodStanding: true,
numDocs: 1,
numWorkspaces: 1,
numMembers: 1,
lastActivity: new Date('2022-12-30T01:23:45'),
}));
assert.doesNotThrow(() => checker('watchedVideoTour', {
watchTimeSeconds: 30,
}));
});
it('throws if event is invalid', function() {
const checker = buildTelemetryEventChecker('full');
assert.throws(
() => checker('invalidEvent' as TelemetryEvent, {}),
/Unknown telemetry event: invalidEvent/
);
});
it('throws if metadata is invalid', function() {
const checker = buildTelemetryEventChecker('full');
assert.throws(
() => checker('apiUsage', {invalidMetadata: '123'}),
/Unknown metadata for telemetry event apiUsage: invalidMetadata/
);
});
it('throws if metadata types do not match expected types', function() {
const checker = buildTelemetryEventChecker('full');
assert.throws(
() => checker('siteUsage', {siteId: '1'}),
// eslint-disable-next-line max-len
/Telemetry metadata siteId of event siteUsage expected a value of type number but received a value of type string/
);
assert.throws(
() => checker('siteUsage', {lastActivity: 1234567890}),
// eslint-disable-next-line max-len
/Telemetry metadata lastActivity of event siteUsage expected a value of type Date or string but received a value of type number/
);
assert.throws(
() => checker('siteUsage', {inGoodStanding: 'true'}),
// eslint-disable-next-line max-len
/Telemetry metadata inGoodStanding of event siteUsage expected a value of type boolean but received a value of type string/
);
assert.throws(
() => checker('siteUsage', {numDocs: '1'}),
// eslint-disable-next-line max-len
/Telemetry metadata numDocs of event siteUsage expected a value of type number but received a value of type string/
);
assert.throws(
() => checker('documentUsage', {attachmentTypes: '1,2,3'}),
// eslint-disable-next-line max-len
/Telemetry metadata attachmentTypes of event documentUsage expected a value of type array but received a value of type string/
);
assert.throws(
() => checker('documentUsage', {attachmentTypes: ['.txt', 1, true]}),
// eslint-disable-next-line max-len
/Telemetry metadata attachmentTypes of event documentUsage expected a value of type string\[\] but received a value of type object\[\]/
);
});
it('throws if event requires an elevated telemetry level', function() {
const checker = buildTelemetryEventChecker('limited');
assert.throws(
() => checker('signupVerified', {}),
// eslint-disable-next-line max-len
/Telemetry event signupVerified requires a minimum telemetry level of 2 but the current level is 1/
);
});
it('throws if metadata requires an elevated telemetry level', function() {
const checker = buildTelemetryEventChecker('limited');
assert.throws(
() => checker('watchedVideoTour', {
watchTimeSeconds: 30,
userId: 1,
altSessionId: 'altSessionId',
}),
// eslint-disable-next-line max-len
/Telemetry metadata userId of event watchedVideoTour requires a minimum telemetry level of 2 but the current level is 1/
);
});
});
describe('filterMetadata', function() {
it('returns filtered and flattened metadata when maxLevel is "full"', function() {
const metadata = {
limited: {
foo: 'abc',
},
full: {
bar: '123',
},
};
assert.deepEqual(filterMetadata(metadata, 'full'), {
foo: 'abc',
bar: '123',
});
});
it('returns filtered and flattened metadata when maxLevel is "limited"', function() {
const metadata = {
limited: {
foo: 'abc',
},
full: {
bar: '123',
},
};
assert.deepEqual(filterMetadata(metadata, 'limited'), {
foo: 'abc',
});
});
it('returns undefined when maxLevel is "off"', function() {
assert.isUndefined(filterMetadata(undefined, 'off'));
});
it('returns an empty object when metadata is empty', function() {
assert.isEmpty(filterMetadata({}, 'full'));
});
it('returns undefined when metadata is undefined', function() {
assert.isUndefined(filterMetadata(undefined, 'full'));
});
it('does not mutate metadata', function() {
const metadata = {
limited: {
foo: 'abc',
},
full: {
bar: '123',
},
};
filterMetadata(metadata, 'limited');
assert.deepEqual(metadata, {
limited: {
foo: 'abc',
},
full: {
bar: '123',
},
});
});
it('excludes keys with nullish values', function() {
const metadata = {
limited: {
foo1: null,
foo2: 'abc',
},
full: {
bar1: undefined,
bar2: '123',
},
};
assert.deepEqual(filterMetadata(metadata, 'full'), {
foo2: 'abc',
bar2: '123',
});
});
});
});

6
test/nbrowser/testServer.ts
View File

@ -205,7 +205,11 @@ export class TestServerMerged implements IMochaServer {
}
const state: IGristUrlState = { org: team };
const baseDomain = parseSubdomain(new URL(this.getHost()).hostname).base;
const gristConfig = makeGristConfig(this.getHost(), {}, baseDomain);
const gristConfig = makeGristConfig({
homeUrl: this.getHost(),
extra: {},
baseDomain,
});
const url = encodeUrl(gristConfig, state, new URL(this.getHost())).replace(/\/$/, "");
return `${url}${relPath}`;
}

2
test/server/lib/Authorizer.ts
View File

@ -34,7 +34,7 @@ async function activateServer(home: FlexServer, docManager: DocManager) {
home.addJsonSupport();
await home.addLandingPages();
home.addHomeApi();
home.addTelemetryEndpoint();
home.addTelemetry();
await home.addDoc();
home.addApiErrorHandlers();
serverUrl = home.getOwnUrl();

309
test/server/lib/Telemetry.ts Normal file
View File

@ -0,0 +1,309 @@
import {GristDeploymentType} from 'app/common/gristUrls';
import {TelemetryEvent, TelemetryLevel, TelemetryMetadata} from 'app/common/Telemetry';
import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods';
import {ITelemetry, Telemetry} from 'app/server/lib/Telemetry';
import axios from 'axios';
import {assert} from 'chai';
import * as sinon from 'sinon';
import {TestServer} from 'test/gen-server/apiUtils';
import {configForUser} from 'test/gen-server/testUtils';
const chimpy = configForUser('Chimpy');
const anon = configForUser('Anonymous');
describe('Telemetry', function() {
const deploymentTypesAndTelemetryLevels: [GristDeploymentType, TelemetryLevel][] = [
['saas', 'off'],
['saas', 'limited'],
['saas', 'full'],
['core', 'off'],
['core', 'limited'],
['core', 'full'],
];
for (const [deploymentType, telemetryLevel] of deploymentTypesAndTelemetryLevels) {
describe(`in grist-${deploymentType} with a telemetry level of "${telemetryLevel}"`, function() {
let homeUrl: string;
let installationId: string;
let server: TestServer;
let telemetry: ITelemetry;
const sandbox = sinon.createSandbox();
const loggedEvents: [TelemetryEvent, ILogMeta][] = [];
const forwardedEvents: [TelemetryEvent, TelemetryMetadata | undefined][] = [];
before(async function() {
process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE = deploymentType;
process.env.GRIST_TELEMETRY_LEVEL = telemetryLevel;
server = new TestServer(this);
homeUrl = await server.start();
installationId = (await server.server.getActivations().current()).id;
sandbox
.stub(LogMethods.prototype, 'rawLog')
.callsFake((_level: string, _info: unknown, name: string, meta: ILogMeta) => {
loggedEvents.push([name as TelemetryEvent, meta]);
});
sandbox
.stub(Telemetry.prototype, 'forwardEvent')
.callsFake((event: TelemetryEvent, metadata?: TelemetryMetadata) => {
forwardedEvents.push([event, metadata]);
});
telemetry = server.server.getTelemetry();
});
after(async function() {
await server.stop();
sandbox.restore();
delete process.env.GRIST_TEST_SERVER_DEPLOYMENT_TYPE;
delete process.env.GRIST_TELEMETRY_LEVEL;
});
it('returns the current telemetry level', async function() {
assert.equal(telemetry.getTelemetryLevel(), telemetryLevel);
});
if (telemetryLevel !== 'off') {
if (deploymentType === 'saas') {
it('logs telemetry events', async function() {
if (telemetryLevel === 'limited') {
await telemetry.logEvent('documentOpened', {
limited: {
docIdDigest: 'digest',
isPublic: false,
},
});
assert.deepEqual(loggedEvents[loggedEvents.length - 1], [
'documentOpened',
{
eventName: 'documentOpened',
eventSource: `grist-${deploymentType}`,
docIdDigest: 'digest',
isPublic: false,
}
]);
}
if (telemetryLevel === 'full') {
await telemetry.logEvent('documentOpened', {
limited: {
docIdDigest: 'digest',
isPublic: false,
},
full: {
userId: 1,
},
});
assert.deepEqual(loggedEvents[loggedEvents.length - 1], [
'documentOpened',
{
eventName: 'documentOpened',
eventSource: `grist-${deploymentType}`,
docIdDigest: 'digest',
isPublic: false,
userId: 1,
}
]);
}
assert.equal(loggedEvents.length, 1);
assert.isEmpty(forwardedEvents);
});
} else {
it('forwards telemetry events', async function() {
if (telemetryLevel === 'limited') {
await telemetry.logEvent('documentOpened', {
limited: {
docIdDigest: 'digest',
isPublic: false,
},
});
assert.deepEqual(forwardedEvents[forwardedEvents.length - 1], [
'documentOpened',
{
docIdDigest: 'digest',
isPublic: false,
}
]);
}
if (telemetryLevel === 'full') {
await telemetry.logEvent('documentOpened', {
limited: {
docIdDigest: 'digest',
isPublic: false,
},
full: {
userId: 1,
},
});
assert.deepEqual(forwardedEvents[forwardedEvents.length - 1], [
'documentOpened',
{
docIdDigest: 'digest',
isPublic: false,
userId: 1,
}
]);
}
assert.equal(forwardedEvents.length, 1);
assert.isEmpty(loggedEvents);
});
}
} else {
it('does not log telemetry events', async function() {
await telemetry.logEvent('documentOpened', {
limited: {
docIdDigest: 'digest',
isPublic: false,
},
});
assert.isEmpty(loggedEvents);
assert.isEmpty(forwardedEvents);
});
}
if (telemetryLevel !== 'off') {
it('throws an error when an event is invalid', async function() {
await assert.isRejected(
telemetry.logEvent('invalidEvent' as TelemetryEvent, {limited: {method: 'GET'}}),
/Unknown telemetry event: invalidEvent/
);
});
it("throws an error when an event's metadata is invalid", async function() {
await assert.isRejected(
telemetry.logEvent('documentOpened', {limited: {invalidMetadata: 'GET'}}),
/Unknown metadata for telemetry event documentOpened: invalidMetadata/
);
});
if (telemetryLevel === 'limited') {
it('throws an error when an event requires an elevated telemetry level', async function() {
await assert.isRejected(
telemetry.logEvent('signupVerified', {}),
/Telemetry event signupVerified requires a minimum telemetry level of 2 but the current level is 1/
);
});
it("throws an error when an event's metadata requires an elevated telemetry level", async function() {
await assert.isRejected(
telemetry.logEvent('documentOpened', {limited: {userId: 1}}),
// eslint-disable-next-line max-len
/Telemetry metadata userId of event documentOpened requires a minimum telemetry level of 2 but the current level is 1/
);
});
}
}
if (telemetryLevel !== 'off') {
if (deploymentType === 'saas') {
it('logs telemetry events sent to /api/telemetry', async function() {
await axios.post(`${homeUrl}/api/telemetry`, {
event: 'watchedVideoTour',
metadata: {
limited: {watchTimeSeconds: 30},
},
}, chimpy);
const [event, metadata] = loggedEvents[loggedEvents.length - 1];
assert.equal(event, 'watchedVideoTour');
if (telemetryLevel === 'limited') {
assert.deepEqual(metadata, {
eventName: 'watchedVideoTour',
eventSource: `grist-${deploymentType}`,
watchTimeSeconds: 30,
});
} else {
assert.containsAllKeys(metadata, [
'eventSource',
'watchTimeSeconds',
'userId',
'altSessionId',
]);
assert.equal(metadata.watchTimeSeconds, 30);
assert.equal(metadata.userId, 1);
}
if (telemetryLevel === 'limited') {
assert.equal(loggedEvents.length, 2);
} else {
// The POST above also triggers an "apiUsage" event.
assert.equal(loggedEvents.length, 3);
assert.equal(loggedEvents[1][0], 'apiUsage');
}
assert.isEmpty(forwardedEvents);
});
if (telemetryLevel === 'limited') {
it('skips checks if event sent to /api/telemetry is from an external source', async function() {
await axios.post(`${homeUrl}/api/telemetry`, {
event: 'watchedVideoTour',
metadata: {
eventSource: 'grist-core',
watchTimeSeconds: 60,
userId: 123,
altSessionId: 'altSessionId',
},
}, anon);
const [event, metadata] = loggedEvents[loggedEvents.length - 1];
assert.equal(event, 'watchedVideoTour');
assert.containsAllKeys(metadata, [
'eventSource',
'watchTimeSeconds',
'userId',
'altSessionId',
]);
assert.equal(metadata.watchTimeSeconds, 60);
assert.equal(metadata.userId, 123);
assert.equal(loggedEvents.length, 3);
assert.isEmpty(forwardedEvents);
});
}
} else {
it('forwards telemetry events sent to /api/telemetry', async function() {
await axios.post(`${homeUrl}/api/telemetry`, {
event: 'watchedVideoTour',
metadata: {
limited: {watchTimeSeconds: 30},
},
}, chimpy);
const [event, metadata] = forwardedEvents[forwardedEvents.length - 1];
assert.equal(event, 'watchedVideoTour');
if (telemetryLevel === 'limited') {
assert.deepEqual(metadata, {
eventSource: `grist-${deploymentType}`,
installationId,
watchTimeSeconds: 30,
});
} else {
assert.containsAllKeys(metadata, [
'eventSource',
'installationId',
'watchTimeSeconds',
'userId',
'altSessionId',
]);
assert.equal(metadata!.watchTimeSeconds, 30);
assert.equal(metadata!.userId, 1);
}
if (telemetryLevel === 'limited') {
assert.equal(forwardedEvents.length, 2);
} else {
// The POST above also triggers an "apiUsage" event.
assert.equal(forwardedEvents.length, 3);
assert.equal(forwardedEvents[1][0], 'apiUsage');
}
assert.isEmpty(loggedEvents);
});
}
} else {
it('does not log telemetry events sent to /api/telemetry', async function() {
await telemetry.logEvent('apiUsage', {limited: {method: 'GET'}});
assert.isEmpty(loggedEvents);
assert.isEmpty(forwardedEvents);
});
}
});
}
});