Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) Add telemetry

Browse Source 
Test Plan: Server tests.

Reviewers: jarek

Differential Revision: https://phab.getgrist.com/D3818
This commit is contained in:
George Gevoian 2023-04-06 11:10:29 -04:00
parent 6a4b7d96e8
commit a19ba0813a
28 changed files with 555 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
app/client/lib/helpScout.ts
View File

@ -15,6 +15,7 @@
// tslint:disable:unified-signatures
import {logTelemetryEvent} from 'app/client/lib/telemetry';
import {AppModel} from 'app/client/models/AppModel';
import {reportWarning} from 'app/client/models/errors';
import {IAppError} from 'app/client/models/NotifyModel';
@ -54,6 +55,11 @@ interface ISessionData {
[key: string]: string;
}
interface ICallbackAttributes {
id?: string;
query?: string;
}
/**
* This provides the HelpScout Beacon API, taking care of initializing Beacon on first use.
*/
@ -65,7 +71,8 @@ export function Beacon(method: 'navigate', route: string): void;
export function Beacon(method: 'identify', userObj: IUserObj): void;
export function Beacon(method: 'prefill', formObj: IFormObj): void;
export function Beacon(method: 'config', configObj: object): void;
export function Beacon(method: 'on'|'once', event: string, callback: () => void): void;
export function Beacon(method: 'on'|'once', event: string,
callback: (attrs?: ICallbackAttributes) => void): void;
export function Beacon(method: 'off', event: string, callback?: () => void): void;
export function Beacon(method: 'session-data', data: ISessionData): void;
export function Beacon(method: BeaconCmd): void;
@ -187,6 +194,15 @@ function _beaconOpen(userObj: IUserObj|null, options: IBeaconOpenOptions) {
if (!skipNav) {
Beacon('navigate', route);
}
Beacon('once', 'open', () => logTelemetryEvent('beaconOpen'));
Beacon('on', 'article-viewed', (article) => logTelemetryEvent('beaconArticleViewed', {
articleId: article!.id,
}));
Beacon('on', 'email-sent', () => logTelemetryEvent('beaconEmailSent'));
Beacon('on', 'search', (search) => logTelemetryEvent('beaconSearch', {
searchQuery: search!.query,
}));
}
function fixBeaconBaseHref() {

23
app/client/lib/telemetry.ts Normal file
View File

@ -0,0 +1,23 @@
import {logError} from 'app/client/models/errors';
import {TelemetryEventName} from 'app/common/Telemetry';
import {fetchFromHome, pageHasHome} from 'app/common/urlUtils';
export async function logTelemetryEvent(name: TelemetryEventName, metadata?: Record<string, any>) {
if (!pageHasHome()) { return; }
await fetchFromHome('/api/telemetry', {
method: 'POST',
body: JSON.stringify({
name,
metadata,
}),
credentials: 'include',
headers: {
'Content-Type': 'application/json',
'X-Requested-With': 'XMLHttpRequest',
},
}).catch((e: Error) => {
console.warn(`Failed to log telemetry event ${name}`, e);
logError(e);
});
}

6
app/client/models/errors.ts
View File

@ -59,7 +59,7 @@ export function reportMessage(msg: MessageType, options?: Partial<INotifyOptions
export function reportWarning(msg: string, options?: Partial<INotifyOptions>) {
options = {level: 'warning', ...options};
log.warn(`${options.level}: `, msg);
_logError(msg);
logError(msg);
return reportMessage(msg, options);
}
@ -84,7 +84,7 @@ export function reportError(err: Error|string): void {
// This error can be emitted while a page is reloaded, and isn't worth reporting.
return;
}
_logError(err);
logError(err);
if (_notifier && !_notifier.isDisposed()) {
if (!isError(err)) {
err = new Error(String(err));
@ -175,7 +175,7 @@ export function setUpErrorHandling(doReportError = reportError, koUtil?: any) {
* over-logging (regular errors such as access rights or account limits) and
* under-logging (javascript errors during startup might never get reported).
*/
function _logError(error: Error|string) {
export function logError(error: Error|string) {
if (!pageHasHome()) { return; }
const docId = G.window.gristDocPageModel?.currentDocId?.get();
fetchFromHome('/api/log', {

35
app/client/ui/OpenVideoTour.ts
View File

@ -1,5 +1,6 @@
import * as commands from 'app/client/components/commands';
import {makeT} from 'app/client/lib/localization';
import {logTelemetryEvent} from 'app/client/lib/telemetry';
import {getMainOrgUrl} from 'app/client/models/gristUrlState';
import {cssLinkText, cssPageEntryMain, cssPageIcon, cssPageLink} from 'app/client/ui/LeftPanelCommon';
import {YouTubePlayer} from 'app/client/ui/YouTubePlayer';
@ -20,7 +21,26 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
*/
export function openVideoTour(refElement: HTMLElement) {
return modal(
(ctl) => {
(ctl, owner) => {
const youtubePlayer = YouTubePlayer.create(owner,
VIDEO_TOUR_YOUTUBE_EMBED_ID,
{
onPlayerReady: (player) => player.playVideo(),
height: '100%',
width: '100%',
origin: getMainOrgUrl(),
},
cssYouTubePlayer.cls(''),
);
owner.onDispose(async () => {
if (youtubePlayer.isLoading()) { return; }
await logTelemetryEvent('watchedVideoTour', {
watchTimeSeconds: Math.floor(youtubePlayer.getCurrentTime()),
});
});
return [
cssModal.cls(''),
cssModalCloseButton(
@ -28,18 +48,7 @@ const VIDEO_TOUR_YOUTUBE_EMBED_ID = 'qnr2Pfnxdlc';
dom.on('click', () => ctl.close()),
testId('close'),
),
cssYouTubePlayerContainer(
dom.create(YouTubePlayer,
VIDEO_TOUR_YOUTUBE_EMBED_ID,
{
onPlayerReady: (player) => player.playVideo(),
height: '100%',
width: '100%',
origin: getMainOrgUrl(),
},
cssYouTubePlayer.cls(''),
),
),
cssYouTubePlayerContainer(youtubePlayer.buildDom()),
testId('modal'),
];
},

9
app/client/ui/YouTubePlayer.ts
View File

@ -10,6 +10,7 @@ export interface Player {
mute(): void;
unMute(): void;
setVolume(volume: number): void;
getCurrentTime(): number;
}
export interface PlayerOptions {
@ -80,6 +81,10 @@ export class YouTubePlayer extends Disposable {
}
}
public isLoading() {
return this._isLoading();
}
public isLoaded() {
return waitObs(this._isLoading, (val) => !val);
}
@ -92,6 +97,10 @@ export class YouTubePlayer extends Disposable {
this._player.setVolume(volume);
}
public getCurrentTime(): number {
return this._player.getCurrentTime();
}
public buildDom() {
return dom('div', {id: this._playerId}, ...this._domArgs);
}

21
app/common/Telemetry.ts Normal file
View File

@ -0,0 +1,21 @@
export const TelemetryTemplateSignupCookieName = 'gr_template_signup_trk';
export const TelemetryEventNames = [
'apiUsage',
'beaconOpen',
'beaconArticleViewed',
'beaconEmailSent',
'beaconSearch',
'documentForked',
'documentOpened',
'documentUsage',
'sendingWebhooks',
'signupVerified',
'siteMembership',
'siteUsage',
'tutorialProgressChanged',
'tutorialRestarted',
'watchedVideoTour',
] as const;
export type TelemetryEventName = typeof TelemetryEventNames[number];

1
app/common/UserAPI.ts
View File

@ -125,6 +125,7 @@ export interface DocumentOptions {
export interface TutorialMetadata {
lastSlideIndex?: number;
numSlides?: number;
}
export interface DocumentProperties extends CommonProperties {

1
app/common/gristUrls.ts
View File

@ -79,6 +79,7 @@ export const commonUrls = {
efcrHelp: 'https://www.nioxus.info/eFCR-Help',
basicTutorial: 'https://templates.getgrist.com/woXtXUBmiN5T/Grist-Basics',
basicTutorialImage: 'https://www.getgrist.com/wp-content/uploads/2021/08/lightweight-crm.png',
gristLabsCustomWidgets: 'https://gristlabs.github.io/grist-widget/',
};
/**

4
app/common/schema.ts
View File

@ -4,7 +4,7 @@ import { GristObjCode } from "app/plugin/GristData";
// tslint:disable:object-literal-key-quotes
export const SCHEMA_VERSION = 36;
export const SCHEMA_VERSION = 37;
export const schema = {
@ -148,6 +148,7 @@ export const schema = {
fileName : "Text",
fileType : "Text",
fileSize : "Int",
fileExt : "Text",
imageHeight : "Int",
imageWidth : "Int",
timeDeleted : "DateTime",
@ -354,6 +355,7 @@ export interface SchemaTypes {
fileName: string;
fileType: string;
fileSize: number;
fileExt: string;
imageHeight: number;
imageWidth: number;
timeDeleted: number;

29
app/gen-server/entity/Document.ts
View File

@ -2,7 +2,8 @@ import {ApiError} from 'app/common/ApiError';
import {DocumentUsage} from 'app/common/DocUsage';
import {Role} from 'app/common/roles';
import {DocumentOptions, DocumentProperties, documentPropertyKeys,
DocumentType, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
DocumentType, NEW_DOCUMENT_CODE, TutorialMetadata} from "app/common/UserAPI";
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {nativeValues} from 'app/gen-server/lib/values';
import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
import {AclRuleDoc} from "./AclRule";
@ -90,7 +91,7 @@ export class Document extends Resource {
return super.checkProperties(props, documentPropertyKeys);
}
public updateFromProperties(props: Partial<DocumentProperties>) {
public updateFromProperties(props: Partial<DocumentProperties>, dbManager?: HomeDBManager) {
super.updateFromProperties(props);
if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
if (props.urlId !== undefined) {
@ -131,6 +132,12 @@ export class Document extends Resource {
if (props.options.tutorial.lastSlideIndex !== undefined) {
this.options.tutorial.lastSlideIndex = props.options.tutorial.lastSlideIndex;
}
if (props.options.tutorial.numSlides !== undefined) {
this.options.tutorial.numSlides = props.options.tutorial.numSlides;
if (dbManager && props.options?.tutorial?.lastSlideIndex !== undefined) {
this._emitTutorialProgressChangeEvent(dbManager, this.options.tutorial);
}
}
}
}
// Normalize so that null equates with absence.
@ -146,6 +153,24 @@ export class Document extends Resource {
}
}
}
private _emitTutorialProgressChangeEvent(
dbManager: HomeDBManager,
tutorialMetadata: TutorialMetadata
) {
const lastSlideIndex = tutorialMetadata.lastSlideIndex;
const numSlides = tutorialMetadata.numSlides;
const percentComplete = lastSlideIndex !== undefined && numSlides !== undefined
? Math.floor((lastSlideIndex / numSlides) * 100)
: undefined;
dbManager?.emit('tutorialProgressChange', {
tutorialForkId: this.id,
tutorialTrunkId: this.trunkId,
lastSlideIndex,
numSlides,
percentComplete,
});
}
}
// Check that icon points to an expected location. This will definitely

13
app/gen-server/lib/HomeDBManager.ts
View File

@ -89,6 +89,14 @@ export const NotifierEvents = StringUnion(
export type NotifierEvent = typeof NotifierEvents.type;
export const TelemetryEvents = StringUnion(
'tutorialProgressChange',
);
export type TelemetryEvent = typeof TelemetryEvents.type;
export type Event = NotifierEvent | TelemetryEvent;
// Nominal email address of a user who can view anything (for thumbnails).
export const PREVIEWER_EMAIL = 'thumbnail@getgrist.com';
@ -276,6 +284,7 @@ export class HomeDBManager extends EventEmitter {
// In restricted mode, documents should be read-only.
private _restrictedMode: boolean = false;
/**
* Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers',
* 'guests', and 'members') are created by default on every new entity (Organization,
@ -315,7 +324,7 @@ export class HomeDBManager extends EventEmitter {
orgOnly: true
}];
public emit(event: NotifierEvent, ...args: any[]): boolean {
public emit(event: Event, ...args: any[]): boolean {
return super.emit(event, ...args);
}
@ -1944,7 +1953,7 @@ export class HomeDBManager extends EventEmitter {
// Update the name and save.
const doc: Document = queryResult.data;
doc.checkProperties(props);
doc.updateFromProperties(props);
doc.updateFromProperties(props, this);
if (forkId) {
await manager.save(doc);
return {status: 200};

98
app/gen-server/lib/Housekeeper.ts
View File

@ -1,6 +1,7 @@
import { ApiError } from 'app/common/ApiError';
import { buildUrlId } from 'app/common/gristUrls';
import { Document } from 'app/gen-server/entity/Document';
import { Organization } from 'app/gen-server/entity/Organization';
import { Workspace } from 'app/gen-server/entity/Workspace';
import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
import { fromNow } from 'app/gen-server/sqlUtils';
@ -14,6 +15,7 @@ import { optStringParam, stringParam } from 'app/server/lib/requestUtils';
import * as express from 'express';
import fetch from 'node-fetch';
import * as Fetch from 'node-fetch';
import { EntityManager } from 'typeorm';
const HOUSEKEEPER_PERIOD_MS = 1 * 60 * 60 * 1000; // operate every 1 hour
const AGE_THRESHOLD_OFFSET = '-30 days'; // should be an interval known by postgres + sqlite
@ -23,6 +25,7 @@ const AGE_THRESHOLD_OFFSET = '-30 days'; // should be an interval kno
*
* - deleting old soft-deleted documents
* - deleting old soft-deleted workspaces
* - logging metrics
*
* Call start(), keep the object around, and call stop() when shutting down.
*
@ -42,7 +45,9 @@ export class Housekeeper {
*/
public async start() {
await this.stop();
this._interval = setInterval(() => this.deleteTrashExclusively().catch(log.warn.bind(log)), HOUSEKEEPER_PERIOD_MS);
this._interval = setInterval(() => {
this.doHousekeepingExclusively().catch(log.warn.bind(log));
}, HOUSEKEEPER_PERIOD_MS);
}
/**
@ -56,16 +61,18 @@ export class Housekeeper {
}
/**
* Deletes old trash if no other server is working on it or worked on it recently.
* Deletes old trash and logs metrics if no other server is working on it or worked on it
* recently.
*/
public async deleteTrashExclusively(): Promise<boolean> {
const electionKey = await this._electionStore.getElection('trash', HOUSEKEEPER_PERIOD_MS / 2.0);
public async doHousekeepingExclusively(): Promise<boolean> {
const electionKey = await this._electionStore.getElection('housekeeping', HOUSEKEEPER_PERIOD_MS / 2.0);
if (!electionKey) {
log.info('Skipping deleteTrash since another server is working on it or worked on it recently');
log.info('Skipping housekeeping since another server is working on it or worked on it recently');
return false;
}
this._electionKey = electionKey;
await this.deleteTrash();
await this.logMetrics();
return true;
}
@ -145,6 +152,39 @@ export class Housekeeper {
}
}
/**
* Logs metrics regardless of what other servers may be doing.
*/
public async logMetrics() {
await this._dbManager.connection.transaction('READ UNCOMMITTED', async (manager) => {
const telemetryManager = this._server.getTelemetryManager();
const usageSummaries = await this._getOrgUsageSummaries(manager);
for (const summary of usageSummaries) {
telemetryManager?.logEvent('siteUsage', {
siteId: summary.site_id,
siteType: summary.site_type,
inGoodStanding: summary.in_good_standing,
stripePlanId: summary.stripe_plan_id,
numDocs: summary.num_docs,
numWorkspaces: summary.num_workspaces,
numMembers: summary.num_members,
lastActivity: summary.last_activity,
});
}
const membershipSummaries = await this._getOrgMembershipSummaries(manager);
for (const summary of membershipSummaries) {
telemetryManager?.logEvent('siteMembership', {
siteId: summary.site_id,
siteType: summary.site_type,
numOwners: summary.num_owners,
numEditors: summary.num_editors,
numViewers: summary.num_viewers,
});
}
});
}
public addEndpoints(app: express.Application) {
// Allow support user to perform housekeeping tasks for a specific
// document. The tasks necessarily bypass user access controls.
@ -208,7 +248,7 @@ export class Housekeeper {
*/
public async testClearExclusivity(): Promise<void> {
if (this._electionKey) {
await this._electionStore.removeElection('trash', this._electionKey);
await this._electionStore.removeElection('housekeeping', this._electionKey);
this._electionKey = undefined;
}
}
@ -249,6 +289,52 @@ export class Housekeeper {
return forks;
}
private async _getOrgUsageSummaries(manager: EntityManager) {
const orgs = await manager.createQueryBuilder()
.select('orgs.id', 'site_id')
.addSelect('products.name', 'site_type')
.addSelect('billing_accounts.in_good_standing', 'in_good_standing')
.addSelect('billing_accounts.stripe_plan_id', 'stripe_plan_id')
.addSelect('COUNT(DISTINCT docs.id)', 'num_docs')
.addSelect('COUNT(DISTINCT workspaces.id)', 'num_workspaces')
.addSelect('COUNT(DISTINCT org_member_users.id)', 'num_members')
.addSelect('MAX(docs.updated_at)', 'last_activity')
.from(Organization, 'orgs')
.leftJoin('orgs.workspaces', 'workspaces')
.leftJoin('workspaces.docs', 'docs')
.leftJoin('orgs.billingAccount', 'billing_accounts')
.leftJoin('billing_accounts.product', 'products')
.leftJoin('orgs.aclRules', 'acl_rules')
.leftJoin('acl_rules.group', 'org_groups')
.leftJoin('org_groups.memberUsers', 'org_member_users')
.where('org_member_users.id IS NOT NULL')
.groupBy('orgs.id')
.addGroupBy('products.id')
.addGroupBy('billing_accounts.id')
.getRawMany();
return orgs;
}
private async _getOrgMembershipSummaries(manager: EntityManager) {
const orgs = await manager.createQueryBuilder()
.select('orgs.id', 'site_id')
.addSelect('products.name', 'site_type')
.addSelect("SUM(CASE WHEN org_groups.name = 'owners' THEN 1 ELSE 0 END)", 'num_owners')
.addSelect("SUM(CASE WHEN org_groups.name = 'editors' THEN 1 ELSE 0 END)", 'num_editors')
.addSelect("SUM(CASE WHEN org_groups.name = 'viewers' THEN 1 ELSE 0 END)", 'num_viewers')
.from(Organization, 'orgs')
.leftJoin('orgs.billingAccount', 'billing_accounts')
.leftJoin('billing_accounts.product', 'products')
.leftJoin('orgs.aclRules', 'acl_rules')
.leftJoin('acl_rules.group', 'org_groups')
.leftJoin('org_groups.memberUsers', 'org_member_users')
.where('org_member_users.id IS NOT NULL')
.groupBy('orgs.id')
.addGroupBy('products.id')
.getRawMany();
return orgs;
}
/**
* TypeORM isn't very adept at handling date representation for
* comparisons, so we construct the threshold date in SQL so that we

5
app/server/generateInitialDocSql.ts
View File

@ -31,7 +31,10 @@ export async function main(baseName: string) {
if (await fse.pathExists(fname)) {
await fse.remove(fname);
}
const docManager = new DocManager(storageManager, pluginManager, null as any, {create} as any);
const docManager = new DocManager(storageManager, pluginManager, null as any, {
create,
getTelemetryManager: () => undefined,
} as any);
const activeDoc = new ActiveDoc(docManager, baseName);
const session = makeExceptionalDocSession('nascent');
await activeDoc.createEmptyDocWithDataEngine(session);

205
app/server/lib/ActiveDoc.ts
View File

@ -65,19 +65,22 @@ import {
import {normalizeEmail} from 'app/common/emails';
import {Product} from 'app/common/Features';
import {FormulaProperties, getFormulaProperties} from 'app/common/GranularAccessClause';
import {parseUrlId} from 'app/common/gristUrls';
import {isHiddenCol} from 'app/common/gristTypes';
import {commonUrls, parseUrlId} from 'app/common/gristUrls';
import {byteString, countIf, retryOnce, safeJsonParse} from 'app/common/gutil';
import {InactivityTimer} from 'app/common/InactivityTimer';
import {Interval} from 'app/common/Interval';
import * as roles from 'app/common/roles';
import {schema, SCHEMA_VERSION} from 'app/common/schema';
import {MetaRowRecord, SingleCell} from 'app/common/TableData';
import {TelemetryEventName} from 'app/common/Telemetry';
import {UIRowId} from 'app/common/UIRowId';
import {FetchUrlOptions, UploadResult} from 'app/common/uploads';
import {DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
import {Document as APIDocument, DocReplacementOptions, DocState, DocStateComparison} from 'app/common/UserAPI';
import {convertFromColumn} from 'app/common/ValueConverter';
import {guessColInfoWithDocData} from 'app/common/ValueGuesser';
import {parseUserAction} from 'app/common/ValueParser';
import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
import {Document} from 'app/gen-server/entity/Document';
import {ParseOptions} from 'app/plugin/FileParserAPI';
import {AccessTokenOptions, AccessTokenResult, GristDocAPI} from 'app/plugin/GristAPI';
@ -117,6 +120,7 @@ import {DocPluginManager} from './DocPluginManager';
import {
DocSession,
getDocSessionAccess,
getDocSessionAltSessionId,
getDocSessionUser,
getDocSessionUserId,
makeExceptionalDocSession,
@ -157,6 +161,9 @@ const UPDATE_CURRENT_TIME_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 100
// Measure and broadcast data size every 5 minutes
const UPDATE_DATA_SIZE_DELAY = {delayMs: 5 * 60 * 1000, varianceMs: 30 * 1000};
// Log document metrics every hour
const LOG_DOCUMENT_METRICS_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 1000};
// A hook for dependency injection.
export const Deps = {ACTIVEDOC_TIMEOUT};
@ -215,6 +222,7 @@ export class ActiveDoc extends EventEmitter {
private _fullyLoaded: boolean = false; // Becomes true once all columns are loaded/computed.
private _lastMemoryMeasurement: number = 0; // Timestamp when memory was last measured.
private _fetchCache = new MapWithTTL<string, Promise<TableDataAction>>(DEFAULT_CACHE_TTL);
private _doc: Document|undefined;
private _docUsage: DocumentUsage|null = null;
private _product?: Product;
private _gracePeriodStart: Date|null = null;
@ -260,6 +268,12 @@ export class ActiveDoc extends EventEmitter {
UPDATE_DATA_SIZE_DELAY,
{onError: (e) => this._log.error(null, 'failed to update data size', e)},
),
// Log document metrics every hour.
new Interval(
() => this._logDocMetrics(makeExceptionalDocSession('system'), 'interval'),
LOG_DOCUMENT_METRICS_DELAY,
{onError: (e) => this._log.error(null, 'failed to log document metrics', e)},
),
];
constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
@ -268,7 +282,8 @@ export class ActiveDoc extends EventEmitter {
this._isForkOrSnapshot = Boolean(forkId || snapshotId);
if (_options?.safeMode) { this._recoveryMode = true; }
if (_options?.doc) {
const {gracePeriodStart, workspace, usage} = _options.doc;
this._doc = _options.doc;
const {gracePeriodStart, workspace, usage} = this._doc;
const billingAccount = workspace.org.billingAccount;
this._product = billingAccount?.product;
this._gracePeriodStart = gracePeriodStart;
@ -1348,6 +1363,17 @@ export class ActiveDoc extends EventEmitter {
}
await dbManager.forkDoc(userId, doc, forkIds.forkId);
// TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
this.logTelemetryEvent(docSession, 'documentForked', {
forkId: forkIds.forkId,
forkDocId: forkIds.docId,
forkUrlId: forkIds.urlId,
trunkId: doc.trunkId,
isTemplate,
lastActivity: doc.updatedAt,
});
} finally {
await permitStore.removePermit(permitKey);
}
@ -1720,6 +1746,17 @@ export class ActiveDoc extends EventEmitter {
return this._triggers.summary();
}
public logTelemetryEvent(
docSession: OptDocSession | null,
eventName: TelemetryEventName,
metadata?: Record<string, any>
) {
this._docManager.gristServer.getTelemetryManager()?.logEvent(eventName, {
...this._getTelemetryMeta(docSession),
...metadata,
});
}
/**
* Loads an open document from DocStorage. Returns a list of the tables it contains.
*/
@ -1883,6 +1920,7 @@ export class ActiveDoc extends EventEmitter {
}
this._syncDocUsageToDatabase(true);
this._logDocMetrics(docSession, 'docClose');
try {
await this._docManager.storageManager.closeDocument(this.docName);
@ -2101,6 +2139,7 @@ export class ActiveDoc extends EventEmitter {
// We used to set fileType, but it's not easily available for native types. Since it's
// also entirely unused, we just skip it until it becomes relevant.
fileSize: fileData.size,
fileExt: fileData.ext,
imageHeight: dimensions.height,
imageWidth: dimensions.width,
timeUploaded: Date.now()
@ -2238,6 +2277,147 @@ export class ActiveDoc extends EventEmitter {
}
}
private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
this.logTelemetryEvent(docSession, 'documentUsage', {
triggeredBy,
access: this._doc?.access,
isPublic: ((this._doc as unknown) as APIDocument)?.public ?? false,
rowCount: this._docUsage?.rowCount?.total,
dataSizeBytes: this._docUsage?.dataSizeBytes,
attachmentsSize: this._docUsage?.attachmentsSizeBytes,
...this._getAccessRuleMetrics(),
...this._getAttachmentMetrics(),
...this._getChartMetrics(),
...this._getWidgetMetrics(),
...this._getColumnMetrics(),
...this._getTableMetrics(),
...this._getCustomWidgetMetrics(),
});
}
private _getAccessRuleMetrics() {
const accessRules = this.docData?.getMetaTable('_grist_ACLRules');
const numAccessRules = accessRules?.numRecords() ?? 0;
const numUserAttributes = accessRules?.getRecords().filter(r => r.userAttributes).length ?? 0;
return {
numAccessRules,
numUserAttributes,
};
}
private _getAttachmentMetrics() {
const attachments = this.docData?.getMetaTable('_grist_Attachments');
const numAttachments = attachments?.numRecords() ?? 0;
const attachmentTypes = attachments?.getRecords()
.map(r => r.fileExt?.slice(1) ?? null)
.filter(ext => ext !== null);
return {
numAttachments,
attachmentTypes,
};
}
private _getChartMetrics() {
const viewSections = this.docData?.getMetaTable('_grist_Views_section');
const viewSectionRecords = viewSections?.getRecords() ?? [];
const chartRecords = viewSectionRecords?.filter(r => r.parentKey === 'chart') ?? [];
const chartTypes = chartRecords.map(r => r.chartType || 'bar');
const numCharts = chartRecords.length;
const numLinkedCharts = chartRecords.filter(r => r.linkSrcSectionRef).length;
return {
numCharts,
chartTypes,
numLinkedCharts,
};
}
private _getWidgetMetrics() {
const viewSections = this.docData?.getMetaTable('_grist_Views_section');
const viewSectionRecords = viewSections?.getRecords() ?? [];
const numLinkedWidgets = viewSectionRecords.filter(r => r.linkSrcSectionRef).length;
return {
numLinkedWidgets,
};
}
private _getColumnMetrics() {
const columns = this.docData?.getMetaTable('_grist_Tables_column');
const columnRecords = columns?.getRecords().filter(r => !isHiddenCol(r.colId)) ?? [];
const numColumns = columnRecords.length;
const numColumnsWithConditionalFormatting = columnRecords.filter(r => r.rules).length;
const numFormulaColumns = columnRecords.filter(r => r.isFormula && r.formula).length;
const numTriggerFormulaColumns = columnRecords.filter(r => !r.isFormula && r.formula).length;
const tables = this.docData?.getMetaTable('_grist_Tables');
const tableRecords = tables?.getRecords().filter(r =>
r.tableId && !r.tableId.startsWith('GristHidden_')) ?? [];
const summaryTables = tableRecords.filter(r => r.summarySourceTable);
const summaryTableIds = new Set([...summaryTables.map(t => t.id)]);
const numSummaryFormulaColumns = columnRecords.filter(r =>
r.isFormula && summaryTableIds.has(r.parentId)).length;
const viewSectionFields = this.docData?.getMetaTable('_grist_Views_section_field');
const viewSectionFieldRecords = viewSectionFields?.getRecords() ?? [];
const numFieldsWithConditionalFormatting = viewSectionFieldRecords.filter(r => r.rules).length;
return {
numColumns,
numColumnsWithConditionalFormatting,
numFormulaColumns,
numTriggerFormulaColumns,
numSummaryFormulaColumns,
numFieldsWithConditionalFormatting,
};
}
private _getTableMetrics() {
const tables = this.docData?.getMetaTable('_grist_Tables');
const tableRecords = tables?.getRecords().filter(r =>
r.tableId && !r.tableId.startsWith('GristHidden_')) ?? [];
const numTables = tableRecords.length;
const numOnDemandTables = tableRecords.filter(r => r.onDemand).length;
const viewSections = this.docData?.getMetaTable('_grist_Views_section');
const viewSectionRecords = viewSections?.getRecords() ?? [];
const numTablesWithConditionalFormatting = viewSectionRecords.filter(r => r.rules).length;
const summaryTables = tableRecords.filter(r => r.summarySourceTable);
const numSummaryTables = summaryTables.length;
return {
numTables,
numOnDemandTables,
numTablesWithConditionalFormatting,
numSummaryTables,
};
}
private _getCustomWidgetMetrics() {
const viewSections = this.docData?.getMetaTable('_grist_Views_section');
const viewSectionRecords = viewSections?.getRecords() ?? [];
const customWidgetUrls: string[] = [];
for (const r of viewSectionRecords) {
const {customView} = safeJsonParse(r.options, {});
if (!customView) { continue; }
const {url} = safeJsonParse(customView, {});
if (!url) { continue; }
const isGristUrl = url.startsWith(commonUrls.gristLabsCustomWidgets);
customWidgetUrls.push(isGristUrl ? url : 'externalURL');
}
const numCustomWidgets = customWidgetUrls.length;
return {
numCustomWidgets,
customWidgetUrls,
};
}
private async _fetchQueryFromDB(query: ServerQuery, onDemand: boolean): Promise<TableDataAction> {
// Expand query to compute formulas (or include placeholders for them).
const expandedQuery = expandQuery(query, this.docData!, onDemand);
@ -2279,7 +2459,10 @@ export class ActiveDoc extends EventEmitter {
if (this._docUsage?.attachmentsSizeBytes === undefined) {
promises.push(this._updateAttachmentsSize(options));
}
if (promises.length === 0) { return; }
if (promises.length === 0) {
this._logDocMetrics(docSession, 'docOpen');
return;
}
try {
await Promise.all(promises);
@ -2288,6 +2471,20 @@ export class ActiveDoc extends EventEmitter {
} catch (e) {
this._log.warn(docSession, 'failed to initialize doc usage', e);
}
this._logDocMetrics(docSession, 'docOpen');
}
private _getTelemetryMeta(docSession: OptDocSession|null) {
return {
...(docSession ? {
...getLogMetaFromDocSession(docSession),
altSessionId: getDocSessionAltSessionId(docSession),
} : {}),
docId: this._docName,
siteId: this._doc?.workspace.org.id,
siteType: this._product?.name,
};
}
/**

41
app/server/lib/AppEndpoint.ts
View File

@ -7,10 +7,12 @@ import * as express from 'express';
import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
import {ApiError} from 'app/common/ApiError';
import {getSlugIfNeeded, parseSubdomainStrictly} from 'app/common/gristUrls';
import {getSlugIfNeeded, parseSubdomainStrictly, parseUrlId} from 'app/common/gristUrls';
import {removeTrailingSlash} from 'app/common/gutil';
import {LocalPlugin} from "app/common/plugin";
import {TelemetryTemplateSignupCookieName} from 'app/common/Telemetry';
import {Document as APIDocument} from 'app/common/UserAPI';
import {TEMPLATES_ORG_DOMAIN} from 'app/gen-server/ApiServer';
import {Document} from "app/gen-server/entity/Document";
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
@ -18,6 +20,7 @@ import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap} from 'app/server/lib/expressWrap';
import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
import {getCookieDomain} from 'app/server/lib/gristSessions';
import {getAssignmentId} from 'app/server/lib/idUtils';
import log from 'app/server/lib/log';
import {adaptServerUrl, addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
@ -299,6 +302,42 @@ export function attachAppEndpoint(options: AttachOptions): void {
body = await workerInfo.resp.json();
}
const isPublic = ((doc as unknown) as APIDocument).public ?? false;
const isSnapshot = parseUrlId(urlId).snapshotId;
// TODO: Need a more precise way to identify a template. (This org now also has tutorials.)
const isTemplate = TEMPLATES_ORG_DOMAIN === doc.workspace.org.domain && doc.type !== 'tutorial';
if (isPublic || isTemplate) {
gristServer.getTelemetryManager()?.logEvent('documentOpened', {
docId,
siteId: doc.workspace.org.id,
siteType: doc.workspace.org.billingAccount.product.name,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
access: doc.access,
isPublic,
isSnapshot,
isTemplate,
lastUpdated: doc.updatedAt,
});
}
if (isTemplate) {
// Keep track of the last template a user visited in the last hour.
// If a sign-up occurs within that time period, we'll know which
// template, if any, was viewed most recently.
const value = {
isAnonymous: isAnonymousUser(mreq),
templateId: docId,
};
res.cookie(TelemetryTemplateSignupCookieName, JSON.stringify(value), {
maxAge: 1000 * 60 * 60,
httpOnly: true,
path: '/',
domain: getCookieDomain(req),
sameSite: 'lax',
});
}
await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
googleTagManager: 'anon', config: {
assignmentId: docId,

9
app/server/lib/Authorizer.ts
View File

@ -160,6 +160,8 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
// about this case.
let authDone: boolean = false;
let hasApiKey: boolean = false;
// Support providing an access token via an `auth` query parameter.
// This is useful for letting the browser load assets like image
// attachments.
@ -191,6 +193,7 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
mreq.user = user;
mreq.userId = user.id;
mreq.userIsAuthorized = true;
hasApiKey = true;
}
}
@ -392,6 +395,12 @@ export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPer
altSessionId: mreq.altSessionId,
};
log.rawDebug(`Auth[${meta.method}]: ${meta.host} ${meta.path}`, meta);
if (hasApiKey) {
options.gristServer.getTelemetryManager()?.logEvent('apiUsage', {
...meta,
userAgent: req.headers['user-agent'],
});
}
return next();
}

8
app/server/lib/DocApi.ts
View File

@ -836,6 +836,7 @@ export class DocWorkerApi {
// This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it
// starts with to become muted.
this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => {
const docSession = docSessionFromRequest(req);
const activeDoc = await this._getActiveDoc(req);
const options: DocReplacementOptions = {};
if (req.body.sourceDocId) {
@ -881,12 +882,17 @@ export class DocWorkerApi {
manager
);
});
const {forkId} = parseUrlId(scope.urlId);
activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
tutorialForkId: forkId,
tutorialForkUrlId: scope.urlId,
tutorialTrunkId,
});
}
}
if (req.body.snapshotId) {
options.snapshotId = String(req.body.snapshotId);
}
const docSession = docSessionFromRequest(req);
await activeDoc.replace(docSession, options);
res.json(null);
}));

34
app/server/lib/FlexServer.ts
View File

@ -6,6 +6,7 @@ import {encodeUrl, getSlugIfNeeded, GristLoadConfig, IGristUrlState, isOrgInPath
import {getOrgUrlInfo} from 'app/common/gristUrls';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {tbind} from 'app/common/tbind';
import {TelemetryEventName, TelemetryEventNames} from 'app/common/Telemetry';
import * as version from 'app/common/version';
import {ApiServer, getOrgFromRequest} from 'app/gen-server/ApiServer';
import {Document} from "app/gen-server/entity/Document";
@ -55,6 +56,7 @@ import {getDatabaseUrl, listenPromise} from 'app/server/lib/serverUtils';
import {Sessions} from 'app/server/lib/Sessions';
import * as shutdown from 'app/server/lib/shutdown';
import {TagChecker} from 'app/server/lib/TagChecker';
import {TelemetryManager} from 'app/server/lib/TelemetryManager';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {addUploadRoute} from 'app/server/lib/uploads';
@ -127,6 +129,7 @@ export class FlexServer implements GristServer {
private _sessions: Sessions;
private _sessionStore: SessionStore;
private _storageManager: IDocStorageManager;
private _telemetryManager: TelemetryManager|undefined;
private _docWorkerMap: IDocWorkerMap;
private _widgetRepository: IWidgetRepository;
private _notifier: INotifier;
@ -338,6 +341,10 @@ export class FlexServer implements GristServer {
return this._storageManager;
}
public getTelemetryManager(): TelemetryManager|undefined {
return this._telemetryManager;
}
public getWidgetRepository(): IWidgetRepository {
if (!this._widgetRepository) { throw new Error('no widget repository available'); }
return this._widgetRepository;
@ -663,7 +670,8 @@ export class FlexServer implements GristServer {
*/
public addLogEndpoint() {
if (this._check('log-endpoint', 'json', 'api-mw')) { return; }
this.app.post('/api/log', expressWrap(async (req, resp) => {
this.app.post('/api/log', async (req, resp) => {
const mreq = req as RequestWithLogin;
log.rawWarn('client error', {
event: req.body.event,
@ -676,7 +684,26 @@ export class FlexServer implements GristServer {
altSessionId: mreq.altSessionId,
});
return resp.status(200).send();
}));
});
}
public addTelemetryEndpoint() {
if (this._check('telemetry-endpoint', 'json', 'api-mw', 'homedb')) { return; }
this._telemetryManager = new TelemetryManager(this._dbManager);
this.app.post('/api/telemetry', async (req, resp) => {
const mreq = req as RequestWithLogin;
const name = stringParam(req.body.name, 'name', TelemetryEventNames);
this._telemetryManager?.logEvent(name as TelemetryEventName, {
userId: mreq.userId,
email: mreq.user?.loginEmail,
altSessionId: mreq.altSessionId,
site: mreq.org,
...req.body.metadata,
});
return resp.status(200).send();
});
}
public async close() {
@ -1076,7 +1103,8 @@ export class FlexServer implements GristServer {
// Add document-related endpoints and related support.
public async addDoc() {
this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ? null : 'homedb', 'api-mw', 'map');
this._check('doc', 'start', 'tag', 'json', isSingleUserMode() ?
null : 'homedb', 'api-mw', 'map', 'telemetry-endpoint');
// add handlers for cleanup, if we are in charge of the doc manager.
if (!this._docManager) { this.addCleanup(); }
await this.loadConfig();

3
app/server/lib/GristServer.ts
View File

@ -16,6 +16,7 @@ import { IPermitStore } from 'app/server/lib/Permit';
import { ISendAppPageOptions } from 'app/server/lib/sendAppPage';
import { fromCallback } from 'app/server/lib/serverUtils';
import { Sessions } from 'app/server/lib/Sessions';
import { TelemetryManager } from 'app/server/lib/TelemetryManager';
import * as express from 'express';
import { IncomingMessage } from 'http';
@ -42,6 +43,7 @@ export interface GristServer {
getHosts(): Hosts;
getHomeDBManager(): HomeDBManager;
getStorageManager(): IDocStorageManager;
getTelemetryManager(): TelemetryManager|undefined;
getNotifier(): INotifier;
getDocTemplate(): Promise<DocTemplate>;
getTag(): string;
@ -118,6 +120,7 @@ export function createDummyGristServer(): GristServer {
getHosts() { throw new Error('no hosts'); },
getHomeDBManager() { throw new Error('no db'); },
getStorageManager() { throw new Error('no storage manager'); },
getTelemetryManager() { return undefined; },
getNotifier() { throw new Error('no notifier'); },
getDocTemplate() { throw new Error('no doc template'); },
getTag() { return 'tag'; },

1
app/server/lib/Triggers.ts
View File

@ -647,6 +647,7 @@ export class DocTriggers {
await this._stats.logStatus(id, 'sending');
meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
this._log("Sending batch of webhook events", meta);
this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', meta);
success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
if (this._loopAbort.signal.aborted) {
continue;

8
app/server/lib/initialDocSql.ts
View File

@ -6,7 +6,7 @@ export const GRIST_DOC_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',36,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',37,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Imports" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "origFileName" TEXT DEFAULT '', "parseFormula" TEXT DEFAULT '', "delimiter" TEXT DEFAULT '', "doublequote" BOOLEAN DEFAULT 0, "escapechar" TEXT DEFAULT '', "quotechar" TEXT DEFAULT '', "skipinitialspace" BOOLEAN DEFAULT 0, "encoding" TEXT DEFAULT '', "hasHeaders" BOOLEAN DEFAULT 0);
@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS "_grist_Views_section" (id INTEGER PRIMARY KEY, "tabl
CREATE TABLE IF NOT EXISTS "_grist_Views_section_field" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colRef" INTEGER DEFAULT 0, "width" INTEGER DEFAULT 0, "widgetOptions" TEXT DEFAULT '', "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "filter" TEXT DEFAULT '', "rules" TEXT DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');
@ -43,7 +43,7 @@ export const GRIST_DOC_WITH_TABLE1_SQL = `
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "_grist_DocInfo" (id INTEGER PRIMARY KEY, "docId" TEXT DEFAULT '', "peers" TEXT DEFAULT '', "basketId" TEXT DEFAULT '', "schemaVersion" INTEGER DEFAULT 0, "timezone" TEXT DEFAULT '', "documentSettings" TEXT DEFAULT '');
INSERT INTO _grist_DocInfo VALUES(1,'','','',36,'','');
INSERT INTO _grist_DocInfo VALUES(1,'','','',37,'','');
CREATE TABLE IF NOT EXISTS "_grist_Tables" (id INTEGER PRIMARY KEY, "tableId" TEXT DEFAULT '', "primaryViewId" INTEGER DEFAULT 0, "summarySourceTable" INTEGER DEFAULT 0, "onDemand" BOOLEAN DEFAULT 0, "rawViewSectionRef" INTEGER DEFAULT 0);
INSERT INTO _grist_Tables VALUES(1,'Table1',1,0,0,2);
CREATE TABLE IF NOT EXISTS "_grist_Tables_column" (id INTEGER PRIMARY KEY, "parentId" INTEGER DEFAULT 0, "parentPos" NUMERIC DEFAULT 1e999, "colId" TEXT DEFAULT '', "type" TEXT DEFAULT '', "widgetOptions" TEXT DEFAULT '', "isFormula" BOOLEAN DEFAULT 0, "formula" TEXT DEFAULT '', "label" TEXT DEFAULT '', "description" TEXT DEFAULT '', "untieColIdFromLabel" BOOLEAN DEFAULT 0, "summarySourceCol" INTEGER DEFAULT 0, "displayCol" INTEGER DEFAULT 0, "visibleCol" INTEGER DEFAULT 0, "rules" TEXT DEFAULT NULL, "recalcWhen" INTEGER DEFAULT 0, "recalcDeps" TEXT DEFAULT NULL);
@ -74,7 +74,7 @@ INSERT INTO _grist_Views_section_field VALUES(5,2,5,3,0,'',0,0,'',NULL);
INSERT INTO _grist_Views_section_field VALUES(6,2,6,4,0,'',0,0,'',NULL);
CREATE TABLE IF NOT EXISTS "_grist_Validations" (id INTEGER PRIMARY KEY, "formula" TEXT DEFAULT '', "name" TEXT DEFAULT '', "tableRef" INTEGER DEFAULT 0);
CREATE TABLE IF NOT EXISTS "_grist_REPL_Hist" (id INTEGER PRIMARY KEY, "code" TEXT DEFAULT '', "outputText" TEXT DEFAULT '', "errorText" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Attachments" (id INTEGER PRIMARY KEY, "fileIdent" TEXT DEFAULT '', "fileName" TEXT DEFAULT '', "fileType" TEXT DEFAULT '', "fileSize" INTEGER DEFAULT 0, "fileExt" TEXT DEFAULT '', "imageHeight" INTEGER DEFAULT 0, "imageWidth" INTEGER DEFAULT 0, "timeDeleted" DATETIME DEFAULT NULL, "timeUploaded" DATETIME DEFAULT NULL);
CREATE TABLE IF NOT EXISTS "_grist_Triggers" (id INTEGER PRIMARY KEY, "tableRef" INTEGER DEFAULT 0, "eventTypes" TEXT DEFAULT NULL, "isReadyColRef" INTEGER DEFAULT 0, "actions" TEXT DEFAULT '');
CREATE TABLE IF NOT EXISTS "_grist_ACLRules" (id INTEGER PRIMARY KEY, "resource" INTEGER DEFAULT 0, "permissions" INTEGER DEFAULT 0, "principals" TEXT DEFAULT '', "aclFormula" TEXT DEFAULT '', "aclColumn" INTEGER DEFAULT 0, "aclFormulaParsed" TEXT DEFAULT '', "permissionsText" TEXT DEFAULT '', "rulePos" NUMERIC DEFAULT 1e999, "userAttributes" TEXT DEFAULT '', "memo" TEXT DEFAULT '');
INSERT INTO _grist_ACLRules VALUES(1,1,63,'[1]','',0,'','',1e999,'','');

2
app/server/lib/requestUtils.ts
View File

@ -260,7 +260,7 @@ export function optStringParam(p: any): string|undefined {
return undefined;
}
export function stringParam(p: any, name: string, allowed?: string[]): string {
export function stringParam(p: any, name: string, allowed?: readonly string[]): string {
if (typeof p !== 'string') {
throw new ApiError(`${name} parameter should be a string: ${p}`, 400);
}

2
app/server/mergedServerMain.ts
View File

@ -125,11 +125,13 @@ export async function main(port: number, serverTypes: ServerType[],
server.addBillingPages();
server.addWelcomePaths();
server.addLogEndpoint();
server.addTelemetryEndpoint();
server.addGoogleAuthEndpoint();
}
if (includeDocs) {
server.addJsonSupport();
server.addTelemetryEndpoint();
await server.addDoc();
}

8
sandbox/grist/migrations.py
View File

@ -1196,3 +1196,11 @@ def migration36(tdset):
Add description to column
"""
return tdset.apply_doc_actions([add_column('_grist_Tables_column', 'description', 'Text')])
@migration(schema_version=37)
def migration37(tdset):
"""
Add fileExt column to _grist_Attachments.
"""
return tdset.apply_doc_actions([add_column('_grist_Attachments', 'fileExt', 'Text')])

3
sandbox/grist/schema.py
View File

@ -15,7 +15,7 @@ import six
import actions
SCHEMA_VERSION = 36
SCHEMA_VERSION = 37
def make_column(col_id, col_type, formula='', isFormula=False):
return {
@ -236,6 +236,7 @@ def schema_create_actions():
make_column("fileName", "Text"), # User defined file name
make_column("fileType", "Text"), # A string indicating the MIME type of the data
make_column("fileSize", "Int"), # The size in bytes
make_column("fileExt", "Text"), # The file extension
make_column("imageHeight", "Int"), # height in pixels
make_column("imageWidth", "Int"), # width in pixels
make_column("timeDeleted", "DateTime"),

11
stubs/app/server/lib/TelemetryManager.ts Normal file
View File

@ -0,0 +1,11 @@
import {TelemetryEventName} from 'app/common/Telemetry';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
export class TelemetryManager {
constructor(_dbManager: HomeDBManager) {}
public logEvent(
_name: TelemetryEventName,
_metadata?: Record<string, any>
) {}
}

BIN
test/fixtures/docs/Hello.grist vendored
View File

Binary file not shown.

1
test/server/lib/Authorizer.ts
View File

@ -34,6 +34,7 @@ async function activateServer(home: FlexServer, docManager: DocManager) {
home.addJsonSupport();
await home.addLandingPages();
home.addHomeApi();
home.addTelemetryEndpoint();
await home.addDoc();
home.addApiErrorHandlers();
serverUrl = home.getOwnUrl();