From 0a342925366f3b45f48aecdce65f9ebe0f66860e Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Thu, 20 Jul 2023 10:25:26 -0400 Subject: [PATCH] (core) Add telemetry for AI Assistant Summary: Also fixes a few bugs with some telemetry events not being recorded. Test Plan: Manual. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3960 --- app/client/models/entities/ColumnRec.ts | 4 +- app/client/widgets/FormulaAssistant.ts | 133 ++++++++-- app/common/AssistancePrompts.ts | 1 + app/common/Telemetry.ts | 286 ++++++++++++++++++++- app/server/lib/Assistance.ts | 44 +++- app/server/lib/Telemetry.ts | 50 ++-- test/formula-dataset/runCompletion_impl.ts | 1 + test/server/lib/Telemetry.ts | 7 +- 8 files changed, 467 insertions(+), 59 deletions(-) diff --git a/app/client/models/entities/ColumnRec.ts b/app/client/models/entities/ColumnRec.ts index e38a945f..90e62b66 100644 --- a/app/client/models/entities/ColumnRec.ts +++ b/app/client/models/entities/ColumnRec.ts @@ -16,6 +16,7 @@ import { import {createParser} from 'app/common/ValueParser'; import {Observable} from 'grainjs'; import * as ko from 'knockout'; +import {v4 as uuidv4} from 'uuid'; // Column behavior type, used primarily in the UI. export type BEHAVIOR = "empty"|"formula"|"data"; @@ -165,7 +166,7 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void { const docId = urlState().state.get().doc ?? ''; // Changed key name from history to history-v2 when ChatHistory changed in incompatible way. const key = `formula-assistant-history-v2-${docId}-${this.table().tableId()}-${this.colId()}`; - return localStorageJsonObs(key, {messages: []} as ChatHistory); + return localStorageJsonObs(key, {messages: [], conversationId: uuidv4()} as ChatHistory); })); } @@ -217,5 +218,6 @@ export interface ChatMessage { */ export interface ChatHistory { messages: ChatMessage[]; + conversationId?: string; state?: AssistanceState; } diff --git a/app/client/widgets/FormulaAssistant.ts b/app/client/widgets/FormulaAssistant.ts index 644f0394..7aa8ee89 100644 --- a/app/client/widgets/FormulaAssistant.ts +++ b/app/client/widgets/FormulaAssistant.ts @@ -1,5 +1,6 @@ import * as commands from 'app/client/components/commands'; import {GristDoc} from 'app/client/components/GristDoc'; +import {logTelemetryEvent} from 'app/client/lib/telemetry'; import {makeT} from 'app/client/lib/localization'; import {localStorageBoolObs} from 'app/client/lib/localStorageObs'; import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel'; @@ -10,7 +11,7 @@ import {buildHighlightedCode} from 'app/client/ui/CodeHighlight'; import {sanitizeHTML} from 'app/client/ui/sanitizeHTML'; import {createUserImage} from 'app/client/ui/UserImage'; import {FormulaEditor} from 'app/client/widgets/FormulaEditor'; -import {AssistanceResponse, AssistanceState} from 'app/common/AssistancePrompts'; +import {AssistanceResponse, AssistanceState, FormulaAssistanceContext} from 'app/common/AssistancePrompts'; import {basicButton, bigPrimaryButtonLink, primaryButton} from 'app/client/ui2018/buttons'; import {theme, vars} from 'app/client/ui2018/cssVars'; import {autoGrow} from 'app/client/ui/forms'; @@ -19,12 +20,14 @@ import {cssLink} from 'app/client/ui2018/links'; import {commonUrls} from 'app/common/gristUrls'; import {movable} from 'app/client/lib/popupUtils'; import {loadingDots} from 'app/client/ui2018/loaders'; -import {menu, menuCssClass, menuItem} from "app/client/ui2018/menus"; +import {menu, menuCssClass, menuItem} from 'app/client/ui2018/menus'; +import {TelemetryEvent, TelemetryMetadata} from 'app/common/Telemetry'; import {Computed, Disposable, dom, DomElementArg, makeTestId, MutableObsArray, obsArray, Observable, styled} from 'grainjs'; import debounce from 'lodash/debounce'; import noop from 'lodash/noop'; import {marked} from 'marked'; +import {v4 as uuidv4} from 'uuid'; const t = makeT('FormulaEditor'); const testId = makeTestId('test-formula-editor-'); @@ -67,6 +70,8 @@ export class FormulaAssistant extends Disposable { private _chatPanelBody: HTMLElement; /** Client height of the chat panel body element. */ private _chatPanelBodyClientHeight = Observable.create(this, 0); + /** Set to true once the panel has been expanded (including by default). */ + private _hasExpanded = false; /** * Last known height of the chat panel. * @@ -108,6 +113,7 @@ export class FormulaAssistant extends Disposable { this._chat = ChatHistory.create(this, { ...this._options, apply: this._apply.bind(this), + logTelemetryEvent: this._logTelemetryEvent.bind(this), }); this.autoDispose(commands.createGroup({ @@ -143,6 +149,15 @@ export class FormulaAssistant extends Disposable { this._triggerFinalize = bundleInfo.triggerFinalize; this.onDispose(() => { + if (this._hasExpanded) { + this._logTelemetryEvent('assistantClose', false, { + suggestionApplied: this._chat.conversationSuggestedFormulas.get() + .includes(this._options.column.formula.peek()), + conversationLength: this._chat.conversationLength.get(), + conversationHistoryLength: this._chat.conversationHistoryLength.get(), + }); + } + // This will be noop if already called. this._triggerFinalize(); }); @@ -178,6 +193,11 @@ export class FormulaAssistant extends Disposable { this._chatPanelBody.style.setProperty('height', '999px'); } + if (this._assistantEnabled.get() && this._assistantExpanded.get()) { + this._logTelemetryEvent('assistantOpen', true); + this._hasExpanded = true; + } + return this._domElement; } @@ -198,6 +218,21 @@ export class FormulaAssistant extends Disposable { }); } + private _logTelemetryEvent(event: TelemetryEvent, includeContext = false, metadata: TelemetryMetadata = {}) { + logTelemetryEvent(event, { + full: { + docIdDigest: this._gristDoc.docId, + conversationId: this._chat.conversationId.get(), + ...(!includeContext ? {} : {context: { + type: 'formula', + tableId: this._options.column.table.peek().tableId.peek(), + colId: this._options.column.colId.peek(), + } as FormulaAssistanceContext}), + ...metadata, + }, + }); + } + private _buildChatPanelHeader() { return cssChatPanelHeader( cssChatPanelHeaderTitle( @@ -266,6 +301,12 @@ export class FormulaAssistant extends Disposable { * Save button handler. We just store the action and wait for the bundler to finalize. */ private _saveOrClose() { + if (this._hasExpanded) { + this._logTelemetryEvent('assistantSave', true, { + newFormula: this._options.column.formula.peek(), + oldFormula: this._options.editor.getTextValue(), + }); + } this._action = 'save'; this._triggerFinalize(); } @@ -274,6 +315,9 @@ export class FormulaAssistant extends Disposable { * Cancel button handler. */ private _cancel() { + if (this._hasExpanded) { + this._logTelemetryEvent('assistantCancel', true); + } this._action = 'cancel'; this._triggerFinalize(); } @@ -374,6 +418,11 @@ export class FormulaAssistant extends Disposable { } private _expandChatPanel() { + if (!this._hasExpanded) { + this._logTelemetryEvent('assistantOpen', true); + this._hasExpanded = true; + } + this._assistantExpanded.set(true); const editor = this._options.editor.getDom(); let availableSpace = editor.clientHeight - MIN_FORMULA_EDITOR_HEIGHT_PX @@ -565,13 +614,17 @@ export class FormulaAssistant extends Disposable { // Destruct options. const {column, gristDoc} = this._options; // Get the state of the chat from the column. + const conversationId = this._chat.conversationId.get(); const prevState = column.chatHistory.peek().get().state; // Send the message back to the AI with previous state and a mark that we want to regenerate. // We can't modify the state here as we treat it as a black box, so we only removed last message // from ai from the chat, we grabbed last question and we are sending it back to the AI with a // flag that it should clear last response and regenerate it. const {reply, suggestedActions, state} = await askAI(gristDoc, { - column, description, state: prevState, + conversationId, + column, + description, + state: prevState, regenerate, }); console.debug('suggestedActions', {suggestedActions, reply, state}); @@ -639,8 +692,12 @@ export class FormulaAssistant extends Disposable { * sending messages to the AI. */ class ChatHistory extends Disposable { - public history: MutableObsArray; - public length: Computed; + public conversationId: Observable; + public conversation: MutableObsArray; + public conversationHistory: MutableObsArray; + public conversationLength: Computed; + public conversationHistoryLength: Computed; + public conversationSuggestedFormulas: Computed; public lastSuggestedFormula: Computed; private _element: HTMLElement; @@ -649,31 +706,56 @@ class ChatHistory extends Disposable { column: ColumnRec, gristDoc: GristDoc, apply: (formula: string) => void, + logTelemetryEvent: (event: TelemetryEvent, includeContext?: boolean, metadata?: TelemetryMetadata) => void, }) { super(); + const column = this._options.column; + let conversationId = column.chatHistory.peek().get().conversationId; + if (!conversationId) { + conversationId = uuidv4(); + const chatHistory = column.chatHistory.peek(); + chatHistory.set({...chatHistory.get(), conversationId}); + } + this.conversationId = Observable.create(this, conversationId); + this.autoDispose(this.conversationId.addListener((newConversationId) => { + // If a new conversation id was generated (e.g. on Clear Conversation), save it + // to the column's history. + const chatHistory = column.chatHistory.peek(); + chatHistory.set({...chatHistory.get(), conversationId: newConversationId}); + })); + // Create observable array of messages that is connected to the column's chatHistory. - this.history = this.autoDispose(obsArray(column.chatHistory.peek().get().messages)); - this.autoDispose(this.history.addListener((cur) => { + this.conversationHistory = this.autoDispose(obsArray(column.chatHistory.peek().get().messages)); + this.autoDispose(this.conversationHistory.addListener((cur) => { const chatHistory = column.chatHistory.peek(); chatHistory.set({...chatHistory.get(), messages: [...cur]}); })); - this.length = Computed.create(this, use => use(this.history).length); // ?? + this.conversation = this.autoDispose(obsArray()); + + this.conversationHistoryLength = Computed.create(this, use => use(this.conversationHistory).length); + this.conversationLength = Computed.create(this, use => use(this.conversation).length); + + this.conversationSuggestedFormulas = Computed.create(this, use => { + return use(this.conversation) + .map(({formula}) => formula) + .filter((formula): formula is string => Boolean(formula)); + }); this.lastSuggestedFormula = Computed.create(this, use => { - return [...use(this.history)].reverse().find(entry => entry.formula)?.formula ?? null; + return [...use(this.conversationHistory)].reverse().find(({formula}) => formula)?.formula ?? null; }); } public thinking(on = true) { if (!on) { // Find all index of all thinking messages. - const messages = [...this.history.get()].filter(m => m.message === '...'); + const messages = [...this.conversationHistory.get()].filter(m => m.message === '...'); // Remove all thinking messages. for (const message of messages) { - this.history.splice(this.history.get().indexOf(message), 1); + this.conversationHistory.splice(this.conversationHistory.get().indexOf(message), 1); } } else { - this.history.push({ + this.conversationHistory.push({ message: '...', sender: 'ai', }); @@ -688,20 +770,21 @@ class ChatHistory extends Disposable { public addResponse(message: ChatMessage) { // Clear any thinking from messages. this.thinking(false); - this.history.push({...message, sender: 'ai'}); + const entry: ChatMessage = {...message, sender: 'ai'}; + this.conversationHistory.push(entry); + this.conversation.push(entry); this.scrollDown(); } public addQuestion(message: string) { this.thinking(false); - this.history.push({ - message, - sender: 'user', - }); + const entry: ChatMessage = {message, sender: 'user'}; + this.conversationHistory.push(entry); + this.conversation.push(entry); } public lastQuestion() { - const list = this.history.get(); + const list = this.conversationHistory.get(); if (list.length === 0) { return null; } @@ -713,14 +796,16 @@ class ChatHistory extends Disposable { } public removeLastResponse() { - const lastMessage = this.history.get()[this.history.get().length - 1]; + const lastMessage = this.conversationHistory.get()[this.conversationHistory.get().length - 1]; if (lastMessage?.sender === 'ai') { - this.history.pop(); + this.conversationHistory.pop(); } } public clear() { - this.history.set([]); + this._options.logTelemetryEvent('assistantClearConversation', true); + this.conversationId.set(uuidv4()); + this.conversationHistory.set([]); const {column} = this._options; // Get the state of the chat from the column. const prevState = column.chatHistory.peek().get(); @@ -737,7 +822,7 @@ class ChatHistory extends Disposable { public buildDom() { return this._element = cssHistory( this._buildIntroMessage(), - dom.forEach(this.history, entry => { + dom.forEach(this.conversationHistory, entry => { if (entry.sender === 'user') { return cssMessage( dom('span', @@ -857,13 +942,15 @@ class ChatHistory extends Disposable { async function askAI(grist: GristDoc, options: { column: ColumnRec, description: string, + conversationId: string, regenerate?: boolean, state?: AssistanceState }): Promise { - const {column, description, state, regenerate} = options; + const {column, description, conversationId, state, regenerate} = options; const tableId = column.table.peek().tableId.peek(); const colId = column.colId.peek(); const result = await grist.docApi.getAssistance({ + conversationId, context: {type: 'formula', tableId, colId}, text: description, state, diff --git a/app/common/AssistancePrompts.ts b/app/common/AssistancePrompts.ts index a7d648ce..fce742f8 100644 --- a/app/common/AssistancePrompts.ts +++ b/app/common/AssistancePrompts.ts @@ -35,6 +35,7 @@ export type AssistanceContext = FormulaAssistanceContext; * A request for assistance. */ export interface AssistanceRequest { + conversationId: string; context: AssistanceContext; state?: AssistanceState; text: string; diff --git a/app/common/Telemetry.ts b/app/common/Telemetry.ts index a6a55a8c..33b8f54a 100644 --- a/app/common/Telemetry.ts +++ b/app/common/Telemetry.ts @@ -25,6 +25,7 @@ export const TelemetryContracts: TelemetryContracts = { apiUsage: { description: 'Triggered when an HTTP request with an API key is made.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { method: { description: 'The HTTP request method (e.g. GET, POST, PUT).', @@ -40,9 +41,254 @@ export const TelemetryContracts: TelemetryContracts = { }, }, }, + assistantOpen: { + description: 'Triggered when the AI Assistant is first opened.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + }, + }, + assistantSend: { + description: 'Triggered when a message is sent to the AI Assistant.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + siteId: { + description: 'The id of the site.', + dataType: 'number', + }, + siteType: { + description: 'The type of the site.', + dataType: 'string', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + access: { + description: 'The document access level of the user that triggered this event.', + dataType: 'string', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + prompt: { + description: 'The role ("user" or "system"), content, and index of the message sent to the AI Assistant.', + dataType: 'object', + }, + }, + }, + assistantReceive: { + description: 'Triggered when a message is received from the AI Assistant is received.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + siteId: { + description: 'The id of the site.', + dataType: 'number', + }, + siteType: { + description: 'The type of the site.', + dataType: 'string', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + access: { + description: 'The document access level of the user that triggered this event.', + dataType: 'string', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + message: { + description: 'The content and index of the message received from the AI Assistant.', + dataType: 'object', + }, + suggestedFormula: { + description: 'The formula suggested by the AI Assistant, if present.', + dataType: 'string', + }, + }, + }, + assistantSave: { + description: 'Triggered when changes in the expanded formula editor are saved after the AI Assistant ' + + 'was opened.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + newFormula: { + description: 'The formula that was saved.', + dataType: 'string', + }, + oldFormula: { + description: 'The formula that was overwritten.', + dataType: 'string', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + }, + }, + assistantCancel: { + description: 'Triggered when changes in the expanded formula editor are discarded after the AI Assistant ' + + 'was opened.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + }, + }, + assistantClearConversation: { + description: 'Triggered when a conversation in the AI Assistant is cleared.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'short', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + context: { + description: 'The type of assistant (e.g. "formula"), table id, and column id.', + dataType: 'object', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + }, + }, + assistantClose: { + description: 'Triggered when a formula is saved or discarded after the AI Assistant was opened.', + minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', + metadataContracts: { + docIdDigest: { + description: 'A hash of the doc id.', + dataType: 'string', + }, + conversationId: { + description: 'A random identifier for the current conversation with the assistant.', + dataType: 'string', + }, + suggestionApplied: { + description: 'True if a suggested formula from one of the received messages was applied.', + dataType: 'boolean', + }, + conversationLength: { + description: 'The number of messages sent and received since opening the AI Assistant.', + dataType: 'number', + }, + conversationHistoryLength: { + description: "The number of messages in the conversation's history. May be less than conversationLength " + + "if the conversation history was cleared in the same session.", + dataType: 'number', + }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, + }, + }, beaconOpen: { description: 'Triggered when HelpScout Beacon is opened.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { userId: { description: 'The id of the user that triggered this event.', @@ -57,6 +303,7 @@ export const TelemetryContracts: TelemetryContracts = { beaconArticleViewed: { description: 'Triggered when an article is opened in HelpScout Beacon.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { articleId: { description: 'The id of the article.', @@ -75,6 +322,7 @@ export const TelemetryContracts: TelemetryContracts = { beaconEmailSent: { description: 'Triggered when an email is sent in HelpScout Beacon.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { userId: { description: 'The id of the user that triggered this event.', @@ -89,6 +337,7 @@ export const TelemetryContracts: TelemetryContracts = { beaconSearch: { description: 'Triggered when a search is made in HelpScout Beacon.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { searchQuery: { description: 'The search query.', @@ -107,6 +356,7 @@ export const TelemetryContracts: TelemetryContracts = { documentForked: { description: 'Triggered when a document is forked.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { docIdDigest: { description: 'A hash of the doc id.', @@ -161,6 +411,7 @@ export const TelemetryContracts: TelemetryContracts = { documentOpened: { description: 'Triggered when a public document or template is opened.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { docIdDigest: { description: 'A hash of the doc id.', @@ -211,6 +462,7 @@ export const TelemetryContracts: TelemetryContracts = { documentUsage: { description: 'Triggered on doc open and close, as well as hourly while a document is open.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { docIdDigest: { description: 'A hash of the doc id.', @@ -346,6 +598,7 @@ export const TelemetryContracts: TelemetryContracts = { processMonitor: { description: 'Triggered every 5 seconds.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { heapUsedMB: { description: 'Size of JS heap in use, in MiB.', @@ -368,6 +621,7 @@ export const TelemetryContracts: TelemetryContracts = { sendingWebhooks: { description: 'Triggered when sending webhooks.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { numEvents: { description: 'The number of events in the batch of webhooks being sent.', @@ -406,6 +660,7 @@ export const TelemetryContracts: TelemetryContracts = { signupFirstVisit: { description: 'Triggered when a new user first opens the Grist app', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { siteId: { description: 'The site id of first visit after signup.', @@ -429,6 +684,7 @@ export const TelemetryContracts: TelemetryContracts = { description: 'Triggered after a user successfully verifies their account during sign-up. ' + 'Not triggered in grist-core.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { isAnonymousTemplateSignup: { description: 'Whether the user viewed any templates before signing up.', @@ -443,6 +699,7 @@ export const TelemetryContracts: TelemetryContracts = { siteMembership: { description: 'Triggered daily.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { siteId: { description: 'The site id.', @@ -469,6 +726,7 @@ export const TelemetryContracts: TelemetryContracts = { siteUsage: { description: 'Triggered daily.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { siteId: { description: 'The site id.', @@ -508,6 +766,7 @@ export const TelemetryContracts: TelemetryContracts = { tutorialProgressChanged: { description: 'Triggered on changes to tutorial progress.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { tutorialForkIdDigest: { description: 'A hash of the tutorial fork id.', @@ -529,11 +788,20 @@ export const TelemetryContracts: TelemetryContracts = { description: 'Percentage of tutorial completion.', dataType: 'number', }, + userId: { + description: 'The id of the user that triggered this event.', + dataType: 'number', + }, + altSessionId: { + description: 'A random, session-based identifier for the user that triggered this event.', + dataType: 'string', + }, }, }, tutorialRestarted: { description: 'Triggered when a tutorial is restarted.', minimumTelemetryLevel: Level.full, + retentionPeriod: 'indefinitely', metadataContracts: { tutorialForkIdDigest: { description: 'A hash of the tutorial fork id.', @@ -572,6 +840,7 @@ export const TelemetryContracts: TelemetryContracts = { watchedVideoTour: { description: 'Triggered when the video tour is closed.', minimumTelemetryLevel: Level.limited, + retentionPeriod: 'indefinitely', metadataContracts: { watchTimeSeconds: { description: 'The number of seconds elapsed in the video player.', @@ -595,6 +864,13 @@ type TelemetryContracts = Record; export const TelemetryEvents = StringUnion( 'apiUsage', + 'assistantOpen', + 'assistantSend', + 'assistantReceive', + 'assistantSave', + 'assistantCancel', + 'assistantClearConversation', + 'assistantClose', 'beaconOpen', 'beaconArticleViewed', 'beaconEmailSent', @@ -617,12 +893,15 @@ export type TelemetryEvent = typeof TelemetryEvents.type; interface TelemetryEventContract { description: string; minimumTelemetryLevel: Level; + retentionPeriod: TelemetryRetentionPeriod; metadataContracts?: Record; } +export type TelemetryRetentionPeriod = 'short' | 'indefinitely'; + interface MetadataContract { description: string; - dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date'; + dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date' | 'object'; minimumTelemetryLevel?: Level; } @@ -643,9 +922,6 @@ export type TelemetryMetadata = Record; */ export const TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME = 'gr_template_signup_trk'; -// A set of metadata keys that are always allowed when logging. -const ALLOWED_METADATA_KEYS = new Set(['eventSource', 'installationId']); - /** * Returns a function that accepts a telemetry event and metadata, and performs various * checks on it based on a set of contracts and the `telemetryLevel`. @@ -670,8 +946,6 @@ export function buildTelemetryEventChecker(telemetryLevel: TelemetryLevel) { } for (const [key, value] of Object.entries(metadata ?? {})) { - if (ALLOWED_METADATA_KEYS.has(key)) { continue; } - const metadataContract = eventContract.metadataContracts?.[key]; if (!metadataContract) { throw new Error(`Unknown metadata for telemetry event ${event}: ${key}`); diff --git a/app/server/lib/Assistance.ts b/app/server/lib/Assistance.ts index b02d7b00..b8e72469 100644 --- a/app/server/lib/Assistance.ts +++ b/app/server/lib/Assistance.ts @@ -5,6 +5,7 @@ import {AssistanceMessage, AssistanceRequest, AssistanceResponse} from 'app/common/AssistancePrompts'; import {delay} from 'app/common/delay'; import {DocAction} from 'app/common/DocActions'; +import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {OptDocSession} from 'app/server/lib/DocSession'; import log from 'app/server/lib/log'; import fetch from 'node-fetch'; @@ -17,7 +18,7 @@ export const DEPS = { fetch, delayTime: 1000 }; * An assistant can help a user do things with their document, * by interfacing with an external LLM endpoint. */ -export interface Assistant { +interface Assistant { apply(session: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise; } @@ -25,7 +26,7 @@ export interface Assistant { * Document-related methods for use in the implementation of assistants. * Somewhat ad-hoc currently. */ -export interface AssistanceDoc { +interface AssistanceDoc extends ActiveDoc { /** * Generate a particular prompt coded in the data engine for some reason. * It makes python code for some tables, and starts a function body with @@ -117,10 +118,11 @@ export class OpenAIAssistant implements Assistant { public async apply( optSession: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise { const messages = request.state?.messages || []; + const newMessages = []; const chatMode = this._chatMode; if (chatMode) { if (messages.length === 0) { - messages.push({ + newMessages.push({ role: 'system', content: 'You are a helpful assistant for a user of software called Grist. ' + 'Below are one or more Python classes. ' + @@ -141,7 +143,7 @@ export class OpenAIAssistant implements Assistant { await makeSchemaPromptV1(optSession, doc, request) + '\n```', }); - messages.push({ + newMessages.push({ role: 'user', content: request.text, }); } else { @@ -150,21 +152,49 @@ export class OpenAIAssistant implements Assistant { messages.pop(); } } - messages.push({ + newMessages.push({ role: 'user', content: request.text, }); } } else { messages.length = 0; - messages.push({ + newMessages.push({ role: 'user', content: await makeSchemaPromptV1(optSession, doc, request), }); } + messages.push(...newMessages); + + const newMessagesStartIndex = messages.length - newMessages.length; + for (const [index, {role, content}] of newMessages.entries()) { + doc.logTelemetryEvent(optSession, 'assistantSend', { + full: { + conversationId: request.conversationId, + context: request.context, + prompt: { + index: newMessagesStartIndex + index, + role, + content, + }, + }, + }); + } + const completion: string = await this._getCompletion(messages); const response = await completionToResponse(doc, request, completion, completion); if (chatMode) { response.state = {messages}; } + doc.logTelemetryEvent(optSession, 'assistantReceive', { + full: { + conversationId: request.conversationId, + context: request.context, + message: { + index: messages.length - 1, + content: completion, + }, + suggestedFormula: (response.suggestedActions[0]?.[3] as any)?.formula, + }, + }); return response; } @@ -307,7 +337,7 @@ export class HuggingFaceAssistant implements Assistant { /** * Test assistant that mimics ChatGPT and just returns the input. */ -export class EchoAssistant implements Assistant { +class EchoAssistant implements Assistant { public async apply(sess: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise { if (request.text === "ERROR") { throw new Error(`ERROR`); diff --git a/app/server/lib/Telemetry.ts b/app/server/lib/Telemetry.ts index 44f7b571..d2948926 100644 --- a/app/server/lib/Telemetry.ts +++ b/app/server/lib/Telemetry.ts @@ -12,6 +12,7 @@ import { TelemetryLevels, TelemetryMetadata, TelemetryMetadataByLevel, + TelemetryRetentionPeriod, } from 'app/common/Telemetry'; import {TelemetryPrefsWithSources} from 'app/common/InstallAPI'; import {Activation} from 'app/gen-server/entity/Activation'; @@ -44,19 +45,15 @@ const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25; */ export class Telemetry implements ITelemetry { private _activation: Activation | undefined; - private readonly _deploymentType = this._gristServer.getDeploymentType(); - private _telemetryPrefs: TelemetryPrefsWithSources | undefined; - + private readonly _deploymentType = this._gristServer.getDeploymentType(); private readonly _shouldForwardTelemetryEvents = this._deploymentType !== 'saas'; private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL || 'https://telemetry.getgrist.com/api/telemetry'; private _numPendingForwardEventRequests = 0; - - private readonly _logger = new LogMethods('Telemetry ', () => ({})); - private readonly _telemetryLogger = new LogMethods('Telemetry ', () => ({ - eventType: 'telemetry', + private readonly _telemetryLogger = new LogMethods('Telemetry ', (eventType) => ({ + eventType, })); private _checkTelemetryEvent: TelemetryEventChecker | undefined; @@ -153,21 +150,17 @@ export class Telemetry implements ITelemetry { */ app.post('/api/telemetry', expressWrap(async (req, resp) => { const mreq = req as RequestWithLogin; - const event = stringParam(req.body.event, 'event', TelemetryEvents.values); + const event = stringParam(req.body.event, 'event', TelemetryEvents.values) as TelemetryEvent; if ('eventSource' in req.body.metadata) { - this._telemetryLogger.rawLog('info', null, event, { - eventName: event, + this._telemetryLogger.rawLog('info', getEventType(event), event, { ...(removeNullishKeys(req.body.metadata)), + eventName: event, }); } else { try { this._assertTelemetryIsReady(); - await this.logEvent(event as TelemetryEvent, merge( + await this.logEvent(event, merge( { - limited: { - eventSource: `grist-${this._deploymentType}`, - ...(this._deploymentType !== 'saas' ? {installationId: this._activation!.id} : {}), - }, full: { userId: mreq.userId, altSessionId: mreq.altSessionId, @@ -219,10 +212,11 @@ export class Telemetry implements ITelemetry { event: TelemetryEvent, metadata?: TelemetryMetadata ) { - this._telemetryLogger.rawLog('info', null, event, { + this._telemetryLogger.rawLog('info', getEventType(event), event, { + ...metadata, eventName: event, eventSource: `grist-${this._deploymentType}`, - ...metadata, + installationId: this._activation!.id, }); } @@ -238,7 +232,15 @@ export class Telemetry implements ITelemetry { try { this._numPendingForwardEventRequests += 1; - await this._doForwardEvent(JSON.stringify({event, metadata})); + await this._doForwardEvent(JSON.stringify({ + event, + metadata: { + ...metadata, + eventName: event, + eventSource: `grist-${this._deploymentType}`, + installationId: this._activation!.id, + } + })); } catch (e) { this._logger.error(undefined, `failed to forward telemetry event ${event}`, e); } finally { @@ -342,3 +344,15 @@ export function hashDigestKeys(metadata: TelemetryMetadata): TelemetryMetadata { }); return filteredMetadata; } + +type TelemetryEventType = 'telemetry' | 'telemetry-short-retention'; + +const EventTypeByRetentionPeriod: Record = { + indefinitely: 'telemetry', + short: 'telemetry-short-retention', +}; + +function getEventType(event: TelemetryEvent) { + const {retentionPeriod} = TelemetryContracts[event]; + return EventTypeByRetentionPeriod[retentionPeriod]; +} diff --git a/test/formula-dataset/runCompletion_impl.ts b/test/formula-dataset/runCompletion_impl.ts index 7ec13917..930ae6e5 100644 --- a/test/formula-dataset/runCompletion_impl.ts +++ b/test/formula-dataset/runCompletion_impl.ts @@ -164,6 +164,7 @@ where c.colId = ? and t.tableId = ? formula = colInfo?.formula; const result = await sendForCompletion(session, activeDoc, { + conversationId: 'conversationId', context: {type: 'formula', tableId, colId}, state: history, text: followUp || description, diff --git a/test/server/lib/Telemetry.ts b/test/server/lib/Telemetry.ts index 0c3d82e6..b9644697 100644 --- a/test/server/lib/Telemetry.ts +++ b/test/server/lib/Telemetry.ts @@ -111,6 +111,7 @@ describe('Telemetry', function() { eventSource: `grist-${deploymentType}`, docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=', isPublic: false, + installationId, } ]); } @@ -133,6 +134,7 @@ describe('Telemetry', function() { docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=', isPublic: false, userId: 1, + installationId, } ]); } @@ -239,6 +241,7 @@ describe('Telemetry', function() { eventName: 'watchedVideoTour', eventSource: `grist-${deploymentType}`, watchTimeSeconds: 30, + installationId, }); } else { assert.containsAllKeys(metadata, [ @@ -298,14 +301,10 @@ describe('Telemetry', function() { assert.equal(event, 'watchedVideoTour'); if (telemetryLevel === 'limited') { assert.deepEqual(metadata, { - eventSource: `grist-${deploymentType}`, - installationId, watchTimeSeconds: 30, }); } else { assert.containsAllKeys(metadata, [ - 'eventSource', - 'installationId', 'watchTimeSeconds', 'userId', 'altSessionId',