mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Add telemetry for AI Assistant
Summary: Also fixes a few bugs with some telemetry events not being recorded. Test Plan: Manual. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3960
This commit is contained in:
George Gevoian
parent
0040716006
commit
0a34292536
@ -16,6 +16,7 @@ import {
|
||||
import {createParser} from 'app/common/ValueParser';
|
||||
import {Observable} from 'grainjs';
|
||||
import * as ko from 'knockout';
|
||||
import {v4 as uuidv4} from 'uuid';
|
||||
|
||||
// Column behavior type, used primarily in the UI.
|
||||
export type BEHAVIOR = "empty"|"formula"|"data";
|
||||
@ -165,7 +166,7 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
|
||||
const docId = urlState().state.get().doc ?? '';
|
||||
// Changed key name from history to history-v2 when ChatHistory changed in incompatible way.
|
||||
const key = `formula-assistant-history-v2-${docId}-${this.table().tableId()}-${this.colId()}`;
|
||||
return localStorageJsonObs(key, {messages: []} as ChatHistory);
|
||||
return localStorageJsonObs(key, {messages: [], conversationId: uuidv4()} as ChatHistory);
|
||||
}));
|
||||
}
|
||||
|
||||
@ -217,5 +218,6 @@ export interface ChatMessage {
|
||||
*/
|
||||
export interface ChatHistory {
|
||||
messages: ChatMessage[];
|
||||
conversationId?: string;
|
||||
state?: AssistanceState;
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import * as commands from 'app/client/components/commands';
|
||||
import {GristDoc} from 'app/client/components/GristDoc';
|
||||
import {logTelemetryEvent} from 'app/client/lib/telemetry';
|
||||
import {makeT} from 'app/client/lib/localization';
|
||||
import {localStorageBoolObs} from 'app/client/lib/localStorageObs';
|
||||
import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel';
|
||||
@ -10,7 +11,7 @@ import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
|
||||
import {sanitizeHTML} from 'app/client/ui/sanitizeHTML';
|
||||
import {createUserImage} from 'app/client/ui/UserImage';
|
||||
import {FormulaEditor} from 'app/client/widgets/FormulaEditor';
|
||||
import {AssistanceResponse, AssistanceState} from 'app/common/AssistancePrompts';
|
||||
import {AssistanceResponse, AssistanceState, FormulaAssistanceContext} from 'app/common/AssistancePrompts';
|
||||
import {basicButton, bigPrimaryButtonLink, primaryButton} from 'app/client/ui2018/buttons';
|
||||
import {theme, vars} from 'app/client/ui2018/cssVars';
|
||||
import {autoGrow} from 'app/client/ui/forms';
|
||||
@ -19,12 +20,14 @@ import {cssLink} from 'app/client/ui2018/links';
|
||||
import {commonUrls} from 'app/common/gristUrls';
|
||||
import {movable} from 'app/client/lib/popupUtils';
|
||||
import {loadingDots} from 'app/client/ui2018/loaders';
|
||||
import {menu, menuCssClass, menuItem} from "app/client/ui2018/menus";
|
||||
import {menu, menuCssClass, menuItem} from 'app/client/ui2018/menus';
|
||||
import {TelemetryEvent, TelemetryMetadata} from 'app/common/Telemetry';
|
||||
import {Computed, Disposable, dom, DomElementArg, makeTestId,
|
||||
MutableObsArray, obsArray, Observable, styled} from 'grainjs';
|
||||
import debounce from 'lodash/debounce';
|
||||
import noop from 'lodash/noop';
|
||||
import {marked} from 'marked';
|
||||
import {v4 as uuidv4} from 'uuid';
|
||||
|
||||
const t = makeT('FormulaEditor');
|
||||
const testId = makeTestId('test-formula-editor-');
|
||||
@ -67,6 +70,8 @@ export class FormulaAssistant extends Disposable {
|
||||
private _chatPanelBody: HTMLElement;
|
||||
/** Client height of the chat panel body element. */
|
||||
private _chatPanelBodyClientHeight = Observable.create<number>(this, 0);
|
||||
/** Set to true once the panel has been expanded (including by default). */
|
||||
private _hasExpanded = false;
|
||||
/**
|
||||
* Last known height of the chat panel.
|
||||
*
|
||||
@ -108,6 +113,7 @@ export class FormulaAssistant extends Disposable {
|
||||
this._chat = ChatHistory.create(this, {
|
||||
...this._options,
|
||||
apply: this._apply.bind(this),
|
||||
logTelemetryEvent: this._logTelemetryEvent.bind(this),
|
||||
});
|
||||
|
||||
this.autoDispose(commands.createGroup({
|
||||
@ -143,6 +149,15 @@ export class FormulaAssistant extends Disposable {
|
||||
|
||||
this._triggerFinalize = bundleInfo.triggerFinalize;
|
||||
this.onDispose(() => {
|
||||
if (this._hasExpanded) {
|
||||
this._logTelemetryEvent('assistantClose', false, {
|
||||
suggestionApplied: this._chat.conversationSuggestedFormulas.get()
|
||||
.includes(this._options.column.formula.peek()),
|
||||
conversationLength: this._chat.conversationLength.get(),
|
||||
conversationHistoryLength: this._chat.conversationHistoryLength.get(),
|
||||
});
|
||||
}
|
||||
|
||||
// This will be noop if already called.
|
||||
this._triggerFinalize();
|
||||
});
|
||||
@ -178,6 +193,11 @@ export class FormulaAssistant extends Disposable {
|
||||
this._chatPanelBody.style.setProperty('height', '999px');
|
||||
}
|
||||
|
||||
if (this._assistantEnabled.get() && this._assistantExpanded.get()) {
|
||||
this._logTelemetryEvent('assistantOpen', true);
|
||||
this._hasExpanded = true;
|
||||
}
|
||||
|
||||
return this._domElement;
|
||||
}
|
||||
|
||||
@ -198,6 +218,21 @@ export class FormulaAssistant extends Disposable {
|
||||
});
|
||||
}
|
||||
|
||||
private _logTelemetryEvent(event: TelemetryEvent, includeContext = false, metadata: TelemetryMetadata = {}) {
|
||||
logTelemetryEvent(event, {
|
||||
full: {
|
||||
docIdDigest: this._gristDoc.docId,
|
||||
conversationId: this._chat.conversationId.get(),
|
||||
...(!includeContext ? {} : {context: {
|
||||
type: 'formula',
|
||||
tableId: this._options.column.table.peek().tableId.peek(),
|
||||
colId: this._options.column.colId.peek(),
|
||||
} as FormulaAssistanceContext}),
|
||||
...metadata,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private _buildChatPanelHeader() {
|
||||
return cssChatPanelHeader(
|
||||
cssChatPanelHeaderTitle(
|
||||
@ -266,6 +301,12 @@ export class FormulaAssistant extends Disposable {
|
||||
* Save button handler. We just store the action and wait for the bundler to finalize.
|
||||
*/
|
||||
private _saveOrClose() {
|
||||
if (this._hasExpanded) {
|
||||
this._logTelemetryEvent('assistantSave', true, {
|
||||
newFormula: this._options.column.formula.peek(),
|
||||
oldFormula: this._options.editor.getTextValue(),
|
||||
});
|
||||
}
|
||||
this._action = 'save';
|
||||
this._triggerFinalize();
|
||||
}
|
||||
@ -274,6 +315,9 @@ export class FormulaAssistant extends Disposable {
|
||||
* Cancel button handler.
|
||||
*/
|
||||
private _cancel() {
|
||||
if (this._hasExpanded) {
|
||||
this._logTelemetryEvent('assistantCancel', true);
|
||||
}
|
||||
this._action = 'cancel';
|
||||
this._triggerFinalize();
|
||||
}
|
||||
@ -374,6 +418,11 @@ export class FormulaAssistant extends Disposable {
|
||||
}
|
||||
|
||||
private _expandChatPanel() {
|
||||
if (!this._hasExpanded) {
|
||||
this._logTelemetryEvent('assistantOpen', true);
|
||||
this._hasExpanded = true;
|
||||
}
|
||||
|
||||
this._assistantExpanded.set(true);
|
||||
const editor = this._options.editor.getDom();
|
||||
let availableSpace = editor.clientHeight - MIN_FORMULA_EDITOR_HEIGHT_PX
|
||||
@ -565,13 +614,17 @@ export class FormulaAssistant extends Disposable {
|
||||
// Destruct options.
|
||||
const {column, gristDoc} = this._options;
|
||||
// Get the state of the chat from the column.
|
||||
const conversationId = this._chat.conversationId.get();
|
||||
const prevState = column.chatHistory.peek().get().state;
|
||||
// Send the message back to the AI with previous state and a mark that we want to regenerate.
|
||||
// We can't modify the state here as we treat it as a black box, so we only removed last message
|
||||
// from ai from the chat, we grabbed last question and we are sending it back to the AI with a
|
||||
// flag that it should clear last response and regenerate it.
|
||||
const {reply, suggestedActions, state} = await askAI(gristDoc, {
|
||||
column, description, state: prevState,
|
||||
conversationId,
|
||||
column,
|
||||
description,
|
||||
state: prevState,
|
||||
regenerate,
|
||||
});
|
||||
console.debug('suggestedActions', {suggestedActions, reply, state});
|
||||
@ -639,8 +692,12 @@ export class FormulaAssistant extends Disposable {
|
||||
* sending messages to the AI.
|
||||
*/
|
||||
class ChatHistory extends Disposable {
|
||||
public history: MutableObsArray<ChatMessage>;
|
||||
public length: Computed<number>;
|
||||
public conversationId: Observable<string>;
|
||||
public conversation: MutableObsArray<ChatMessage>;
|
||||
public conversationHistory: MutableObsArray<ChatMessage>;
|
||||
public conversationLength: Computed<number>;
|
||||
public conversationHistoryLength: Computed<number>;
|
||||
public conversationSuggestedFormulas: Computed<string[]>;
|
||||
public lastSuggestedFormula: Computed<string|null>;
|
||||
|
||||
private _element: HTMLElement;
|
||||
@ -649,31 +706,56 @@ class ChatHistory extends Disposable {
|
||||
column: ColumnRec,
|
||||
gristDoc: GristDoc,
|
||||
apply: (formula: string) => void,
|
||||
logTelemetryEvent: (event: TelemetryEvent, includeContext?: boolean, metadata?: TelemetryMetadata) => void,
|
||||
}) {
|
||||
super();
|
||||
|
||||
const column = this._options.column;
|
||||
let conversationId = column.chatHistory.peek().get().conversationId;
|
||||
if (!conversationId) {
|
||||
conversationId = uuidv4();
|
||||
const chatHistory = column.chatHistory.peek();
|
||||
chatHistory.set({...chatHistory.get(), conversationId});
|
||||
}
|
||||
this.conversationId = Observable.create(this, conversationId);
|
||||
this.autoDispose(this.conversationId.addListener((newConversationId) => {
|
||||
// If a new conversation id was generated (e.g. on Clear Conversation), save it
|
||||
// to the column's history.
|
||||
const chatHistory = column.chatHistory.peek();
|
||||
chatHistory.set({...chatHistory.get(), conversationId: newConversationId});
|
||||
}));
|
||||
|
||||
// Create observable array of messages that is connected to the column's chatHistory.
|
||||
this.history = this.autoDispose(obsArray(column.chatHistory.peek().get().messages));
|
||||
this.autoDispose(this.history.addListener((cur) => {
|
||||
this.conversationHistory = this.autoDispose(obsArray(column.chatHistory.peek().get().messages));
|
||||
this.autoDispose(this.conversationHistory.addListener((cur) => {
|
||||
const chatHistory = column.chatHistory.peek();
|
||||
chatHistory.set({...chatHistory.get(), messages: [...cur]});
|
||||
}));
|
||||
this.length = Computed.create(this, use => use(this.history).length); // ??
|
||||
this.conversation = this.autoDispose(obsArray());
|
||||
|
||||
this.conversationHistoryLength = Computed.create(this, use => use(this.conversationHistory).length);
|
||||
this.conversationLength = Computed.create(this, use => use(this.conversation).length);
|
||||
|
||||
this.conversationSuggestedFormulas = Computed.create(this, use => {
|
||||
return use(this.conversation)
|
||||
.map(({formula}) => formula)
|
||||
.filter((formula): formula is string => Boolean(formula));
|
||||
});
|
||||
this.lastSuggestedFormula = Computed.create(this, use => {
|
||||
return [...use(this.history)].reverse().find(entry => entry.formula)?.formula ?? null;
|
||||
return [...use(this.conversationHistory)].reverse().find(({formula}) => formula)?.formula ?? null;
|
||||
});
|
||||
}
|
||||
|
||||
public thinking(on = true) {
|
||||
if (!on) {
|
||||
// Find all index of all thinking messages.
|
||||
const messages = [...this.history.get()].filter(m => m.message === '...');
|
||||
const messages = [...this.conversationHistory.get()].filter(m => m.message === '...');
|
||||
// Remove all thinking messages.
|
||||
for (const message of messages) {
|
||||
this.history.splice(this.history.get().indexOf(message), 1);
|
||||
this.conversationHistory.splice(this.conversationHistory.get().indexOf(message), 1);
|
||||
}
|
||||
} else {
|
||||
this.history.push({
|
||||
this.conversationHistory.push({
|
||||
message: '...',
|
||||
sender: 'ai',
|
||||
});
|
||||
@ -688,20 +770,21 @@ class ChatHistory extends Disposable {
|
||||
public addResponse(message: ChatMessage) {
|
||||
// Clear any thinking from messages.
|
||||
this.thinking(false);
|
||||
this.history.push({...message, sender: 'ai'});
|
||||
const entry: ChatMessage = {...message, sender: 'ai'};
|
||||
this.conversationHistory.push(entry);
|
||||
this.conversation.push(entry);
|
||||
this.scrollDown();
|
||||
}
|
||||
|
||||
public addQuestion(message: string) {
|
||||
this.thinking(false);
|
||||
this.history.push({
|
||||
message,
|
||||
sender: 'user',
|
||||
});
|
||||
const entry: ChatMessage = {message, sender: 'user'};
|
||||
this.conversationHistory.push(entry);
|
||||
this.conversation.push(entry);
|
||||
}
|
||||
|
||||
public lastQuestion() {
|
||||
const list = this.history.get();
|
||||
const list = this.conversationHistory.get();
|
||||
if (list.length === 0) {
|
||||
return null;
|
||||
}
|
||||
@ -713,14 +796,16 @@ class ChatHistory extends Disposable {
|
||||
}
|
||||
|
||||
public removeLastResponse() {
|
||||
const lastMessage = this.history.get()[this.history.get().length - 1];
|
||||
const lastMessage = this.conversationHistory.get()[this.conversationHistory.get().length - 1];
|
||||
if (lastMessage?.sender === 'ai') {
|
||||
this.history.pop();
|
||||
this.conversationHistory.pop();
|
||||
}
|
||||
}
|
||||
|
||||
public clear() {
|
||||
this.history.set([]);
|
||||
this._options.logTelemetryEvent('assistantClearConversation', true);
|
||||
this.conversationId.set(uuidv4());
|
||||
this.conversationHistory.set([]);
|
||||
const {column} = this._options;
|
||||
// Get the state of the chat from the column.
|
||||
const prevState = column.chatHistory.peek().get();
|
||||
@ -737,7 +822,7 @@ class ChatHistory extends Disposable {
|
||||
public buildDom() {
|
||||
return this._element = cssHistory(
|
||||
this._buildIntroMessage(),
|
||||
dom.forEach(this.history, entry => {
|
||||
dom.forEach(this.conversationHistory, entry => {
|
||||
if (entry.sender === 'user') {
|
||||
return cssMessage(
|
||||
dom('span',
|
||||
@ -857,13 +942,15 @@ class ChatHistory extends Disposable {
|
||||
async function askAI(grist: GristDoc, options: {
|
||||
column: ColumnRec,
|
||||
description: string,
|
||||
conversationId: string,
|
||||
regenerate?: boolean,
|
||||
state?: AssistanceState
|
||||
}): Promise<AssistanceResponse> {
|
||||
const {column, description, state, regenerate} = options;
|
||||
const {column, description, conversationId, state, regenerate} = options;
|
||||
const tableId = column.table.peek().tableId.peek();
|
||||
const colId = column.colId.peek();
|
||||
const result = await grist.docApi.getAssistance({
|
||||
conversationId,
|
||||
context: {type: 'formula', tableId, colId},
|
||||
text: description,
|
||||
state,
|
||||
|
||||
@ -35,6 +35,7 @@ export type AssistanceContext = FormulaAssistanceContext;
|
||||
* A request for assistance.
|
||||
*/
|
||||
export interface AssistanceRequest {
|
||||
conversationId: string;
|
||||
context: AssistanceContext;
|
||||
state?: AssistanceState;
|
||||
text: string;
|
||||
|
||||
@ -25,6 +25,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
apiUsage: {
|
||||
description: 'Triggered when an HTTP request with an API key is made.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
method: {
|
||||
description: 'The HTTP request method (e.g. GET, POST, PUT).',
|
||||
@ -40,9 +41,254 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantOpen: {
|
||||
description: 'Triggered when the AI Assistant is first opened.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantSend: {
|
||||
description: 'Triggered when a message is sent to the AI Assistant.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
siteId: {
|
||||
description: 'The id of the site.',
|
||||
dataType: 'number',
|
||||
},
|
||||
siteType: {
|
||||
description: 'The type of the site.',
|
||||
dataType: 'string',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
access: {
|
||||
description: 'The document access level of the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
prompt: {
|
||||
description: 'The role ("user" or "system"), content, and index of the message sent to the AI Assistant.',
|
||||
dataType: 'object',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantReceive: {
|
||||
description: 'Triggered when a message is received from the AI Assistant is received.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
siteId: {
|
||||
description: 'The id of the site.',
|
||||
dataType: 'number',
|
||||
},
|
||||
siteType: {
|
||||
description: 'The type of the site.',
|
||||
dataType: 'string',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
access: {
|
||||
description: 'The document access level of the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
message: {
|
||||
description: 'The content and index of the message received from the AI Assistant.',
|
||||
dataType: 'object',
|
||||
},
|
||||
suggestedFormula: {
|
||||
description: 'The formula suggested by the AI Assistant, if present.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantSave: {
|
||||
description: 'Triggered when changes in the expanded formula editor are saved after the AI Assistant ' +
|
||||
'was opened.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
newFormula: {
|
||||
description: 'The formula that was saved.',
|
||||
dataType: 'string',
|
||||
},
|
||||
oldFormula: {
|
||||
description: 'The formula that was overwritten.',
|
||||
dataType: 'string',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantCancel: {
|
||||
description: 'Triggered when changes in the expanded formula editor are discarded after the AI Assistant ' +
|
||||
'was opened.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantClearConversation: {
|
||||
description: 'Triggered when a conversation in the AI Assistant is cleared.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'short',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
context: {
|
||||
description: 'The type of assistant (e.g. "formula"), table id, and column id.',
|
||||
dataType: 'object',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
assistantClose: {
|
||||
description: 'Triggered when a formula is saved or discarded after the AI Assistant was opened.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
dataType: 'string',
|
||||
},
|
||||
conversationId: {
|
||||
description: 'A random identifier for the current conversation with the assistant.',
|
||||
dataType: 'string',
|
||||
},
|
||||
suggestionApplied: {
|
||||
description: 'True if a suggested formula from one of the received messages was applied.',
|
||||
dataType: 'boolean',
|
||||
},
|
||||
conversationLength: {
|
||||
description: 'The number of messages sent and received since opening the AI Assistant.',
|
||||
dataType: 'number',
|
||||
},
|
||||
conversationHistoryLength: {
|
||||
description: "The number of messages in the conversation's history. May be less than conversationLength "
|
||||
+ "if the conversation history was cleared in the same session.",
|
||||
dataType: 'number',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
beaconOpen: {
|
||||
description: 'Triggered when HelpScout Beacon is opened.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
@ -57,6 +303,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
beaconArticleViewed: {
|
||||
description: 'Triggered when an article is opened in HelpScout Beacon.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
articleId: {
|
||||
description: 'The id of the article.',
|
||||
@ -75,6 +322,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
beaconEmailSent: {
|
||||
description: 'Triggered when an email is sent in HelpScout Beacon.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
@ -89,6 +337,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
beaconSearch: {
|
||||
description: 'Triggered when a search is made in HelpScout Beacon.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
searchQuery: {
|
||||
description: 'The search query.',
|
||||
@ -107,6 +356,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
documentForked: {
|
||||
description: 'Triggered when a document is forked.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
@ -161,6 +411,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
documentOpened: {
|
||||
description: 'Triggered when a public document or template is opened.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
@ -211,6 +462,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
documentUsage: {
|
||||
description: 'Triggered on doc open and close, as well as hourly while a document is open.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
docIdDigest: {
|
||||
description: 'A hash of the doc id.',
|
||||
@ -346,6 +598,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
processMonitor: {
|
||||
description: 'Triggered every 5 seconds.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
heapUsedMB: {
|
||||
description: 'Size of JS heap in use, in MiB.',
|
||||
@ -368,6 +621,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
sendingWebhooks: {
|
||||
description: 'Triggered when sending webhooks.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
numEvents: {
|
||||
description: 'The number of events in the batch of webhooks being sent.',
|
||||
@ -406,6 +660,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
signupFirstVisit: {
|
||||
description: 'Triggered when a new user first opens the Grist app',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
siteId: {
|
||||
description: 'The site id of first visit after signup.',
|
||||
@ -429,6 +684,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
description: 'Triggered after a user successfully verifies their account during sign-up. '
|
||||
+ 'Not triggered in grist-core.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
isAnonymousTemplateSignup: {
|
||||
description: 'Whether the user viewed any templates before signing up.',
|
||||
@ -443,6 +699,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
siteMembership: {
|
||||
description: 'Triggered daily.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
siteId: {
|
||||
description: 'The site id.',
|
||||
@ -469,6 +726,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
siteUsage: {
|
||||
description: 'Triggered daily.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
siteId: {
|
||||
description: 'The site id.',
|
||||
@ -508,6 +766,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
tutorialProgressChanged: {
|
||||
description: 'Triggered on changes to tutorial progress.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
tutorialForkIdDigest: {
|
||||
description: 'A hash of the tutorial fork id.',
|
||||
@ -529,11 +788,20 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
description: 'Percentage of tutorial completion.',
|
||||
dataType: 'number',
|
||||
},
|
||||
userId: {
|
||||
description: 'The id of the user that triggered this event.',
|
||||
dataType: 'number',
|
||||
},
|
||||
altSessionId: {
|
||||
description: 'A random, session-based identifier for the user that triggered this event.',
|
||||
dataType: 'string',
|
||||
},
|
||||
},
|
||||
},
|
||||
tutorialRestarted: {
|
||||
description: 'Triggered when a tutorial is restarted.',
|
||||
minimumTelemetryLevel: Level.full,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
tutorialForkIdDigest: {
|
||||
description: 'A hash of the tutorial fork id.',
|
||||
@ -572,6 +840,7 @@ export const TelemetryContracts: TelemetryContracts = {
|
||||
watchedVideoTour: {
|
||||
description: 'Triggered when the video tour is closed.',
|
||||
minimumTelemetryLevel: Level.limited,
|
||||
retentionPeriod: 'indefinitely',
|
||||
metadataContracts: {
|
||||
watchTimeSeconds: {
|
||||
description: 'The number of seconds elapsed in the video player.',
|
||||
@ -595,6 +864,13 @@ type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
|
||||
|
||||
export const TelemetryEvents = StringUnion(
|
||||
'apiUsage',
|
||||
'assistantOpen',
|
||||
'assistantSend',
|
||||
'assistantReceive',
|
||||
'assistantSave',
|
||||
'assistantCancel',
|
||||
'assistantClearConversation',
|
||||
'assistantClose',
|
||||
'beaconOpen',
|
||||
'beaconArticleViewed',
|
||||
'beaconEmailSent',
|
||||
@ -617,12 +893,15 @@ export type TelemetryEvent = typeof TelemetryEvents.type;
|
||||
interface TelemetryEventContract {
|
||||
description: string;
|
||||
minimumTelemetryLevel: Level;
|
||||
retentionPeriod: TelemetryRetentionPeriod;
|
||||
metadataContracts?: Record<string, MetadataContract>;
|
||||
}
|
||||
|
||||
export type TelemetryRetentionPeriod = 'short' | 'indefinitely';
|
||||
|
||||
interface MetadataContract {
|
||||
description: string;
|
||||
dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date';
|
||||
dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date' | 'object';
|
||||
minimumTelemetryLevel?: Level;
|
||||
}
|
||||
|
||||
@ -643,9 +922,6 @@ export type TelemetryMetadata = Record<string, any>;
|
||||
*/
|
||||
export const TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME = 'gr_template_signup_trk';
|
||||
|
||||
// A set of metadata keys that are always allowed when logging.
|
||||
const ALLOWED_METADATA_KEYS = new Set(['eventSource', 'installationId']);
|
||||
|
||||
/**
|
||||
* Returns a function that accepts a telemetry event and metadata, and performs various
|
||||
* checks on it based on a set of contracts and the `telemetryLevel`.
|
||||
@ -670,8 +946,6 @@ export function buildTelemetryEventChecker(telemetryLevel: TelemetryLevel) {
|
||||
}
|
||||
|
||||
for (const [key, value] of Object.entries(metadata ?? {})) {
|
||||
if (ALLOWED_METADATA_KEYS.has(key)) { continue; }
|
||||
|
||||
const metadataContract = eventContract.metadataContracts?.[key];
|
||||
if (!metadataContract) {
|
||||
throw new Error(`Unknown metadata for telemetry event ${event}: ${key}`);
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
import {AssistanceMessage, AssistanceRequest, AssistanceResponse} from 'app/common/AssistancePrompts';
|
||||
import {delay} from 'app/common/delay';
|
||||
import {DocAction} from 'app/common/DocActions';
|
||||
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
|
||||
import {OptDocSession} from 'app/server/lib/DocSession';
|
||||
import log from 'app/server/lib/log';
|
||||
import fetch from 'node-fetch';
|
||||
@ -17,7 +18,7 @@ export const DEPS = { fetch, delayTime: 1000 };
|
||||
* An assistant can help a user do things with their document,
|
||||
* by interfacing with an external LLM endpoint.
|
||||
*/
|
||||
export interface Assistant {
|
||||
interface Assistant {
|
||||
apply(session: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse>;
|
||||
}
|
||||
|
||||
@ -25,7 +26,7 @@ export interface Assistant {
|
||||
* Document-related methods for use in the implementation of assistants.
|
||||
* Somewhat ad-hoc currently.
|
||||
*/
|
||||
export interface AssistanceDoc {
|
||||
interface AssistanceDoc extends ActiveDoc {
|
||||
/**
|
||||
* Generate a particular prompt coded in the data engine for some reason.
|
||||
* It makes python code for some tables, and starts a function body with
|
||||
@ -117,10 +118,11 @@ export class OpenAIAssistant implements Assistant {
|
||||
public async apply(
|
||||
optSession: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
|
||||
const messages = request.state?.messages || [];
|
||||
const newMessages = [];
|
||||
const chatMode = this._chatMode;
|
||||
if (chatMode) {
|
||||
if (messages.length === 0) {
|
||||
messages.push({
|
||||
newMessages.push({
|
||||
role: 'system',
|
||||
content: 'You are a helpful assistant for a user of software called Grist. ' +
|
||||
'Below are one or more Python classes. ' +
|
||||
@ -141,7 +143,7 @@ export class OpenAIAssistant implements Assistant {
|
||||
await makeSchemaPromptV1(optSession, doc, request) +
|
||||
'\n```',
|
||||
});
|
||||
messages.push({
|
||||
newMessages.push({
|
||||
role: 'user', content: request.text,
|
||||
});
|
||||
} else {
|
||||
@ -150,21 +152,49 @@ export class OpenAIAssistant implements Assistant {
|
||||
messages.pop();
|
||||
}
|
||||
}
|
||||
messages.push({
|
||||
newMessages.push({
|
||||
role: 'user', content: request.text,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
messages.length = 0;
|
||||
messages.push({
|
||||
newMessages.push({
|
||||
role: 'user', content: await makeSchemaPromptV1(optSession, doc, request),
|
||||
});
|
||||
}
|
||||
messages.push(...newMessages);
|
||||
|
||||
const newMessagesStartIndex = messages.length - newMessages.length;
|
||||
for (const [index, {role, content}] of newMessages.entries()) {
|
||||
doc.logTelemetryEvent(optSession, 'assistantSend', {
|
||||
full: {
|
||||
conversationId: request.conversationId,
|
||||
context: request.context,
|
||||
prompt: {
|
||||
index: newMessagesStartIndex + index,
|
||||
role,
|
||||
content,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const completion: string = await this._getCompletion(messages);
|
||||
const response = await completionToResponse(doc, request, completion, completion);
|
||||
if (chatMode) {
|
||||
response.state = {messages};
|
||||
}
|
||||
doc.logTelemetryEvent(optSession, 'assistantReceive', {
|
||||
full: {
|
||||
conversationId: request.conversationId,
|
||||
context: request.context,
|
||||
message: {
|
||||
index: messages.length - 1,
|
||||
content: completion,
|
||||
},
|
||||
suggestedFormula: (response.suggestedActions[0]?.[3] as any)?.formula,
|
||||
},
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
@ -307,7 +337,7 @@ export class HuggingFaceAssistant implements Assistant {
|
||||
/**
|
||||
* Test assistant that mimics ChatGPT and just returns the input.
|
||||
*/
|
||||
export class EchoAssistant implements Assistant {
|
||||
class EchoAssistant implements Assistant {
|
||||
public async apply(sess: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
|
||||
if (request.text === "ERROR") {
|
||||
throw new Error(`ERROR`);
|
||||
|
||||
@ -12,6 +12,7 @@ import {
|
||||
TelemetryLevels,
|
||||
TelemetryMetadata,
|
||||
TelemetryMetadataByLevel,
|
||||
TelemetryRetentionPeriod,
|
||||
} from 'app/common/Telemetry';
|
||||
import {TelemetryPrefsWithSources} from 'app/common/InstallAPI';
|
||||
import {Activation} from 'app/gen-server/entity/Activation';
|
||||
@ -44,19 +45,15 @@ const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25;
|
||||
*/
|
||||
export class Telemetry implements ITelemetry {
|
||||
private _activation: Activation | undefined;
|
||||
private readonly _deploymentType = this._gristServer.getDeploymentType();
|
||||
|
||||
private _telemetryPrefs: TelemetryPrefsWithSources | undefined;
|
||||
|
||||
private readonly _deploymentType = this._gristServer.getDeploymentType();
|
||||
private readonly _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
|
||||
private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
|
||||
'https://telemetry.getgrist.com/api/telemetry';
|
||||
private _numPendingForwardEventRequests = 0;
|
||||
|
||||
|
||||
private readonly _logger = new LogMethods('Telemetry ', () => ({}));
|
||||
private readonly _telemetryLogger = new LogMethods('Telemetry ', () => ({
|
||||
eventType: 'telemetry',
|
||||
private readonly _telemetryLogger = new LogMethods<string>('Telemetry ', (eventType) => ({
|
||||
eventType,
|
||||
}));
|
||||
|
||||
private _checkTelemetryEvent: TelemetryEventChecker | undefined;
|
||||
@ -153,21 +150,17 @@ export class Telemetry implements ITelemetry {
|
||||
*/
|
||||
app.post('/api/telemetry', expressWrap(async (req, resp) => {
|
||||
const mreq = req as RequestWithLogin;
|
||||
const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
|
||||
const event = stringParam(req.body.event, 'event', TelemetryEvents.values) as TelemetryEvent;
|
||||
if ('eventSource' in req.body.metadata) {
|
||||
this._telemetryLogger.rawLog('info', null, event, {
|
||||
eventName: event,
|
||||
this._telemetryLogger.rawLog('info', getEventType(event), event, {
|
||||
...(removeNullishKeys(req.body.metadata)),
|
||||
eventName: event,
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
this._assertTelemetryIsReady();
|
||||
await this.logEvent(event as TelemetryEvent, merge(
|
||||
await this.logEvent(event, merge(
|
||||
{
|
||||
limited: {
|
||||
eventSource: `grist-${this._deploymentType}`,
|
||||
...(this._deploymentType !== 'saas' ? {installationId: this._activation!.id} : {}),
|
||||
},
|
||||
full: {
|
||||
userId: mreq.userId,
|
||||
altSessionId: mreq.altSessionId,
|
||||
@ -219,10 +212,11 @@ export class Telemetry implements ITelemetry {
|
||||
event: TelemetryEvent,
|
||||
metadata?: TelemetryMetadata
|
||||
) {
|
||||
this._telemetryLogger.rawLog('info', null, event, {
|
||||
this._telemetryLogger.rawLog('info', getEventType(event), event, {
|
||||
...metadata,
|
||||
eventName: event,
|
||||
eventSource: `grist-${this._deploymentType}`,
|
||||
...metadata,
|
||||
installationId: this._activation!.id,
|
||||
});
|
||||
}
|
||||
|
||||
@ -238,7 +232,15 @@ export class Telemetry implements ITelemetry {
|
||||
|
||||
try {
|
||||
this._numPendingForwardEventRequests += 1;
|
||||
await this._doForwardEvent(JSON.stringify({event, metadata}));
|
||||
await this._doForwardEvent(JSON.stringify({
|
||||
event,
|
||||
metadata: {
|
||||
...metadata,
|
||||
eventName: event,
|
||||
eventSource: `grist-${this._deploymentType}`,
|
||||
installationId: this._activation!.id,
|
||||
}
|
||||
}));
|
||||
} catch (e) {
|
||||
this._logger.error(undefined, `failed to forward telemetry event ${event}`, e);
|
||||
} finally {
|
||||
@ -342,3 +344,15 @@ export function hashDigestKeys(metadata: TelemetryMetadata): TelemetryMetadata {
|
||||
});
|
||||
return filteredMetadata;
|
||||
}
|
||||
|
||||
type TelemetryEventType = 'telemetry' | 'telemetry-short-retention';
|
||||
|
||||
const EventTypeByRetentionPeriod: Record<TelemetryRetentionPeriod, TelemetryEventType> = {
|
||||
indefinitely: 'telemetry',
|
||||
short: 'telemetry-short-retention',
|
||||
};
|
||||
|
||||
function getEventType(event: TelemetryEvent) {
|
||||
const {retentionPeriod} = TelemetryContracts[event];
|
||||
return EventTypeByRetentionPeriod[retentionPeriod];
|
||||
}
|
||||
|
||||
@ -164,6 +164,7 @@ where c.colId = ? and t.tableId = ?
|
||||
formula = colInfo?.formula;
|
||||
|
||||
const result = await sendForCompletion(session, activeDoc, {
|
||||
conversationId: 'conversationId',
|
||||
context: {type: 'formula', tableId, colId},
|
||||
state: history,
|
||||
text: followUp || description,
|
||||
|
||||
@ -111,6 +111,7 @@ describe('Telemetry', function() {
|
||||
eventSource: `grist-${deploymentType}`,
|
||||
docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=',
|
||||
isPublic: false,
|
||||
installationId,
|
||||
}
|
||||
]);
|
||||
}
|
||||
@ -133,6 +134,7 @@ describe('Telemetry', function() {
|
||||
docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=',
|
||||
isPublic: false,
|
||||
userId: 1,
|
||||
installationId,
|
||||
}
|
||||
]);
|
||||
}
|
||||
@ -239,6 +241,7 @@ describe('Telemetry', function() {
|
||||
eventName: 'watchedVideoTour',
|
||||
eventSource: `grist-${deploymentType}`,
|
||||
watchTimeSeconds: 30,
|
||||
installationId,
|
||||
});
|
||||
} else {
|
||||
assert.containsAllKeys(metadata, [
|
||||
@ -298,14 +301,10 @@ describe('Telemetry', function() {
|
||||
assert.equal(event, 'watchedVideoTour');
|
||||
if (telemetryLevel === 'limited') {
|
||||
assert.deepEqual(metadata, {
|
||||
eventSource: `grist-${deploymentType}`,
|
||||
installationId,
|
||||
watchTimeSeconds: 30,
|
||||
});
|
||||
} else {
|
||||
assert.containsAllKeys(metadata, [
|
||||
'eventSource',
|
||||
'installationId',
|
||||
'watchTimeSeconds',
|
||||
'userId',
|
||||
'altSessionId',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user