(core) Add telemetry for AI Assistant

Summary: Also fixes a few bugs with some telemetry events not being recorded.

Test Plan: Manual.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3960

app/client/models/entities/ColumnRec.ts

@@ -16,6 +16,7 @@ import {
 import {createParser} from 'app/common/ValueParser';
 import {Observable} from 'grainjs';
 import * as ko from 'knockout';
+import {v4 as uuidv4} from 'uuid';
 
 // Column behavior type, used primarily in the UI.
 export type BEHAVIOR = "empty"|"formula"|"data";
@@ -165,7 +166,7 @@ export function createColumnRec(this: ColumnRec, docModel: DocModel): void {
     const docId = urlState().state.get().doc ?? '';
     // Changed key name from history to history-v2 when ChatHistory changed in incompatible way.
     const key = `formula-assistant-history-v2-${docId}-${this.table().tableId()}-${this.colId()}`;
-    return localStorageJsonObs(key, {messages: []} as ChatHistory);
+    return localStorageJsonObs(key, {messages: [], conversationId: uuidv4()} as ChatHistory);
   }));
 }
 
@@ -217,5 +218,6 @@ export interface ChatMessage {
  */
 export interface ChatHistory {
   messages: ChatMessage[];
+  conversationId?: string;
   state?: AssistanceState;
 }

app/client/widgets/FormulaAssistant.ts

@@ -1,5 +1,6 @@
 import * as commands from 'app/client/components/commands';
 import {GristDoc} from 'app/client/components/GristDoc';
+import {logTelemetryEvent} from 'app/client/lib/telemetry';
 import {makeT} from 'app/client/lib/localization';
 import {localStorageBoolObs} from 'app/client/lib/localStorageObs';
 import {ColumnRec, ViewFieldRec} from 'app/client/models/DocModel';
@@ -10,7 +11,7 @@ import {buildHighlightedCode} from 'app/client/ui/CodeHighlight';
 import {sanitizeHTML} from 'app/client/ui/sanitizeHTML';
 import {createUserImage} from 'app/client/ui/UserImage';
 import {FormulaEditor} from 'app/client/widgets/FormulaEditor';
-import {AssistanceResponse, AssistanceState} from 'app/common/AssistancePrompts';
+import {AssistanceResponse, AssistanceState, FormulaAssistanceContext} from 'app/common/AssistancePrompts';
 import {basicButton, bigPrimaryButtonLink, primaryButton} from 'app/client/ui2018/buttons';
 import {theme, vars} from 'app/client/ui2018/cssVars';
 import {autoGrow} from 'app/client/ui/forms';
@@ -19,12 +20,14 @@ import {cssLink} from 'app/client/ui2018/links';
 import {commonUrls} from 'app/common/gristUrls';
 import {movable} from 'app/client/lib/popupUtils';
 import {loadingDots} from 'app/client/ui2018/loaders';
-import {menu, menuCssClass, menuItem} from "app/client/ui2018/menus";
+import {menu, menuCssClass, menuItem} from 'app/client/ui2018/menus';
+import {TelemetryEvent, TelemetryMetadata} from 'app/common/Telemetry';
 import {Computed, Disposable, dom, DomElementArg, makeTestId,
   MutableObsArray, obsArray, Observable, styled} from 'grainjs';
 import debounce from 'lodash/debounce';
 import noop from 'lodash/noop';
 import {marked} from 'marked';
+import {v4 as uuidv4} from 'uuid';
 
 const t = makeT('FormulaEditor');
 const testId = makeTestId('test-formula-editor-');
@@ -67,6 +70,8 @@ export class FormulaAssistant extends Disposable {
   private _chatPanelBody: HTMLElement;
   /** Client height of the chat panel body element. */
   private _chatPanelBodyClientHeight = Observable.create<number>(this, 0);
+  /** Set to true once the panel has been expanded (including by default). */
+  private _hasExpanded = false;
   /**
    * Last known height of the chat panel.
    *
@@ -108,6 +113,7 @@ export class FormulaAssistant extends Disposable {
     this._chat = ChatHistory.create(this, {
       ...this._options,
       apply: this._apply.bind(this),
+      logTelemetryEvent: this._logTelemetryEvent.bind(this),
     });
 
     this.autoDispose(commands.createGroup({
@@ -143,6 +149,15 @@ export class FormulaAssistant extends Disposable {
 
     this._triggerFinalize = bundleInfo.triggerFinalize;
     this.onDispose(() => {
+      if (this._hasExpanded) {
+        this._logTelemetryEvent('assistantClose', false, {
+          suggestionApplied: this._chat.conversationSuggestedFormulas.get()
+            .includes(this._options.column.formula.peek()),
+          conversationLength: this._chat.conversationLength.get(),
+          conversationHistoryLength: this._chat.conversationHistoryLength.get(),
+        });
+      }
+
       // This will be noop if already called.
       this._triggerFinalize();
     });
@@ -178,6 +193,11 @@ export class FormulaAssistant extends Disposable {
       this._chatPanelBody.style.setProperty('height', '999px');
     }
 
+    if (this._assistantEnabled.get() && this._assistantExpanded.get()) {
+      this._logTelemetryEvent('assistantOpen', true);
+      this._hasExpanded = true;
+    }
+
     return this._domElement;
   }
 
@@ -198,6 +218,21 @@ export class FormulaAssistant extends Disposable {
     });
   }
 
+  private _logTelemetryEvent(event: TelemetryEvent, includeContext = false, metadata: TelemetryMetadata = {}) {
+    logTelemetryEvent(event, {
+      full: {
+        docIdDigest: this._gristDoc.docId,
+        conversationId: this._chat.conversationId.get(),
+        ...(!includeContext ? {} : {context: {
+          type: 'formula',
+          tableId: this._options.column.table.peek().tableId.peek(),
+          colId: this._options.column.colId.peek(),
+        } as FormulaAssistanceContext}),
+        ...metadata,
+      },
+    });
+  }
+
   private _buildChatPanelHeader() {
     return cssChatPanelHeader(
       cssChatPanelHeaderTitle(
@@ -266,6 +301,12 @@ export class FormulaAssistant extends Disposable {
    * Save button handler. We just store the action and wait for the bundler to finalize.
    */
   private _saveOrClose() {
+    if (this._hasExpanded) {
+      this._logTelemetryEvent('assistantSave', true, {
+        newFormula: this._options.column.formula.peek(),
+        oldFormula: this._options.editor.getTextValue(),
+      });
+    }
     this._action = 'save';
     this._triggerFinalize();
   }
@@ -274,6 +315,9 @@ export class FormulaAssistant extends Disposable {
    * Cancel button handler.
    */
   private _cancel() {
+    if (this._hasExpanded) {
+      this._logTelemetryEvent('assistantCancel', true);
+    }
     this._action = 'cancel';
     this._triggerFinalize();
   }
@@ -374,6 +418,11 @@ export class FormulaAssistant extends Disposable {
   }
 
   private _expandChatPanel() {
+    if (!this._hasExpanded) {
+      this._logTelemetryEvent('assistantOpen', true);
+      this._hasExpanded = true;
+    }
+
     this._assistantExpanded.set(true);
     const editor = this._options.editor.getDom();
     let availableSpace = editor.clientHeight - MIN_FORMULA_EDITOR_HEIGHT_PX
@@ -565,13 +614,17 @@ export class FormulaAssistant extends Disposable {
     // Destruct options.
     const {column, gristDoc} = this._options;
     // Get the state of the chat from the column.
+    const conversationId = this._chat.conversationId.get();
     const prevState = column.chatHistory.peek().get().state;
     // Send the message back to the AI with previous state and a mark that we want to regenerate.
     // We can't modify the state here as we treat it as a black box, so we only removed last message
     // from ai from the chat, we grabbed last question and we are sending it back to the AI with a
     // flag that it should clear last response and regenerate it.
     const {reply, suggestedActions, state} = await askAI(gristDoc, {
-      column, description, state: prevState,
+      conversationId,
+      column,
+      description,
+      state: prevState,
       regenerate,
     });
     console.debug('suggestedActions', {suggestedActions, reply, state});
@@ -639,8 +692,12 @@ export class FormulaAssistant extends Disposable {
  * sending messages to the AI.
  */
 class ChatHistory extends Disposable {
-  public history: MutableObsArray<ChatMessage>;
-  public length: Computed<number>;
+  public conversationId: Observable<string>;
+  public conversation: MutableObsArray<ChatMessage>;
+  public conversationHistory: MutableObsArray<ChatMessage>;
+  public conversationLength: Computed<number>;
+  public conversationHistoryLength: Computed<number>;
+  public conversationSuggestedFormulas: Computed<string[]>;
   public lastSuggestedFormula: Computed<string|null>;
 
   private _element: HTMLElement;
@@ -649,31 +706,56 @@ class ChatHistory extends Disposable {
     column: ColumnRec,
     gristDoc: GristDoc,
     apply: (formula: string) => void,
+    logTelemetryEvent: (event: TelemetryEvent, includeContext?: boolean, metadata?: TelemetryMetadata) => void,
   }) {
     super();
+
     const column = this._options.column;
+    let conversationId = column.chatHistory.peek().get().conversationId;
+    if (!conversationId) {
+      conversationId = uuidv4();
+      const chatHistory = column.chatHistory.peek();
+      chatHistory.set({...chatHistory.get(), conversationId});
+    }
+    this.conversationId = Observable.create(this, conversationId);
+    this.autoDispose(this.conversationId.addListener((newConversationId) => {
+      // If a new conversation id was generated (e.g. on Clear Conversation), save it
+      // to the column's history.
+      const chatHistory = column.chatHistory.peek();
+      chatHistory.set({...chatHistory.get(), conversationId: newConversationId});
+    }));
+
     // Create observable array of messages that is connected to the column's chatHistory.
-    this.history = this.autoDispose(obsArray(column.chatHistory.peek().get().messages));
-    this.autoDispose(this.history.addListener((cur) => {
+    this.conversationHistory = this.autoDispose(obsArray(column.chatHistory.peek().get().messages));
+    this.autoDispose(this.conversationHistory.addListener((cur) => {
       const chatHistory = column.chatHistory.peek();
       chatHistory.set({...chatHistory.get(), messages: [...cur]});
     }));
-    this.length = Computed.create(this, use => use(this.history).length); // ??
+    this.conversation = this.autoDispose(obsArray());
+
+    this.conversationHistoryLength = Computed.create(this, use => use(this.conversationHistory).length);
+    this.conversationLength = Computed.create(this, use => use(this.conversation).length);
+
+    this.conversationSuggestedFormulas = Computed.create(this, use => {
+      return use(this.conversation)
+        .map(({formula}) => formula)
+        .filter((formula): formula is string => Boolean(formula));
+    });
     this.lastSuggestedFormula = Computed.create(this, use => {
-      return [...use(this.history)].reverse().find(entry => entry.formula)?.formula ?? null;
+      return [...use(this.conversationHistory)].reverse().find(({formula}) => formula)?.formula ?? null;
     });
   }
 
   public thinking(on = true) {
     if (!on) {
       // Find all index of all thinking messages.
-      const messages = [...this.history.get()].filter(m => m.message === '...');
+      const messages = [...this.conversationHistory.get()].filter(m => m.message === '...');
       // Remove all thinking messages.
       for (const message of messages) {
-        this.history.splice(this.history.get().indexOf(message), 1);
+        this.conversationHistory.splice(this.conversationHistory.get().indexOf(message), 1);
       }
     } else {
-      this.history.push({
+      this.conversationHistory.push({
         message: '...',
         sender: 'ai',
       });
@@ -688,20 +770,21 @@ class ChatHistory extends Disposable {
   public addResponse(message: ChatMessage) {
     // Clear any thinking from messages.
     this.thinking(false);
-    this.history.push({...message, sender: 'ai'});
+    const entry: ChatMessage = {...message, sender: 'ai'};
+    this.conversationHistory.push(entry);
+    this.conversation.push(entry);
     this.scrollDown();
   }
 
   public addQuestion(message: string) {
     this.thinking(false);
-    this.history.push({
-      message,
-      sender: 'user',
-    });
+    const entry: ChatMessage = {message, sender: 'user'};
+    this.conversationHistory.push(entry);
+    this.conversation.push(entry);
   }
 
   public lastQuestion() {
-    const list = this.history.get();
+    const list = this.conversationHistory.get();
     if (list.length === 0) {
       return null;
     }
@@ -713,14 +796,16 @@ class ChatHistory extends Disposable {
   }
 
   public removeLastResponse() {
-    const lastMessage = this.history.get()[this.history.get().length - 1];
+    const lastMessage = this.conversationHistory.get()[this.conversationHistory.get().length - 1];
     if (lastMessage?.sender === 'ai') {
-      this.history.pop();
+      this.conversationHistory.pop();
     }
   }
 
   public clear() {
-    this.history.set([]);
+    this._options.logTelemetryEvent('assistantClearConversation', true);
+    this.conversationId.set(uuidv4());
+    this.conversationHistory.set([]);
     const {column} = this._options;
     // Get the state of the chat from the column.
     const prevState = column.chatHistory.peek().get();
@@ -737,7 +822,7 @@ class ChatHistory extends Disposable {
   public buildDom() {
     return this._element = cssHistory(
       this._buildIntroMessage(),
-      dom.forEach(this.history, entry => {
+      dom.forEach(this.conversationHistory, entry => {
         if (entry.sender === 'user') {
           return cssMessage(
             dom('span',
@@ -857,13 +942,15 @@ class ChatHistory extends Disposable {
 async function askAI(grist: GristDoc, options: {
   column: ColumnRec,
   description: string,
+  conversationId: string,
   regenerate?: boolean,
   state?: AssistanceState
 }): Promise<AssistanceResponse> {
-  const {column, description, state, regenerate} = options;
+  const {column, description, conversationId, state, regenerate} = options;
   const tableId = column.table.peek().tableId.peek();
   const colId = column.colId.peek();
   const result = await grist.docApi.getAssistance({
+    conversationId,
     context: {type: 'formula', tableId, colId},
     text: description,
     state,

app/common/AssistancePrompts.ts

@@ -35,6 +35,7 @@ export type AssistanceContext = FormulaAssistanceContext;
  * A request for assistance.
  */
 export interface AssistanceRequest {
+  conversationId: string;
   context: AssistanceContext;
   state?: AssistanceState;
   text: string;

app/common/Telemetry.ts

@@ -25,6 +25,7 @@ export const TelemetryContracts: TelemetryContracts = {
   apiUsage: {
     description: 'Triggered when an HTTP request with an API key is made.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       method: {
         description: 'The HTTP request method (e.g. GET, POST, PUT).',
@@ -40,9 +41,254 @@ export const TelemetryContracts: TelemetryContracts = {
       },
     },
   },
+  assistantOpen: {
+    description: 'Triggered when the AI Assistant is first opened.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  assistantSend: {
+    description: 'Triggered when a message is sent to the AI Assistant.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      siteId: {
+        description: 'The id of the site.',
+        dataType: 'number',
+      },
+      siteType: {
+        description: 'The type of the site.',
+        dataType: 'string',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+      access: {
+        description: 'The document access level of the user that triggered this event.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      prompt: {
+        description: 'The role ("user" or "system"), content, and index of the message sent to the AI Assistant.',
+        dataType: 'object',
+      },
+    },
+  },
+  assistantReceive: {
+    description: 'Triggered when a message is received from the AI Assistant is received.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      siteId: {
+        description: 'The id of the site.',
+        dataType: 'number',
+      },
+      siteType: {
+        description: 'The type of the site.',
+        dataType: 'string',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+      access: {
+        description: 'The document access level of the user that triggered this event.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      message: {
+        description: 'The content and index of the message received from the AI Assistant.',
+        dataType: 'object',
+      },
+      suggestedFormula: {
+        description: 'The formula suggested by the AI Assistant, if present.',
+        dataType: 'string',
+      },
+    },
+  },
+  assistantSave: {
+    description: 'Triggered when changes in the expanded formula editor are saved after the AI Assistant ' +
+      'was opened.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      newFormula: {
+        description: 'The formula that was saved.',
+        dataType: 'string',
+      },
+      oldFormula: {
+        description: 'The formula that was overwritten.',
+        dataType: 'string',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  assistantCancel: {
+    description: 'Triggered when changes in the expanded formula editor are discarded after the AI Assistant ' +
+      'was opened.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  assistantClearConversation: {
+    description: 'Triggered when a conversation in the AI Assistant is cleared.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'short',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      context: {
+        description: 'The type of assistant (e.g. "formula"), table id, and column id.',
+        dataType: 'object',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
+  assistantClose: {
+    description: 'Triggered when a formula is saved or discarded after the AI Assistant was opened.',
+    minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
+    metadataContracts: {
+      docIdDigest: {
+        description: 'A hash of the doc id.',
+        dataType: 'string',
+      },
+      conversationId: {
+        description: 'A random identifier for the current conversation with the assistant.',
+        dataType: 'string',
+      },
+      suggestionApplied: {
+        description: 'True if a suggested formula from one of the received messages was applied.',
+        dataType: 'boolean',
+      },
+      conversationLength: {
+        description: 'The number of messages sent and received since opening the AI Assistant.',
+        dataType: 'number',
+      },
+      conversationHistoryLength: {
+        description: "The number of messages in the conversation's history. May be less than conversationLength "
+          + "if the conversation history was cleared in the same session.",
+        dataType: 'number',
+      },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
+    },
+  },
   beaconOpen: {
     description: 'Triggered when HelpScout Beacon is opened.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       userId: {
         description: 'The id of the user that triggered this event.',
@@ -57,6 +303,7 @@ export const TelemetryContracts: TelemetryContracts = {
   beaconArticleViewed: {
     description: 'Triggered when an article is opened in HelpScout Beacon.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       articleId: {
         description: 'The id of the article.',
@@ -75,6 +322,7 @@ export const TelemetryContracts: TelemetryContracts = {
   beaconEmailSent: {
     description: 'Triggered when an email is sent in HelpScout Beacon.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       userId: {
         description: 'The id of the user that triggered this event.',
@@ -89,6 +337,7 @@ export const TelemetryContracts: TelemetryContracts = {
   beaconSearch: {
     description: 'Triggered when a search is made in HelpScout Beacon.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       searchQuery: {
         description: 'The search query.',
@@ -107,6 +356,7 @@ export const TelemetryContracts: TelemetryContracts = {
   documentForked: {
     description: 'Triggered when a document is forked.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       docIdDigest: {
         description: 'A hash of the doc id.',
@@ -161,6 +411,7 @@ export const TelemetryContracts: TelemetryContracts = {
   documentOpened: {
     description: 'Triggered when a public document or template is opened.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       docIdDigest: {
         description: 'A hash of the doc id.',
@@ -211,6 +462,7 @@ export const TelemetryContracts: TelemetryContracts = {
   documentUsage: {
     description: 'Triggered on doc open and close, as well as hourly while a document is open.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       docIdDigest: {
         description: 'A hash of the doc id.',
@@ -346,6 +598,7 @@ export const TelemetryContracts: TelemetryContracts = {
   processMonitor: {
     description: 'Triggered every 5 seconds.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       heapUsedMB: {
         description: 'Size of JS heap in use, in MiB.',
@@ -368,6 +621,7 @@ export const TelemetryContracts: TelemetryContracts = {
   sendingWebhooks: {
     description: 'Triggered when sending webhooks.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       numEvents: {
         description: 'The number of events in the batch of webhooks being sent.',
@@ -406,6 +660,7 @@ export const TelemetryContracts: TelemetryContracts = {
   signupFirstVisit: {
     description: 'Triggered when a new user first opens the Grist app',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       siteId: {
         description: 'The site id of first visit after signup.',
@@ -429,6 +684,7 @@ export const TelemetryContracts: TelemetryContracts = {
     description: 'Triggered after a user successfully verifies their account during sign-up. '
       + 'Not triggered in grist-core.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       isAnonymousTemplateSignup: {
         description: 'Whether the user viewed any templates before signing up.',
@@ -443,6 +699,7 @@ export const TelemetryContracts: TelemetryContracts = {
   siteMembership: {
     description: 'Triggered daily.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       siteId: {
         description: 'The site id.',
@@ -469,6 +726,7 @@ export const TelemetryContracts: TelemetryContracts = {
   siteUsage: {
     description: 'Triggered daily.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       siteId: {
         description: 'The site id.',
@@ -508,6 +766,7 @@ export const TelemetryContracts: TelemetryContracts = {
   tutorialProgressChanged: {
     description: 'Triggered on changes to tutorial progress.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       tutorialForkIdDigest: {
         description: 'A hash of the tutorial fork id.',
@@ -529,11 +788,20 @@ export const TelemetryContracts: TelemetryContracts = {
         description: 'Percentage of tutorial completion.',
         dataType: 'number',
       },
+      userId: {
+        description: 'The id of the user that triggered this event.',
+        dataType: 'number',
+      },
+      altSessionId: {
+        description: 'A random, session-based identifier for the user that triggered this event.',
+        dataType: 'string',
+      },
     },
   },
   tutorialRestarted: {
     description: 'Triggered when a tutorial is restarted.',
     minimumTelemetryLevel: Level.full,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       tutorialForkIdDigest: {
         description: 'A hash of the tutorial fork id.',
@@ -572,6 +840,7 @@ export const TelemetryContracts: TelemetryContracts = {
   watchedVideoTour: {
     description: 'Triggered when the video tour is closed.',
     minimumTelemetryLevel: Level.limited,
+    retentionPeriod: 'indefinitely',
     metadataContracts: {
       watchTimeSeconds: {
         description: 'The number of seconds elapsed in the video player.',
@@ -595,6 +864,13 @@ type TelemetryContracts = Record<TelemetryEvent, TelemetryEventContract>;
 
 export const TelemetryEvents = StringUnion(
   'apiUsage',
+  'assistantOpen',
+  'assistantSend',
+  'assistantReceive',
+  'assistantSave',
+  'assistantCancel',
+  'assistantClearConversation',
+  'assistantClose',
   'beaconOpen',
   'beaconArticleViewed',
   'beaconEmailSent',
@@ -617,12 +893,15 @@ export type TelemetryEvent = typeof TelemetryEvents.type;
 interface TelemetryEventContract {
   description: string;
   minimumTelemetryLevel: Level;
+  retentionPeriod: TelemetryRetentionPeriod;
   metadataContracts?: Record<string, MetadataContract>;
 }
 
+export type TelemetryRetentionPeriod = 'short' | 'indefinitely';
+
 interface MetadataContract {
   description: string;
-  dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date';
+  dataType: 'boolean' | 'number' | 'string' | 'string[]' | 'date' | 'object';
   minimumTelemetryLevel?: Level;
 }
 
@@ -643,9 +922,6 @@ export type TelemetryMetadata = Record<string, any>;
  */
 export const TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME = 'gr_template_signup_trk';
 
-// A set of metadata keys that are always allowed when logging.
-const ALLOWED_METADATA_KEYS = new Set(['eventSource', 'installationId']);
-
 /**
  * Returns a function that accepts a telemetry event and metadata, and performs various
  * checks on it based on a set of contracts and the `telemetryLevel`.
@@ -670,8 +946,6 @@ export function buildTelemetryEventChecker(telemetryLevel: TelemetryLevel) {
     }
 
     for (const [key, value] of Object.entries(metadata ?? {})) {
-      if (ALLOWED_METADATA_KEYS.has(key))  { continue; }
-
       const metadataContract = eventContract.metadataContracts?.[key];
       if (!metadataContract) {
         throw new Error(`Unknown metadata for telemetry event ${event}: ${key}`);

app/server/lib/Assistance.ts

@@ -5,6 +5,7 @@
 import {AssistanceMessage, AssistanceRequest, AssistanceResponse} from 'app/common/AssistancePrompts';
 import {delay} from 'app/common/delay';
 import {DocAction} from 'app/common/DocActions';
+import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {OptDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
 import fetch from 'node-fetch';
@@ -17,7 +18,7 @@ export const DEPS = { fetch, delayTime: 1000 };
  * An assistant can help a user do things with their document,
  * by interfacing with an external LLM endpoint.
  */
-export interface Assistant {
+interface Assistant {
   apply(session: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse>;
 }
 
@@ -25,7 +26,7 @@ export interface Assistant {
  * Document-related methods for use in the implementation of assistants.
  * Somewhat ad-hoc currently.
  */
-export interface AssistanceDoc {
+interface AssistanceDoc extends ActiveDoc {
   /**
    * Generate a particular prompt coded in the data engine for some reason.
    * It makes python code for some tables, and starts a function body with
@@ -117,10 +118,11 @@ export class OpenAIAssistant implements Assistant {
   public async apply(
     optSession: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
     const messages = request.state?.messages || [];
+    const newMessages = [];
     const chatMode = this._chatMode;
     if (chatMode) {
       if (messages.length === 0) {
-        messages.push({
+        newMessages.push({
           role: 'system',
           content: 'You are a helpful assistant for a user of software called Grist. ' +
             'Below are one or more Python classes. ' +
@@ -141,7 +143,7 @@ export class OpenAIAssistant implements Assistant {
             await makeSchemaPromptV1(optSession, doc, request) +
             '\n```',
         });
-        messages.push({
+        newMessages.push({
           role: 'user', content: request.text,
         });
       } else {
@@ -150,21 +152,49 @@ export class OpenAIAssistant implements Assistant {
             messages.pop();
           }
         }
-        messages.push({
+        newMessages.push({
           role: 'user', content: request.text,
         });
       }
     } else {
       messages.length = 0;
-      messages.push({
+      newMessages.push({
         role: 'user', content: await makeSchemaPromptV1(optSession, doc, request),
       });
     }
+    messages.push(...newMessages);
+
+    const newMessagesStartIndex = messages.length - newMessages.length;
+    for (const [index, {role, content}] of newMessages.entries()) {
+      doc.logTelemetryEvent(optSession, 'assistantSend', {
+        full: {
+          conversationId: request.conversationId,
+          context: request.context,
+          prompt: {
+            index: newMessagesStartIndex + index,
+            role,
+            content,
+          },
+        },
+      });
+    }
+
     const completion: string = await this._getCompletion(messages);
     const response = await completionToResponse(doc, request, completion, completion);
     if (chatMode) {
       response.state = {messages};
     }
+    doc.logTelemetryEvent(optSession, 'assistantReceive', {
+      full: {
+        conversationId: request.conversationId,
+        context: request.context,
+        message: {
+          index: messages.length - 1,
+          content: completion,
+        },
+        suggestedFormula: (response.suggestedActions[0]?.[3] as any)?.formula,
+      },
+    });
     return response;
   }
 
@@ -307,7 +337,7 @@ export class HuggingFaceAssistant implements Assistant {
 /**
  * Test assistant that mimics ChatGPT and just returns the input.
  */
-export class EchoAssistant implements Assistant {
+class EchoAssistant implements Assistant {
   public async apply(sess: OptDocSession, doc: AssistanceDoc, request: AssistanceRequest): Promise<AssistanceResponse> {
     if (request.text === "ERROR") {
       throw new Error(`ERROR`);

app/server/lib/Telemetry.ts

@@ -12,6 +12,7 @@ import {
   TelemetryLevels,
   TelemetryMetadata,
   TelemetryMetadataByLevel,
+  TelemetryRetentionPeriod,
 } from 'app/common/Telemetry';
 import {TelemetryPrefsWithSources} from 'app/common/InstallAPI';
 import {Activation} from 'app/gen-server/entity/Activation';
@@ -44,19 +45,15 @@ const MAX_PENDING_FORWARD_EVENT_REQUESTS = 25;
  */
 export class Telemetry implements ITelemetry {
   private _activation: Activation | undefined;
-  private readonly _deploymentType = this._gristServer.getDeploymentType();
-
   private _telemetryPrefs: TelemetryPrefsWithSources | undefined;
-
+  private readonly _deploymentType = this._gristServer.getDeploymentType();
   private readonly _shouldForwardTelemetryEvents = this._deploymentType !== 'saas';
   private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
     'https://telemetry.getgrist.com/api/telemetry';
   private _numPendingForwardEventRequests = 0;
-
-
   private readonly _logger = new LogMethods('Telemetry ', () => ({}));
-  private readonly _telemetryLogger = new LogMethods('Telemetry ', () => ({
-    eventType: 'telemetry',
+  private readonly _telemetryLogger = new LogMethods<string>('Telemetry ', (eventType) => ({
+    eventType,
   }));
 
   private _checkTelemetryEvent: TelemetryEventChecker | undefined;
@@ -153,21 +150,17 @@ export class Telemetry implements ITelemetry {
      */
     app.post('/api/telemetry', expressWrap(async (req, resp) => {
       const mreq = req as RequestWithLogin;
-      const event = stringParam(req.body.event, 'event', TelemetryEvents.values);
+      const event = stringParam(req.body.event, 'event', TelemetryEvents.values) as TelemetryEvent;
       if ('eventSource' in req.body.metadata) {
-        this._telemetryLogger.rawLog('info', null, event, {
-          eventName: event,
+        this._telemetryLogger.rawLog('info', getEventType(event), event, {
           ...(removeNullishKeys(req.body.metadata)),
+          eventName: event,
         });
       } else {
         try {
           this._assertTelemetryIsReady();
-          await this.logEvent(event as TelemetryEvent, merge(
+          await this.logEvent(event, merge(
             {
-              limited: {
-                eventSource: `grist-${this._deploymentType}`,
-                ...(this._deploymentType !== 'saas' ? {installationId: this._activation!.id} : {}),
-              },
               full: {
                 userId: mreq.userId,
                 altSessionId: mreq.altSessionId,
@@ -219,10 +212,11 @@ export class Telemetry implements ITelemetry {
     event: TelemetryEvent,
     metadata?: TelemetryMetadata
   ) {
-    this._telemetryLogger.rawLog('info', null, event, {
+    this._telemetryLogger.rawLog('info', getEventType(event), event, {
+      ...metadata,
       eventName: event,
       eventSource: `grist-${this._deploymentType}`,
-      ...metadata,
+      installationId: this._activation!.id,
     });
   }
 
@@ -238,7 +232,15 @@ export class Telemetry implements ITelemetry {
 
     try {
       this._numPendingForwardEventRequests += 1;
-      await this._doForwardEvent(JSON.stringify({event, metadata}));
+      await this._doForwardEvent(JSON.stringify({
+        event,
+        metadata: {
+          ...metadata,
+          eventName: event,
+          eventSource: `grist-${this._deploymentType}`,
+          installationId: this._activation!.id,
+        }
+      }));
     } catch (e) {
       this._logger.error(undefined, `failed to forward telemetry event ${event}`, e);
     } finally {
@@ -342,3 +344,15 @@ export function hashDigestKeys(metadata: TelemetryMetadata): TelemetryMetadata {
   });
   return filteredMetadata;
 }
+
+type TelemetryEventType = 'telemetry' | 'telemetry-short-retention';
+
+const EventTypeByRetentionPeriod: Record<TelemetryRetentionPeriod, TelemetryEventType> = {
+  indefinitely: 'telemetry',
+  short: 'telemetry-short-retention',
+};
+
+function getEventType(event: TelemetryEvent) {
+  const {retentionPeriod} = TelemetryContracts[event];
+  return EventTypeByRetentionPeriod[retentionPeriod];
+}

test/formula-dataset/runCompletion_impl.ts

@@ -164,6 +164,7 @@ where c.colId = ? and t.tableId = ?
           formula = colInfo?.formula;
 
           const result = await sendForCompletion(session, activeDoc, {
+            conversationId: 'conversationId',
             context: {type: 'formula', tableId, colId},
             state: history,
             text: followUp || description,

test/server/lib/Telemetry.ts

@@ -111,6 +111,7 @@ describe('Telemetry', function() {
                   eventSource: `grist-${deploymentType}`,
                   docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=',
                   isPublic: false,
+                  installationId,
                 }
               ]);
             }
@@ -133,6 +134,7 @@ describe('Telemetry', function() {
                   docIdDigest: 'dige:Vq9L3nCkeufQ8euzDkXtM2Fl1cnsALqakjEeM6QlbXQ=',
                   isPublic: false,
                   userId: 1,
+                  installationId,
                 }
               ]);
             }
@@ -239,6 +241,7 @@ describe('Telemetry', function() {
                 eventName: 'watchedVideoTour',
                 eventSource: `grist-${deploymentType}`,
                 watchTimeSeconds: 30,
+                installationId,
               });
             } else {
               assert.containsAllKeys(metadata, [
@@ -298,14 +301,10 @@ describe('Telemetry', function() {
             assert.equal(event, 'watchedVideoTour');
             if (telemetryLevel === 'limited') {
               assert.deepEqual(metadata, {
-                eventSource: `grist-${deploymentType}`,
-                installationId,
                 watchTimeSeconds: 30,
               });
             } else {
               assert.containsAllKeys(metadata, [
-                'eventSource',
-                'installationId',
                 'watchTimeSeconds',
                 'userId',
                 'altSessionId',