(core) New API to collect timing information from formula evaluation.

Summary:
- /timing/start endpoint to start collecting information
- /timing/stop endpoint to stop collecting
- /timing to retrive data gatherd so far

Timings are collected for all columns (including hidden/helpers/system)

Test Plan: Added new

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D4230

app/client/components/DocComm.ts

@@ -48,6 +48,8 @@ export class DocComm extends Disposable implements ActiveDocAPI {
   public getUsersForViewAs = this._wrapMethod("getUsersForViewAs");
   public getAccessToken = this._wrapMethod("getAccessToken");
   public getShare = this._wrapMethod("getShare");
+  public startTiming = this._wrapMethod("startTiming");
+  public stopTiming = this._wrapMethod("stopTiming");
 
   public changeUrlIdEmitter = this.autoDispose(new Emitter());
 

app/common/ActiveDocAPI.ts

@@ -288,6 +288,48 @@ export interface RemoteShareInfo {
   key: string;
 }
 
+/**
+ * Metrics gathered during formula calculations.
+ */
+export interface TimingInfo {
+  /**
+   * Total time spend evaluating a formula.
+   */
+  total: number;
+  /**
+   * Number of times the formula was evaluated (for all rows).
+   */
+  count: number;
+  average: number;
+  max: number;
+}
+
+/**
+ * Metrics attached to a particular column in a table. Contains also marks if they were gathered.
+ * Currently we only mark the `OrderError` exception (so when formula calculation was restarted due to
+ * order dependency).
+ */
+export interface FormulaTimingInfo extends TimingInfo {
+  tableId: string;
+  colId: string;
+  marks?: Array<TimingInfo & {name: string}>;
+}
+
+/*
+ * Status of timing info collection. Contains intermediate results if engine is not busy at the moment.
+ */
+export interface TimingStatus {
+  /**
+   * If true, timing info is being collected.
+   */
+  status: boolean;
+  /**
+   * Will be undefined if we can't get the timing info (e.g. if the document is locked by other call).
+   * Otherwise, contains the intermediate results gathered so far.
+   */
+  timing?: FormulaTimingInfo[];
+}
+
 export interface ActiveDocAPI {
   /**
    * Closes a document, and unsubscribes from its userAction events.
@@ -449,5 +491,18 @@ export interface ActiveDocAPI {
    */
   getUsersForViewAs(): Promise<PermissionDataWithExtraUsers>;
 
-  getShare(linkId: string): Promise<RemoteShareInfo>;
+  /**
+   * Get a share info associated with the document.
+   */
+  getShare(linkId: string): Promise<RemoteShareInfo|null>;
+
+  /**
+   * Starts collecting timing information from formula evaluations.
+   */
+  startTiming(): Promise<void>;
+
+  /**
+   * Stops collecting timing information and returns the collected data.
+   */
+  stopTiming(): Promise<TimingInfo[]>;
 }

app/common/UserAPI.ts

@@ -507,6 +507,16 @@ export interface DocAPI {
   flushWebhook(webhookId: string): Promise<void>;
 
   getAssistance(params: AssistanceRequest): Promise<AssistanceResponse>;
+  /**
+   * Check if the document is currently in timing mode.
+   */
+  timing(): Promise<{status: boolean}>;
+  /**
+   * Starts recording timing information for the document. Throws exception if timing is already
+   * in progress or you don't have permission to start timing.
+   */
+  startTiming(): Promise<void>;
+  stopTiming(): Promise<void>;
 }
 
 // Operations that are supported by a doc worker.
@@ -1121,6 +1131,18 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
     });
   }
 
+  public async timing(): Promise<{status: boolean}> {
+    return this.requestJson(`${this._url}/timing`);
+  }
+
+  public async startTiming(): Promise<void> {
+    await this.request(`${this._url}/timing/start`, {method: 'POST'});
+  }
+
+  public async stopTiming(): Promise<void> {
+    await this.request(`${this._url}/timing/stop`, {method: 'POST'});
+  }
+
   private _getRecords(tableId: string, endpoint: 'data' | 'records', options?: GetRowsParams): Promise<any> {
     const url = new URL(`${this._url}/tables/${tableId}/${endpoint}`);
     if (options?.filters) {

app/gen-server/lib/DocApiForwarder.ts

@@ -70,6 +70,9 @@ export class DocApiForwarder {
     app.use('/api/docs/:docId/webhooks', withDoc);
     app.use('/api/docs/:docId/assistant', withDoc);
     app.use('/api/docs/:docId/sql', withDoc);
+    app.use('/api/docs/:docId/timing', withDoc);
+    app.use('/api/docs/:docId/timing/start', withDoc);
+    app.use('/api/docs/:docId/timing/stop', withDoc);
     app.use('/api/docs/:docId/forms/:vsId', withDoc);
     app.use('^/api/docs$', withoutDoc);
   }

app/server/lib/ActiveDoc.ts

@@ -22,6 +22,7 @@ import {
   ApplyUAResult,
   DataSourceTransformed,
   ForkResult,
+  FormulaTimingInfo,
   ImportOptions,
   ImportResult,
   ISuggestionWithValue,
@@ -220,6 +221,7 @@ export class ActiveDoc extends EventEmitter {
   public docData: DocData|null = null;
   // Used by DocApi to only allow one webhook-related endpoint to run at a time.
   public readonly triggersLock: Mutex = new Mutex();
+  public isTimingOn = false;
 
   protected _actionHistory: ActionHistory;
   protected _docManager: DocManager;
@@ -1366,6 +1368,7 @@ export class ActiveDoc extends EventEmitter {
    */
   public async reloadDoc(docSession?: DocSession) {
     this._log.debug(docSession || null, 'ActiveDoc.reloadDoc starting shutdown');
+    this._docManager.restoreTimingOn(this.docName, this.isTimingOn);
     return this.shutdown();
   }
 
@@ -1870,6 +1873,40 @@ export class ActiveDoc extends EventEmitter {
     return await this._getHomeDbManagerOrFail().getShareByLinkId(this.docName, linkId);
   }
 
+  public async startTiming(): Promise<void> {
+    // Set the flag to indicate that timing is on.
+    this.isTimingOn = true;
+
+    try {
+      // Call the data engine to start timing.
+      await this._doStartTiming();
+    } catch (e) {
+      this.isTimingOn = false;
+      throw e;
+    }
+
+    // Mark self as in timing mode, in case we get reloaded.
+    this._docManager.restoreTimingOn(this.docName, true);
+  }
+
+  public async stopTiming(): Promise<FormulaTimingInfo[]> {
+    // First call the data engine to stop timing, and gather results.
+    const timingResults = await this._pyCall('stop_timing');
+
+    // Toggle the flag and clear the reminder.
+    this.isTimingOn = false;
+    this._docManager.restoreTimingOn(this.docName, false);
+
+    return timingResults;
+  }
+
+  public async getTimings(): Promise<FormulaTimingInfo[]|void>  {
+    if (this._modificationLock.isLocked()) {
+      return;
+    }
+    return await this._pyCall('get_timings');
+  }
+
   /**
    * Loads an open document from DocStorage.  Returns a list of the tables it contains.
    */
@@ -2377,6 +2414,10 @@ export class ActiveDoc extends EventEmitter {
         });
         await this._pyCall('initialize', this._options?.docUrl);
 
+        if (this.isTimingOn) {
+          await this._doStartTiming();
+        }
+
         // Calculations are not associated specifically with the user opening the document.
         // TODO: be careful with which users can create formulas.
         await this._applyUserActions(makeExceptionalDocSession('system'), [['Calculate']]);
@@ -2686,7 +2727,9 @@ export class ActiveDoc extends EventEmitter {
   }
 
   private async _getEngine(): Promise<ISandbox> {
-    if (this._shuttingDown) { throw new Error('shutting down, data engine unavailable'); }
+    if (this._shuttingDown) {
+      throw new Error('shutting down, data engine unavailable');
+    }
     if (this._dataEngine) { return this._dataEngine; }
 
     this._dataEngine = this._isSnapshot ? this._makeNullEngine() : this._makeEngine();
@@ -2830,6 +2873,10 @@ export class ActiveDoc extends EventEmitter {
     return dbManager;
   }
 
+  private _doStartTiming() {
+    return  this._pyCall('start_timing');
+  }
+
 }
 
 // Helper to initialize a sandbox action bundle with no values.

app/server/lib/DocApi.ts

@@ -1556,6 +1556,40 @@ export class DocWorkerApi {
         });
       })
     );
+
+    // GET /api/docs/:docId/timings
+    // Checks if timing is on for the document.
+    this._app.get('/api/docs/:docId/timing', isOwner, withDoc(async (activeDoc, req, res) => {
+      if (!activeDoc.isTimingOn) {
+        res.json({status: 'disabled'});
+      } else {
+        const timing =  await activeDoc.getTimings();
+        const status = timing ? 'active' : 'pending';
+        res.json({status, timing});
+      }
+    }));
+
+    // POST /api/docs/:docId/timings/start
+    // Start a timing for the document.
+    this._app.post('/api/docs/:docId/timing/start', isOwner, withDoc(async (activeDoc, req, res) => {
+      if (activeDoc.isTimingOn) {
+        res.status(400).json({error:`Timing already started for ${activeDoc.docName}`});
+        return;
+      }
+      // isTimingOn flag is switched synchronously.
+      await activeDoc.startTiming();
+      res.sendStatus(200);
+    }));
+
+    // POST /api/docs/:docId/timings/stop
+    // Stop a timing for the document.
+    this._app.post('/api/docs/:docId/timing/stop', isOwner, withDoc(async (activeDoc, req, res) => {
+      if (!activeDoc.isTimingOn) {
+        res.status(400).json({error:`Timing not started for ${activeDoc.docName}`});
+        return;
+      }
+      res.json(await activeDoc.stopTiming());
+    }));
   }
 
   /**

app/server/lib/DocManager.ts

@@ -43,7 +43,10 @@ export const DEFAULT_CACHE_TTL = 10000;
 
 // How long to remember that a document has been explicitly set in a
 // recovery mode.
-export const RECOVERY_CACHE_TTL = 30000;
+export const RECOVERY_CACHE_TTL = 30000; // 30 seconds
+
+// How long to remember the timing mode of a document.
+export const TIMING_ON_CACHE_TTL = 30000; // 30 seconds
 
 /**
  * DocManager keeps track of "active" Grist documents, i.e. those loaded
@@ -56,6 +59,9 @@ export class DocManager extends EventEmitter {
   // Remember recovery mode of documents.
   private _inRecovery = new MapWithTTL<string, boolean>(RECOVERY_CACHE_TTL);
 
+  // Remember timing mode of documents, when document is recreated it is put in the same mode.
+  private _inTimingOn = new MapWithTTL<string, boolean>(TIMING_ON_CACHE_TTL);
+
   constructor(
     public readonly storageManager: IDocStorageManager,
     public readonly pluginManager: PluginManager|null,
@@ -69,6 +75,13 @@ export class DocManager extends EventEmitter {
     this._inRecovery.set(docId, recovery);
   }
 
+  /**
+   * Will restore timing on a document when it is reloaded.
+   */
+  public restoreTimingOn(docId: string, timingOn: boolean) {
+    this._inTimingOn.set(docId, timingOn);
+  }
+
   // attach a home database to the DocManager.  During some tests, it
   // is awkward to have this set up at the point of construction.
   public testSetHomeDbManager(dbManager: HomeDBManager) {
@@ -437,6 +450,10 @@ export class DocManager extends EventEmitter {
       log.error('DocManager had problem shutting down storage: %s', err.message);
     }
 
+    // Clear any timeouts we might have.
+    this._inRecovery.clear();
+    this._inTimingOn.clear();
+
     // Clear the setInterval that the pidusage module sets up internally.
     pidusage.clear();
   }
@@ -601,7 +618,10 @@ export class DocManager extends EventEmitter {
     const doc = await this._getDoc(docSession, docName);
     // Get URL for document for use with SELF_HYPERLINK().
     const docUrls = doc && await this._getDocUrls(doc);
-    return new ActiveDoc(this, docName, {...docUrls, safeMode, doc});
+    const activeDoc = new ActiveDoc(this, docName, {...docUrls, safeMode, doc});
+    // Restore the timing mode of the document.
+    activeDoc.isTimingOn = this._inTimingOn.get(docName) || false;
+    return activeDoc;
   }
 
   /**

app/server/lib/DocWorker.ts

@@ -132,6 +132,8 @@ export class DocWorker {
       getUsersForViewAs:        activeDocMethod.bind(null, 'viewers', 'getUsersForViewAs'),
       getAccessToken:           activeDocMethod.bind(null, 'viewers', 'getAccessToken'),
       getShare:                 activeDocMethod.bind(null, 'owners', 'getShare'),
+      startTiming:              activeDocMethod.bind(null, 'owners', 'startTiming'),
+      stopTiming:               activeDocMethod.bind(null, 'owners', 'stopTiming'),
     });
   }