(core) Adds a UI panel for managing webhooks

Summary:
This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo:

  * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables.
  * Cell values that would create an error can now be denied and reverted (as for the rest of Grist).
  * Changes made by other users are integrated in a sane way.
  * Basic undo/redo support is added using the regular undo/redo stack.
  * The table list in the drop-down is now updated if schema changes.
  * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation.
  *  Factored out webhook specific logic from general virtual table support.
  * Made a bunch of fixes to various broken behavior.
  * Added tests.

The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case.

I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error.

I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky.

Contains a migration, so needs an extra reviewer for that.

Test Plan: added tests

Reviewers: jarek, dsagal

Reviewed By: jarek, dsagal

Differential Revision: https://phab.getgrist.com/D3856

app/client/models/DocData.ts

@@ -32,6 +32,8 @@ export class DocData extends BaseDocData {
   private _lastActionNum: number|null = null;   // ActionNum of the last action in the current bundle, or null.
   private _bundleSender: BundleSender;
 
+  private _virtualTablesFunc: Map<string, Constructor<TableData>>;
+
   /**
    * Constructor for DocData.
    * @param {Object} docComm: A map of server methods available on this document.
@@ -41,10 +43,12 @@ export class DocData extends BaseDocData {
   constructor(public readonly docComm: DocComm, metaTableData: {[tableId: string]: TableDataAction}) {
     super((tableId) => docComm.fetchTable(tableId), metaTableData);
     this._bundleSender = new BundleSender(this.docComm);
+    this._virtualTablesFunc = new Map();
   }
 
   public createTableData(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap): TableData {
-    return new TableData(this, tableId, tableData, colTypes);
+    const Cons = this._virtualTablesFunc?.get(tableId) || TableData;
+    return new Cons(this, tableId, tableData, colTypes);
   }
 
   // Version of inherited getTable() which returns the enhance TableData type.
@@ -182,8 +186,16 @@ export class DocData extends BaseDocData {
     return this.sendActions([action], optDesc).then((retValues) => retValues[0]);
   }
 
+  public registerVirtualTable(tableId: string, Cons: typeof TableData) {
+    this._virtualTablesFunc.set(tableId, Cons);
+  }
+
   // See documentation of sendActions().
   private async _sendActionsImpl(actions: UserAction[], optDesc?: string): Promise<any[]> {
+    if (this._virtualTablesFunc?.has(actions[0]?.[1] as any)) {
+      // It would be easy to pass along actions, but we don't need this functionality yet.
+      throw new Error('_sendActionsImpl needs updating to direct actions to virtual tables');
+    }
     const eventData = {actions};
     this.sendActionsEmitter.emit(eventData);
     const options = { desc: optDesc };
@@ -282,3 +294,5 @@ export interface BundlingInfo<T = unknown> {
   // Promise for when the bundle has been finalized.
   completionPromise: Promise<void>;
 }
+
+type Constructor<T> = new (...args: any[]) => T;

app/client/models/DocModel.ts

@@ -281,6 +281,7 @@ export class DocModel {
   private _createVisibleTablesArray(): KoArray<TableRec> {
     return createTablesArray(this.tables, r =>
       !isHiddenTable(this.tables.tableData, r) &&
+      !isVirtualTable(this.tables.tableData, r) &&
       (!isTutorialTable(this.tables.tableData, r) || this.showDocTutorialTable)
     );
   }
@@ -326,3 +327,11 @@ function createTablesArray(
 function isTutorialTable(tablesData: TableData, tableRef: UIRowId): boolean {
   return tablesData.getValue(tableRef, 'tableId') === 'GristDocTutorial';
 }
+
+/**
+ * Check whether a table is virtual - currently that is done
+ * by having a string rowId rather than the expected integer.
+ */
+function isVirtualTable(tablesData: TableData, tableRef: UIRowId): boolean {
+  return typeof(tableRef) === 'string';
+}

app/client/models/VirtualTable.ts

@@ -0,0 +1,242 @@
+import { reportError } from 'app/client/models/errors';
+import { GristDoc } from 'app/client/components/GristDoc';
+import { DocData } from 'app/client/models/DocData';
+import { TableData } from 'app/client/models/TableData';
+import { concatenateSummaries, summarizeStoredAndUndo } from 'app/common/ActionSummarizer';
+import { TableDelta } from 'app/common/ActionSummary';
+import { ProcessedAction } from 'app/common/AlternateActions';
+import { DisposableWithEvents } from 'app/common/DisposableWithEvents';
+import { DocAction, TableDataAction, UserAction } from 'app/common/DocActions';
+import { DocDataCache } from 'app/common/DocDataCache';
+import { ColTypeMap } from 'app/common/TableData';
+import { RowRecord } from 'app/plugin/GristData';
+import debounce = require('lodash/debounce');
+
+/**
+ * An interface for use while editing a virtual table.
+ * This is the interface passed to beforeEdit and afterEdit callbacks.
+ * The getRecord method gives access to the record prior to the edit;
+ * the getRecordNew method gives access to (an internal copy of)
+ * the record after the edit.
+ * The same interface is passed in other places, in which case
+ * actions and delta are trivial.
+ */
+export interface IEdit {
+  gristDoc: GristDoc,
+  actions: ProcessedAction[],  // UserActions plus corresponding DocActions (forward and undo).
+  delta: TableDelta,           // A summary of the effect actions would have (or had).
+
+  /**
+   * Apply a set of actions. The result is from the store backing the
+   * virtual table. Will not trigger beforeEdit or afterEdit callbacks.
+   */
+  patch(actions: UserAction[]): Promise<ProcessedAction[]>;
+
+  getRecord(rowId: number): RowRecord|undefined;     // A record in the table.
+  getRecordNew(rowId: number): RowRecord|undefined;  // A record in the table, after the edit.
+  getRowIds(): readonly number[];  // All rowIds in the table.
+}
+
+/**
+ * Interface with a back-end for a specific virtual table.
+ */
+export interface IExternalTable {
+  name: string;  // the tableId of the virtual table (e.g. GristHidden_WebhookTable)
+  initialActions: DocAction[];  // actions to create the table.
+  fetchAll(): Promise<TableDataAction>;  // get initial state of the table.
+  sync(editor: IEdit): Promise<void>;    // incorporate external changes.
+  beforeEdit(editor: IEdit): Promise<void>;  // called prior to committing a change.
+  afterEdit(editor: IEdit): Promise<void>;   // called after committing a change.
+  afterAnySchemaChange(editor: IEdit): Promise<void>;  // called after any schema change in the document.
+}
+
+// A counter to generate unique actionNums for undo actions.
+let _counterForUndoActions: number = 1;
+
+/**
+ * A flavor of TableData that is backed by external operations and local cache.
+ * This lets virtual tables "fit in" to a DocData instance.
+ */
+export class VirtualTableData extends TableData {
+
+  public gristDoc: GristDoc;
+  public ext: IExternalTable;
+  public cache: DocDataCache;
+
+  constructor(docData: DocData, tableId: string, tableData: TableDataAction|null, columnTypes: ColTypeMap) {
+    super(docData, tableId, tableData, columnTypes);
+  }
+
+  public setExt(_ext: IExternalTable) {
+    this.ext = _ext;
+    this.cache = new DocDataCache(this.ext.initialActions);
+  }
+
+  public get name() {
+    return this.ext.name;
+  }
+
+  public fetchData() {
+    return super.fetchData(async () => {
+      const data = await this.ext.fetchAll();
+      this.cache.docData.getTable(this.name)?.loadData(data);
+      return data;
+    });
+  }
+
+  public async sendTableActions(userActions: UserAction[]): Promise<any[]> {
+    const actions = await this._sendTableActionsCore(userActions,
+                                                     {isUser: true});
+    await this.ext.afterEdit(this._editor(actions));
+    return actions.map(action => action.retValues);
+  }
+
+  public sync() {
+    return this.ext.sync(this._editor());
+  }
+
+  public async sendTableAction(action: UserAction): Promise<any> {
+    const retValues = await this.sendTableActions([action]);
+    return retValues[0];
+  }
+
+  public async schemaChange() {
+    await this.ext.afterAnySchemaChange(this._editor());
+  }
+
+  private _editor(actions: ProcessedAction[] = []): IEdit {
+    const summary = concatenateSummaries(
+      actions
+        .map(action => summarizeStoredAndUndo(action.stored, action.undo)));
+    const delta = summary.tableDeltas[this.name];
+    return {
+      actions,
+      delta,
+      gristDoc: this.gristDoc,
+      getRecord: rowId => this.getRecord(rowId),
+      getRecordNew: rowId => this.getRecord(rowId),
+      getRowIds: () => this.getRowIds(),
+      patch: userActions => this._sendTableActionsCore(userActions, {
+        hasTableIds: true,
+        isUser: false,
+      })
+    };
+  }
+
+  private async _sendTableActionsCore(userActions: UserAction[], options: {
+    isUser: boolean,
+    isUndo?: boolean,
+    hasTableIds?: boolean,
+    actionNum?: any,
+  }): Promise<ProcessedAction[]> {
+    const {isUndo, isUser, hasTableIds} = options;
+    if (!hasTableIds) {
+      userActions.forEach((action) => action.splice(1, 0, this.tableId));
+    }
+    const actions = await this.cache.sendTableActions(userActions);
+    if (isUser) {
+      const newTable = await this.cache.docData.requireTable(this.name);
+      try {
+        await this.ext.beforeEdit({
+          ...this._editor(actions),
+          getRecordNew: rowId => newTable.getRecord(rowId),
+        });
+      } catch (e) {
+        actions.reverse();
+        for (const action of actions) {
+          await this.cache.sendTableActions(action.undo);
+        }
+        throw e;
+      }
+    }
+
+    for (const action of actions) {
+      for (const docAction of action.stored) {
+        this.docData.receiveAction(docAction);
+        this.cache.docData.receiveAction(docAction);
+        if (isUser) {
+          const code = `ext-${this.name}-${_counterForUndoActions}`;
+          _counterForUndoActions++;
+          this.gristDoc.getUndoStack().pushAction({
+            actionNum: code,
+            actionHash: 'hash',
+            fromSelf: true,
+            otherId: options.actionNum || 0,
+            linkId: 0,
+            rowIdHint: 0,
+            isUndo,
+            action,
+            op: this._doUndo.bind(this),
+          } as any);
+        }
+      }
+    }
+    return actions;
+  }
+
+
+  private async _doUndo(actionGroup: {
+    action: ProcessedAction,
+    actionNum: number|string,
+  }, isUndo: boolean) {
+    await this._sendTableActionsCore(
+      isUndo ? actionGroup.action.undo : actionGroup.action.stored,
+      {
+        isUndo,
+        isUser: true,
+        actionNum: actionGroup.actionNum,
+        hasTableIds: true,
+      });
+  }
+}
+
+/**
+ * Everything needed to run a virtual table. Contains a tableData instance.
+ * Subscribes to schema changes. Offers a debouncing lazySync method that
+ * will attempt to synchronize the virtual table with the external source
+ * one second after last call (or at most 2 seconds after the first
+ * call).
+ */
+export class VirtualTable {
+  public lazySync = debounce(this.sync, 1000, {
+    maxWait: 2000,
+    trailing: true,
+  });
+  public tableData: VirtualTableData;
+
+  public constructor(private _owner: DisposableWithEvents,
+                     _gristDoc: GristDoc,
+                     _ext: IExternalTable) {
+    if (!_gristDoc.docModel.docData.getTable(_ext.name)) {
+
+      // register the virtual table
+      _gristDoc.docModel.docData.registerVirtualTable(_ext.name, VirtualTableData);
+
+      // then process initial actions
+      for (const action of _ext.initialActions) {
+        _gristDoc.docData.receiveAction(action);
+      }
+
+      // pass in gristDoc and external interface
+      this.tableData = _gristDoc.docModel.docData.getTable(_ext.name)! as VirtualTableData;
+      //this.tableData.docApi = this.docApi;
+      this.tableData.gristDoc = _gristDoc;
+      this.tableData.setExt(_ext);
+
+      // subscribe to schema changes
+      this.tableData.schemaChange().catch(e => reportError(e));
+      _owner.listenTo(_gristDoc, 'schemaUpdateAction', () => this.tableData.schemaChange());
+    } else {
+      this.tableData = _gristDoc.docModel.docData.getTable(_ext.name)! as VirtualTableData;
+    }
+    // debounce is typed as returning a promise, but doesn't appear to actually do so?
+    Promise.resolve(this.lazySync()).catch(e => reportError(e));
+  }
+
+  public async sync() {
+    if (this._owner.isDisposed()) {
+      return;
+    }
+    await this.tableData.sync();
+  }
+}

app/client/models/entities/ViewSectionRec.ts

@@ -56,6 +56,7 @@ export interface ViewSectionRec extends IRowModel<"_grist_Views_section">, RuleO
   // true if this record is its table's rawViewSection, i.e. a 'raw data view'
   // in which case the UI prevents various things like hiding columns or changing the widget type.
   isRaw: ko.Computed<boolean>;
+  isVirtual: ko.Computed<boolean>;
   isCollapsed: ko.Computed<boolean>;
 
   borderWidthPx: ko.Computed<string>;
@@ -367,6 +368,8 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
   // in which case the UI prevents various things like hiding columns or changing the widget type.
   this.isRaw = this.autoDispose(ko.pureComputed(() => this.table().rawViewSectionRef() === this.getRowId()));
 
+  this.isVirtual = this.autoDispose(ko.pureComputed(() => typeof this.id() === 'string'));
+
   this.borderWidthPx = ko.pureComputed(() => this.borderWidth() + 'px');
 
   this.layoutSpecObj = modelUtil.jsonObservable(this.layoutSpec);