(core) support for bundling custom widgets with the Grist app

Summary:
This adds support for bundling custom widgets with the Grist app, as follows:

 * Adds a new `widgets` component to plugins mechanism.
 * When a set of widgets is provided in a plugin, the html/js/css assets for those widgets are served on the existing untrusted user content port.
 * Any bundled `grist-plugin-api.js` will be served with the Grist app's own version of that file. It is important that bundled widgets not refer to https://docs.getgrist.com for the plugin js, since they must be capable of working offline.
 * The logic for configuring that port is updated a bit.
 * I removed the CustomAttachedView class in favor of applying settings of bundled custom widgets more directly, without modification on view.

Any Grist installation via docker will need an extra step now, since there is an extra port that needs exposing for full functionality. I did add a `GRIST_TRUST_PLUGINS` option for anyone who really doesn't want to do this, and would prefer to trust the plugins and have them served on the same port.

Actually making use of bundling will be another step. It'll be important to mesh it with our SaaS's use of APP_STATIC_URL for serving most static assets.

Design sketch: https://grist.quip.com/bJlWACWzr2R9/Bundled-custom-widgets

Test Plan: added a test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4069

app/client/components/CustomCalendarView.ts

@@ -1,46 +1,11 @@
-import {AccessLevel} from "app/common/CustomWidget";
-import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
-import {CustomView} from "app/client/components/CustomView";
-import {GristDoc} from "app/client/components/GristDoc";
-import {reportError} from 'app/client/models/errors';
+import { CustomView, CustomViewSettings } from "app/client/components/CustomView";
+import { AccessLevel } from "app/common/CustomWidget";
 
-//Abstract class for more future inheritances
-abstract class CustomAttachedView extends CustomView {
-  public override create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) {
-    super.create(gristDoc, viewSectionModel);
-    if (viewSectionModel.customDef.access.peek() !== AccessLevel.full) {
-      void viewSectionModel.customDef.access.setAndSave(AccessLevel.full).catch((err)=>{
-        if (err?.code === "ACL_DENY") {
-          // do nothing, we might be in a readonly mode.
-          return;
-        }
-        reportError(err);
-      });
-    }
-
-    const widgetsApi = this.gristDoc.app.topAppModel.api;
-    widgetsApi.getWidgets().then(async result=>{
-      const widget = result.find(w=>w.name == this.getWidgetName());
-      if (widget && this.customDef.url.peek() !== widget.url) {
-        await this.customDef.url.setAndSave(widget.url);
-      }
-    }).catch((err)=>{
-      if (err?.code !== "ACL_DENY") {
-        // TODO: revisit it later. getWidgets() is async call, and non of the code
-        // above is checking if we are still alive.
-        console.error(err);
-      } else {
-        // do nothing, we might be in a readonly mode.
-      }
-    });
-  }
-
-  protected abstract getWidgetName(): string;
-
-}
-
-export class CustomCalendarView extends CustomAttachedView {
-  protected getWidgetName(): string {
-    return "Calendar";
+export class CustomCalendarView extends CustomView {
+  protected getBuiltInSettings(): CustomViewSettings {
+    return {
+      widgetId: '@gristlabs/widget-calendar',
+      accessLevel: AccessLevel.full,
+    };
   }
 }

app/client/components/CustomView.ts

@@ -34,6 +34,17 @@ import {dom as grains} from 'grainjs';
 import * as ko from 'knockout';
 import defaults = require('lodash/defaults');
 
+/**
+ *
+ * Built in settings for a custom widget. Used when the custom
+ * widget is the implementation of a native-looking widget,
+ * for example the calendar widget.
+ *
+ */
+export interface CustomViewSettings {
+  widgetId?: string;
+  accessLevel?: AccessLevel;
+}
 
 /**
  * CustomView components displays arbitrary html. There are two modes available, in the "url" mode
@@ -83,11 +94,10 @@ export class CustomView extends Disposable {
 
   private _frame: WidgetFrame;  // plugin frame (holding external page)
 
-
   public create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) {
     BaseView.call(this as any, gristDoc, viewSectionModel, { 'addNewRow': true });
 
-    this.customDef =  this.viewSection.customDef;
+    this.customDef = this.viewSection.customDef;
 
     this.autoDisposeCallback(() => {
       if (this._customSection) {
@@ -107,16 +117,20 @@ export class CustomView extends Disposable {
     this._updatePluginInstance();
   }
 
-
   public async triggerPrint() {
     if (!this.isDisposed() && this._frame) {
       return await this._frame.callRemote('print');
     }
   }
 
+  protected getBuiltInSettings(): CustomViewSettings {
+    return {};
+  }
+
   protected getEmptyWidgetPage(): string {
     return new URL("custom-widget.html", getGristConfig().homeUrl!).href;
   }
+
   /**
    * Find a plugin instance that matches the plugin id, update the `found` observables, then tries to
    * find a matching section.
@@ -156,13 +170,13 @@ export class CustomView extends Disposable {
   }
 
   private _buildDom() {
-    const {mode, url, access, renderAfterReady} = this.customDef;
+    const {mode, url, access, renderAfterReady, widgetDef, widgetId, pluginId} = this.customDef;
     const showPlugin = ko.pureComputed(() => this.customDef.mode() === "plugin");
     const showAfterReady = () => {
       // The empty widget page calls `grist.ready()`.
-      if (!url()) { return true; }
+      if (!url() && !widgetId()) { return true; }
 
-      return this.customDef.widgetDef()?.renderAfterReady ?? renderAfterReady();
+      return renderAfterReady();
     };
 
     // When both plugin and section are not found, let's show only plugin notification.
@@ -172,18 +186,27 @@ export class CustomView extends Disposable {
         // For the view to update when switching from one section to another one, the computed
         // observable must always notify.
         .extend({notify: 'always'});
+    // Some widgets have built-in settings that should override anything
+    // that is in the rest of the view options. Ideally, everything would
+    // be consistent. We could fix inconsistencies if we find them, but
+    // we are not guaranteed to have write privileges at this point.
+    const builtInSettings = this.getBuiltInSettings();
     return dom('div.flexauto.flexvbox.custom_view_container',
       dom.autoDispose(showPlugin),
       dom.autoDispose(showPluginNotification),
       dom.autoDispose(showSectionNotification),
       dom.autoDispose(showPluginContent),
       // todo: should display content in webview when running electron
-      kd.scope(() => [mode(), url(), access()], ([_mode, _url, _access]: string[]) =>
+      // prefer widgetId; spelunk in widgetDef for older docs
+      kd.scope(() => [mode(), url(), access(), widgetId() || widgetDef()?.widgetId || '', pluginId()],
+               ([_mode, _url, _access, _widgetId, _pluginId]: string[]) =>
         _mode === "url" ?
           this._buildIFrame({
             baseUrl: _url,
-            access: (_access as AccessLevel || AccessLevel.none),
+            access: builtInSettings.accessLevel || (_access as AccessLevel || AccessLevel.none),
             showAfterReady: showAfterReady(),
+            widgetId: builtInSettings.widgetId || _widgetId,
+            pluginId: _pluginId,
           })
           : null
       ),
@@ -213,12 +236,16 @@ export class CustomView extends Disposable {
     baseUrl: string|null,
     access: AccessLevel,
     showAfterReady?: boolean,
+    widgetId?: string|null,
+    pluginId?: string
   }) {
-    const {baseUrl, access, showAfterReady} = options;
+    const {baseUrl, access, showAfterReady, widgetId, pluginId} = options;
     const documentSettings = this.gristDoc.docData.docSettings();
     const readonly = this.gristDoc.isReadonly.get();
     return grains.create(WidgetFrame, {
       url: baseUrl || this.getEmptyWidgetPage(),
+      widgetId,
+      pluginId,
       access,
       preferences:
       {
@@ -273,7 +300,8 @@ export class CustomView extends Disposable {
         }
         // allow menus to close if any
         closeRegisteredMenu();
-      })
+      }),
+      gristDoc: this.gristDoc,
     });
 
   }

app/client/components/WidgetFrame.ts

@@ -6,7 +6,8 @@ import {hooks} from 'app/client/Hooks';
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {makeTestId} from 'app/client/lib/domUtils';
 import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel';
-import {AccessLevel, isSatisfied} from 'app/common/CustomWidget';
+import {reportError} from 'app/client/models/errors';
+import {AccessLevel, ICustomWidget, isSatisfied, matchWidget} from 'app/common/CustomWidget';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
 import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions';
 import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes';
@@ -45,6 +46,15 @@ export interface WidgetFrameOptions {
    * Url of external page. Iframe is rebuild each time the URL changes.
    */
   url: string;
+  /**
+   * ID of widget, if known. When set, the url for the specified widget
+   * in the WidgetRepository, if found, will take precedence.
+   */
+  widgetId?: string|null;
+  /**
+   * ID of the plugin that provided the widget (if it came from a plugin).
+   */
+  pluginId?: string;
   /**
    * Assigned access level. Iframe is rebuild each time access level is changed.
    */
@@ -72,6 +82,10 @@ export interface WidgetFrameOptions {
    * Optional language to use for the widget.
    */
   preferences: {language?: string, timeZone?: any, currency?: string, culture?: string};
+  /**
+   * The containing document.
+   */
+  gristDoc: GristDoc;
 }
 
 /**
@@ -86,6 +100,7 @@ export class WidgetFrame extends DisposableWithEvents {
   private _readyCalled = Observable.create(this, false);
   // Whether the iframe is visible.
   private _visible = Observable.create(this, !this._options.showAfterReady);
+  private readonly _widget = Observable.create<ICustomWidget|null>(this, null);
 
   constructor(private _options: WidgetFrameOptions) {
     super();
@@ -112,7 +127,10 @@ export class WidgetFrame extends DisposableWithEvents {
 
     // Call custom configuration handler.
     _options.configure?.(this);
+
+    this._checkWidgetRepository().catch(reportError);
   }
+
   /**
    * Attach an EventSource with desired access level.
    */
@@ -166,6 +184,21 @@ export class WidgetFrame extends DisposableWithEvents {
   }
 
   public buildDom() {
+    const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el);
+    return onElem(
+      (this._iframe = dom(
+        'iframe',
+        dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'),
+        dom.cls('clipboard_focus'),
+        dom.cls('custom_view'),
+        dom.attr('src', use => this._getUrl(use(this._widget))),
+        hooks.iframeAttributes,
+        testId('ready', this._readyCalled),
+      ))
+    );
+  }
+
+  private _getUrl(widget: ICustomWidget|null): string {
     // Append access level to query string.
     const urlWithAccess = (url: string) => {
       if (!url) {
@@ -175,23 +208,12 @@ export class WidgetFrame extends DisposableWithEvents {
       urlObj.searchParams.append('access', this._options.access);
       urlObj.searchParams.append('readonly', String(this._options.readonly));
       // Append user and document preferences to query string.
-       const settingsParams = new URLSearchParams(this._options.preferences);
-       settingsParams.forEach((value, key) => urlObj.searchParams.append(key, value));
+      const settingsParams = new URLSearchParams(this._options.preferences);
+      settingsParams.forEach((value, key) => urlObj.searchParams.append(key, value));
       return urlObj.href;
     };
-    const fullUrl = urlWithAccess(this._options.url);
-    const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el);
-    return onElem(
-      (this._iframe = dom('iframe',
-        dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'),
-        dom.cls('clipboard_focus'),
-        dom.cls('custom_view'), {
-          src: fullUrl,
-          ...hooks.iframeAttributes,
-        },
-        testId('ready', this._readyCalled),
-      ))
-    );
+    const url = widget?.url || this._options.url || 'about:blank';
+    return urlWithAccess(url);
   }
 
   private _onMessage(event: MessageEvent) {
@@ -217,6 +239,19 @@ export class WidgetFrame extends DisposableWithEvents {
       this._rpc.receiveMessage(event.data);
     }
   }
+
+  /**
+   * If we have a widgetId, look it up in the WidgetRepository and
+   * get the best URL we can for it.
+   */
+  private async _checkWidgetRepository() {
+    const {widgetId, pluginId} = this._options;
+    if (this.isDisposed() || !widgetId) { return; }
+    const widgets = await this._options.gristDoc.app.topAppModel.getWidgets();
+    if (this.isDisposed()) { return; }
+    const widget = matchWidget(widgets, {widgetId, pluginId});
+    this._widget.set(widget || null);
+  }
 }
 
 const throwError = (access: AccessLevel) => {