(core) support for bundling custom widgets with the Grist app

Summary:
This adds support for bundling custom widgets with the Grist app, as follows:

 * Adds a new `widgets` component to plugins mechanism.
 * When a set of widgets is provided in a plugin, the html/js/css assets for those widgets are served on the existing untrusted user content port.
 * Any bundled `grist-plugin-api.js` will be served with the Grist app's own version of that file. It is important that bundled widgets not refer to https://docs.getgrist.com for the plugin js, since they must be capable of working offline.
 * The logic for configuring that port is updated a bit.
 * I removed the CustomAttachedView class in favor of applying settings of bundled custom widgets more directly, without modification on view.

Any Grist installation via docker will need an extra step now, since there is an extra port that needs exposing for full functionality. I did add a `GRIST_TRUST_PLUGINS` option for anyone who really doesn't want to do this, and would prefer to trust the plugins and have them served on the same port.

Actually making use of bundling will be another step. It'll be important to mesh it with our SaaS's use of APP_STATIC_URL for serving most static assets.

Design sketch: https://grist.quip.com/bJlWACWzr2R9/Bundled-custom-widgets

Test Plan: added a test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4069

test/nbrowser/AttachedCustomWidget.ts

@@ -16,13 +16,20 @@ describe('AttachedCustomWidget', function () {
   // Valid widget url.
   const widgetEndpoint = '/widget';
   // Create some widgets:
-  const widget1: ICustomWidget = {widgetId: '1', name: 'Calendar', url: widgetEndpoint + '?name=Calendar'};
+  const widget1: ICustomWidget = {
+    widgetId: '@gristlabs/widget-calendar',
+    name: 'Calendar',
+    url: widgetEndpoint + '?name=Calendar',
+  };
   let widgetServerUrl = '';
   // Holds widgets manifest content.
   let widgets: ICustomWidget[] = [];
   // Switches widget manifest url
-  function useManifest(url: string) {
-    return server.testingHooks.setWidgetRepositoryUrl(url ? `${widgetServerUrl}${url}` : '');
+  async function useManifest(url: string) {
+    await server.testingHooks.setWidgetRepositoryUrl(url ? `${widgetServerUrl}${url}` : '');
+    await driver.executeAsyncScript(
+      (done: any) => (window as any).gristApp?.topAppModel.testReloadWidgets().then(done).catch(done) || done()
+    );
   }
 
   async function buildWidgetServer(){
@@ -77,8 +84,8 @@ describe('AttachedCustomWidget', function () {
 
   it('should be able to attach Calendar Widget', async () => {
     await gu.openAddWidgetToPage();
-    const notepadElement = await driver.findContent('.test-wselect-type', /Calendar/);
-    assert.exists(notepadElement, 'Calendar widget is not found in the list of widgets');
+    const calendarElement = await driver.findContent('.test-wselect-type', /Calendar/);
+    assert.exists(calendarElement, 'Calendar widget is not found in the list of widgets');
   });
 
   it('should not ask for permission', async () => {

test/nbrowser/CustomWidgets.ts

@@ -1,13 +1,16 @@
-import {assert, driver, Key} from 'mocha-webdriver';
-import * as gu from 'test/nbrowser/gristUtils';
-import {server, setupTestSuite} from 'test/nbrowser/testUtils';
-import {serveSomething} from 'test/server/customUtil';
 import {AccessLevel, ICustomWidget} from 'app/common/CustomWidget';
 import {AccessTokenResult} from 'app/plugin/GristAPI';
 import {TableOperations} from 'app/plugin/TableOperations';
 import {getAppRoot} from 'app/server/lib/places';
+import * as fse from 'fs-extra';
+import {assert, driver, Key} from 'mocha-webdriver';
 import fetch from 'node-fetch';
 import * as path from 'path';
+import * as gu from 'test/nbrowser/gristUtils';
+import {server, setupTestSuite} from 'test/nbrowser/testUtils';
+import {serveSomething} from 'test/server/customUtil';
+import {createTmpDir} from 'test/server/docTools';
+import {EnvironmentSnapshot} from 'test/server/testUtils';
 
 // Valid manifest url.
 const manifestEndpoint = '/manifest.json';
@@ -45,14 +48,18 @@ function getCustomWidgetFrame() {
 
 describe('CustomWidgets', function () {
   this.timeout(20000);
+  gu.bigScreen();
   const cleanup = setupTestSuite();
 
   // Holds url for sample widget server.
   let widgetServerUrl = '';
 
   // Switches widget manifest url
-  function useManifest(url: string) {
-    return server.testingHooks.setWidgetRepositoryUrl(url ? `${widgetServerUrl}${url}` : '');
+  async function useManifest(url: string) {
+    await server.testingHooks.setWidgetRepositoryUrl(url ? `${widgetServerUrl}${url}` : '');
+    await driver.executeAsyncScript(
+      (done: any) => (window as any).gristApp?.topAppModel.testReloadWidgets().then(done).catch(done) || done()
+    );
   }
 
 
@@ -113,14 +120,6 @@ describe('CustomWidgets', function () {
     await server.testingHooks.setWidgetRepositoryUrl('');
   });
 
-  after(async function() {
-    await server.testingHooks.setWidgetRepositoryUrl('');
-  });
-
-  after(async function() {
-    await server.testingHooks.setWidgetRepositoryUrl('');
-  });
-
   // Open or close widget menu.
   const toggle = async () => await driver.findWait('.test-config-widget-select .test-select-open', 1000).click();
   // Get current value from widget menu.
@@ -215,16 +214,17 @@ describe('CustomWidgets', function () {
   // Rejects new access level.
   const reject = () => driver.find(".test-config-widget-access-reject").click();
 
+  async function enableWidgetsAndShowPanel() {
+    // Override gristConfig to enable widget list.
+    await driver.executeScript('window.gristConfig.enableWidgetRepository = true;');
+    // We need to be sure that widget configuration panel is open all the time.
+    await gu.toggleSidePanel('right', 'open');
+    await recreatePanel();
+    await driver.findWait('.test-right-tab-pagewidget', 100).click();
+  }
 
   describe('RightWidgetMenu', () => {
-    beforeEach(async function () {
-      // Override gristConfig to enable widget list.
-      await driver.executeScript('window.gristConfig.enableWidgetRepository = true;');
-      // We need to be sure that widget configuration panel is open all the time.
-      await gu.toggleSidePanel('right', 'open');
-      await recreatePanel();
-      await driver.findWait('.test-right-tab-pagewidget', 100).click();
-    });
+    beforeEach(enableWidgetsAndShowPanel);
 
     it('should show widgets in dropdown', async () => {
       await gu.toggleSidePanel('right', 'open');
@@ -364,7 +364,12 @@ describe('CustomWidgets', function () {
       await recreatePanel();
     });
 
-    it('should show widget when it was removed from list', async () => {
+    /**
+     * Need to think about whether this is desirable?
+     * The document could be on a different Grist installation to the
+     * one where it was created.
+     */
+    it.skip('should show widget when it was removed from list', async () => {
       // Select widget1 and then remove it from the list.
       await toggle();
       await select(widget1.name);
@@ -721,4 +726,154 @@ describe('CustomWidgets', function () {
       });
     });
   });
+
+  describe('Bundling', function () {
+    let oldEnv: EnvironmentSnapshot;
+
+    before(async function () {
+      oldEnv = new EnvironmentSnapshot();
+    });
+
+    after(async function() {
+      oldEnv.restore();
+      await server.restart();
+    });
+
+    it('can add widgets via plugins', async function () {
+      // Double-check that using one external widget, we see
+      // just that widget listed.
+      widgets = [widget1];
+      await useManifest(manifestEndpoint);
+      await enableWidgetsAndShowPanel();
+      await toggle();
+      assert.deepEqual(await options(), [
+        CUSTOM_URL, widget1.name,
+      ]);
+
+      // Get a temporary directory that will be cleaned up,
+      // and populated it as follows:
+      //   plugins/
+      //     my-widgets/
+      //       manifest.yml   # a plugin manifest, listing widgets.json
+      //       widgets.json   # a widget set manifest, grist-widget style
+      //       p1.html        # one of the widgets in widgets.json
+      //       p2.html        # another of the widgets in widgets.json
+      //       grist-plugin-api.js   # a dummy api file, to check it is overridden
+      const dir = await createTmpDir();
+      const pluginDir = path.join(dir, 'plugins', 'my-widgets');
+      await fse.mkdirp(pluginDir);
+
+      // A plugin, with some widgets in it.
+      await fse.writeFile(path.join(pluginDir, 'manifest.yml'), `
+        name: My Widgets
+        components:
+          widgets: widgets.json
+      `);
+
+      // A list of a pair of custom widgets, with the widget
+      // source in the same directory.
+      await fse.writeFile(
+        path.join(pluginDir, 'widgets.json'),
+        JSON.stringify([
+          {
+            widgetId: 'p1',
+            name: 'P1',
+            url: './p1.html',
+          },
+          {
+            widgetId: 'p2',
+            name: 'P2',
+            url: './p2.html',
+          },
+        ]),
+      );
+
+      // The first widget - just contains the text P1.
+      await fse.writeFile(
+        path.join(pluginDir, 'p1.html'),
+        '<html><body>P1</body></html>',
+      );
+
+      // The second widget. This contains the text P2
+      // if grist is defined after loading grist-plugin-api.js
+      // (but the js bundled with the widget just throws an
+      // alert).
+      await fse.writeFile(
+        path.join(pluginDir, 'p2.html'),
+        `
+        <html>
+        <script src="./grist-plugin-api.js"></script>
+        <body>
+          <div id="readout"></div>
+          <script>
+             if (typeof grist !== 'undefined') {
+               document.getElementById('readout').innerText = 'P2';
+             }
+          </script>
+        </body>
+        </html>
+        `
+      );
+      // A dummy grist-plugin-api.js - hopefully the actual
+      // js for the current version of Grist will be served in
+      // its place.
+      await fse.writeFile(
+        path.join(pluginDir, 'grist-plugin-api.js'),
+        'alert("Error: built in api version used");',
+      );
+
+      // Restart server and reload doc now plugins are in place.
+      process.env.GRIST_USER_ROOT = dir;
+      await server.restart();
+      await gu.reloadDoc();
+
+      // Continue using one external widget.
+      await useManifest(manifestEndpoint);
+      await enableWidgetsAndShowPanel();
+
+      // Check we see one external widget and two bundled ones.
+      await toggle();
+      assert.deepEqual(await options(), [
+        CUSTOM_URL, widget1.name, 'P1 (My Widgets)', 'P2 (My Widgets)',
+      ]);
+
+      // Prepare to check content of widgets.
+      async function getWidgetText(): Promise<string> {
+        return gu.doInIframe(await getCustomWidgetFrame(), () => {
+          return driver.executeScript(
+            () => document.body.innerText
+          );
+        });
+      }
+
+      // Check built-in P1 works as expected.
+      await select(/P1/);
+      assert.equal(await current(), 'P1 (My Widgets)');
+      await gu.waitToPass(async () => {
+        assert.equal(await getWidgetText(), 'P1');
+      });
+
+      // Check external W1 works as expected.
+      await toggle();
+      await select(/W1/);
+      assert.equal(await current(), 'W1');
+      await gu.waitToPass(async () => {
+        assert.equal(await getWidgetText(), 'W1');
+      });
+
+      // Check build-in P2 works as expected.
+      await toggle();
+      await select(/P2/);
+      assert.equal(await current(), 'P2 (My Widgets)');
+      await gu.waitToPass(async () => {
+        assert.equal(await getWidgetText(), 'P2');
+      });
+
+      // Make sure widget setting is sticky.
+      await gu.reloadDoc();
+      await gu.waitToPass(async () => {
+        assert.equal(await getWidgetText(), 'P2');
+      });
+    });
+  });
 });

test/nbrowser/CustomWidgetsConfig.ts

@@ -340,6 +340,10 @@ describe('CustomWidgetsConfig', function () {
   });
 
   it('should hide mappings when there is no good column', async () => {
+    if ((await currentWidget()) !== CUSTOM_URL) {
+      await toggleWidgetMenu();
+      await clickOption(CUSTOM_URL);
+    }
     await gu.setWidgetUrl(
       createConfigUrl({
         columns: [{name: 'M2', type: 'Date'}],
@@ -392,6 +396,10 @@ describe('CustomWidgetsConfig', function () {
 
   it('should clear optional mapping', async () => {
     const revert = await gu.begin();
+    if ((await currentWidget()) !== CUSTOM_URL) {
+      await toggleWidgetMenu();
+      await clickOption(CUSTOM_URL);
+    }
     await gu.setWidgetUrl(
       createConfigUrl({
         columns: [{name: 'M2', type: 'Date', optional: true}],

test/nbrowser/gristUtils.ts

@@ -3271,6 +3271,27 @@ export async function changeWidgetAccess(access: 'read table'|'full'|'none') {
   }
 }
 
+
+/**
+ * Recently, driver.switchTo().window() has become a little flakey,
+ * methods may fail if called immediately after switching to a
+ * window. This method works around the problem by waiting for
+ * driver.getCurrentUrl to succeed.
+ *  https://github.com/SeleniumHQ/selenium/issues/12277
+ */
+export async function switchToWindow(target: string) {
+  await driver.switchTo().window(target);
+  for (let i = 0; i < 10; i++) {
+    try {
+      await driver.getCurrentUrl();
+      break;
+    } catch (e) {
+      console.log("switchToWindow retry after error:", e);
+      await driver.sleep(250);
+    }
+  }
+}
+
 /*
  * Returns an instance of `LockableClipboard`, making sure to unlock it after
  * each test.