bundling experiments (WIP)

Looking at ways to bundle custom widgets with Grist. WIP,
experimental, everything will need rewrite.

app/client/components/CustomCalendarView.ts

@@ -1,11 +1,13 @@
-import {AccessLevel} from "app/common/CustomWidget";
+// import {AccessLevel} from "app/common/CustomWidget";
-import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
+// import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
-import {CustomView} from "app/client/components/CustomView";
+import { CustomView, CustomViewSettings } from "app/client/components/CustomView";
-import {GristDoc} from "app/client/components/GristDoc";
+import { AccessLevel } from "app/common/CustomWidget";
-import {reportError} from 'app/client/models/errors';
+// import {GristDoc} from "app/client/components/GristDoc";
+// import {reportError} from 'app/client/models/errors';
 
 //Abstract class for more future inheritances
-abstract class CustomAttachedView extends CustomView {
+// abstract class CustomAttachedView extends CustomView {
+  /*
   public override create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) {
     super.create(gristDoc, viewSectionModel);
     if (viewSectionModel.customDef.access.peek() !== AccessLevel.full) {
@@ -18,7 +20,7 @@ abstract class CustomAttachedView extends CustomView {
       });
     }
 
-    const widgetsApi = this.gristDoc.app.topAppModel.api;
+    const widgetsApi = this.gristDoc.app.topAppModel;
     widgetsApi.getWidgets().then(async result=>{
       const widget = result.find(w=>w.name == this.getWidgetName());
       if (widget && this.customDef.url.peek() !== widget.url) {
@@ -34,13 +36,17 @@ abstract class CustomAttachedView extends CustomView {
       }
     });
   }
+  */
 
-  protected abstract getWidgetName(): string;
+//  protected abstract getWidgetName(): string;
 
-}
+// }
 
-export class CustomCalendarView extends CustomAttachedView {
+export class CustomCalendarView extends CustomView {
-  protected getWidgetName(): string {
+  protected getInitialSettings(): CustomViewSettings {
-    return "Calendar";
+    return {
+      widgetId: '@gristlabs/widget-calendar',
+      accessLevel: AccessLevel.full,
+    };
   }
 }

app/client/components/CustomView.ts

@@ -32,6 +32,10 @@ import {dom as grains} from 'grainjs';
 import * as ko from 'knockout';
 import defaults = require('lodash/defaults');
 
+export interface CustomViewSettings {
+  widgetId?: string;
+  accessLevel?: AccessLevel;
+}
 
 /**
  * CustomView components displays arbitrary html. There are two modes available, in the "url" mode
@@ -81,11 +85,10 @@ export class CustomView extends Disposable {
 
   private _frame: WidgetFrame;  // plugin frame (holding external page)
 
-
   public create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) {
     BaseView.call(this as any, gristDoc, viewSectionModel, { 'addNewRow': true });
 
-    this.customDef =  this.viewSection.customDef;
+    this.customDef = this.viewSection.customDef;
 
     this.autoDisposeCallback(() => {
       if (this._customSection) {
@@ -103,8 +106,43 @@ export class CustomView extends Disposable {
 
     this.viewPane = this.autoDispose(this._buildDom());
     this._updatePluginInstance();
+
+    this.dealWithBundledWidgets(gristDoc, viewSectionModel);
   }
 
+  public dealWithBundledWidgets(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) {
+    const settings = this.getInitialSettings();
+    console.log("dealWith!", {settings});
+    if (!settings.widgetId) { return; }
+    if (viewSectionModel.customDef.access.peek() !== AccessLevel.full) {
+      void viewSectionModel.customDef.access.setAndSave(AccessLevel.full).catch((err)=>{
+        if (err?.code === "ACL_DENY") {
+          // do nothing, we might be in a readonly mode.
+          return;
+        }
+        reportError(err);
+      });
+    }
+    
+    const widgetsApi = this.gristDoc.app.topAppModel;
+    widgetsApi.getWidgets().then(async result=>{
+      const widget = result.find(w => w.widgetId === settings.widgetId);
+      console.log("FOUND", {widget});
+      if (widget && this.customDef.widgetId.peek() !== widget.widgetId) {
+        console.log("SET!!");
+        await this.customDef.widgetId.setAndSave(widget.widgetId);
+        await this.customDef.pluginId.setAndSave(widget.fromPlugin||'');
+      }
+    }).catch((err)=>{
+      if (err?.code !== "ACL_DENY") {
+        // TODO: revisit it later. getWidgets() is async call, and non of the code
+        // above is checking if we are still alive.
+        console.error(err);
+      } else {
+        // do nothing, we might be in a readonly mode.
+      }
+    });
+  }
 
   public async triggerPrint() {
     if (!this.isDisposed() && this._frame) {
@@ -112,9 +150,14 @@ export class CustomView extends Disposable {
     }
   }
 
+  protected getInitialSettings(): CustomViewSettings {
+    return {};
+  }
+  
   protected getEmptyWidgetPage(): string {
     return new URL("custom-widget.html", getGristConfig().homeUrl!).href;
   }
+
   /**
    * Find a plugin instance that matches the plugin id, update the `found` observables, then tries to
    * find a matching section.
@@ -154,13 +197,16 @@ export class CustomView extends Disposable {
   }
 
   private _buildDom() {
-    const {mode, url, access, renderAfterReady} = this.customDef;
+    const {mode, url, access, renderAfterReady, widgetId, pluginId} = this.customDef;
     const showPlugin = ko.pureComputed(() => this.customDef.mode() === "plugin");
     const showAfterReady = () => {
       // The empty widget page calls `grist.ready()`.
+      // Pending: URLs set now only when user actually enters a URL,
+      // so this could be breaking pages without grist.ready() call
+      // added to manifests.
       if (!url()) { return true; }
 
-      return this.customDef.widgetDef()?.renderAfterReady ?? renderAfterReady();
+      return renderAfterReady();
     };
 
     // When both plugin and section are not found, let's show only plugin notification.
@@ -176,12 +222,14 @@ export class CustomView extends Disposable {
       dom.autoDispose(showSectionNotification),
       dom.autoDispose(showPluginContent),
       // todo: should display content in webview when running electron
-      kd.scope(() => [mode(), url(), access()], ([_mode, _url, _access]: string[]) =>
+      kd.scope(() => [mode(), url(), access(), widgetId(), pluginId()], ([_mode, _url, _access, _widgetId, _pluginId]: string[]) =>
         _mode === "url" ?
           this._buildIFrame({
             baseUrl: _url,
             access: (_access as AccessLevel || AccessLevel.none),
             showAfterReady: showAfterReady(),
+            widgetId: _widgetId,
+            pluginId: _pluginId,
           })
           : null
       ),
@@ -211,10 +259,15 @@ export class CustomView extends Disposable {
     baseUrl: string|null,
     access: AccessLevel,
     showAfterReady?: boolean,
+    widgetId?: string|null,
+    pluginId?: string,
   }) {
-    const {baseUrl, access, showAfterReady} = options;
+    const {baseUrl, access, showAfterReady, widgetId, pluginId} = options;
     return grains.create(WidgetFrame, {
       url: baseUrl || this.getEmptyWidgetPage(),
+      widgetId,
+      pluginId,
+      emptyUrl: this.getEmptyWidgetPage(),
       access,
       readonly: this.gristDoc.isReadonly.get(),
       showAfterReady,
@@ -265,7 +318,8 @@ export class CustomView extends Disposable {
         }
         // allow menus to close if any
         closeRegisteredMenu();
-      })
+      }),
+      gristDoc: this.gristDoc,
     });
 
   }

app/client/components/WidgetFrame.ts

@@ -6,7 +6,7 @@ import {hooks} from 'app/client/Hooks';
 import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals';
 import {makeTestId} from 'app/client/lib/domUtils';
 import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel';
-import {AccessLevel, isSatisfied} from 'app/common/CustomWidget';
+import {AccessLevel, ICustomWidget, isSatisfied, matchWidget } from 'app/common/CustomWidget';
 import {DisposableWithEvents} from 'app/common/DisposableWithEvents';
 import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions';
 import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes';
@@ -19,6 +19,7 @@ import noop = require('lodash/noop');
 import debounce = require('lodash/debounce');
 import isEqual = require('lodash/isEqual');
 import flatMap = require('lodash/flatMap');
+import { reportError } from '../models/errors';
 
 const testId = makeTestId('test-custom-widget-');
 
@@ -43,6 +44,9 @@ export interface WidgetFrameOptions {
    * Url of external page. Iframe is rebuild each time the URL changes.
    */
   url: string;
+  widgetId?: string|null;
+  pluginId?: string;
+  emptyUrl: string;
   /**
    * Assigned access level. Iframe is rebuild each time access level is changed.
    */
@@ -73,6 +77,8 @@ export interface WidgetFrameOptions {
    * Optional handler to modify the iframe.
    */
   onElem?: (iframe: HTMLIFrameElement) => void;
+
+  gristDoc: GristDoc;
 }
 
 /**
@@ -87,6 +93,7 @@ export class WidgetFrame extends DisposableWithEvents {
   private _readyCalled = Observable.create(this, false);
   // Whether the iframe is visible.
   private _visible = Observable.create(this, !this._options.showAfterReady);
+  public readonly _widgets = Observable.create<ICustomWidget[]>(this, []);
 
   constructor(private _options: WidgetFrameOptions) {
     super();
@@ -113,7 +120,10 @@ export class WidgetFrame extends DisposableWithEvents {
 
     // Call custom configuration handler.
     _options.configure?.(this);
+
+    this._fetchWidgets().catch(reportError);
   }
+
   /**
    * Attach an EventSource with desired access level.
    */
@@ -167,6 +177,23 @@ export class WidgetFrame extends DisposableWithEvents {
   }
 
   public buildDom() {
+    const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el);
+    return onElem(
+      (this._iframe = dom(
+        'iframe',
+        dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'),
+        dom.cls('clipboard_focus'),
+        dom.cls('custom_view'),
+        dom.attr('src', use => this._getUrl(use(this._widgets))),
+        {
+          ...hooks.iframeAttributes,
+        },
+        testId('ready', this._readyCalled),
+      ))
+    );
+  }
+
+  private _getUrl(widgets: ICustomWidget[]): string {
     // Append access level to query string.
     const urlWithAccess = (url: string) => {
       if (!url) {
@@ -177,19 +204,20 @@ export class WidgetFrame extends DisposableWithEvents {
       urlObj.searchParams.append('readonly', String(this._options.readonly));
       return urlObj.href;
     };
-    const fullUrl = urlWithAccess(this._options.url);
+    const {widgetId, pluginId} = this._options;
-    const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el);
+    let url = this._options.url;
-    return onElem(
+    if (widgetId) {
-      (this._iframe = dom('iframe',
+      console.log("Iframe match starting");
-        dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'),
+      const widget = matchWidget(widgets, {widgetId, pluginId});
-        dom.cls('clipboard_focus'),
+      console.log("Iframe match done");
-        dom.cls('custom_view'), {
+      if (widget) {
-          src: fullUrl,
+        url = widget.url;
-          ...hooks.iframeAttributes,
+      } else {
-        },
+        return 'about:blank';
-        testId('ready', this._readyCalled),
+      }
-      ))
+    }
-    );
+    const fullUrl = urlWithAccess(url);
+    return fullUrl;
   }
 
   private _onMessage(event: MessageEvent) {
@@ -216,6 +244,14 @@ export class WidgetFrame extends DisposableWithEvents {
       this._rpc.receiveMessage(event.data);
     }
   }
+
+  private async _fetchWidgets() {
+    if (this.isDisposed()) { return; }
+    const widgets = await this._options.gristDoc.app.topAppModel.getWidgets();
+    if (this.isDisposed()) { return; }
+    this._widgets.set(widgets);
+    console.log("SAVED", {widgets});
+  }
 }
 
 const throwError = (access: AccessLevel) => {

app/client/models/AppModel.ts

@@ -27,6 +27,8 @@ import {getOrgName, isTemplatesOrg, Organization, OrgError, UserAPI, UserAPIImpl
 import {getUserPrefObs, getUserPrefsObs, markAsSeen, markAsUnSeen} from 'app/client/models/UserPrefs';
 import {bundleChanges, Computed, Disposable, Observable, subscribe} from 'grainjs';
 import isEqual from 'lodash/isEqual';
+import { ICustomWidget } from 'app/common/CustomWidget';
+import { AsyncCreate } from 'app/common/AsyncCreate';
 
 const t = makeT('AppModel');
 
@@ -75,6 +77,8 @@ export interface TopAppModel {
    * Reloads orgs and accounts for current user.
    */
   fetchUsersAndOrgs(): Promise<void>;
+
+  getWidgets(): Promise<ICustomWidget[]>;
 }
 
 /**
@@ -143,6 +147,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel {
   public readonly users = Observable.create<FullUser[]>(this, []);
   public readonly plugins: LocalPlugin[] = [];
   private readonly _gristConfig?: GristLoadConfig;
+  private readonly _widgets: AsyncCreate<ICustomWidget[]>;
 
   constructor(
     window: {gristConfig?: GristLoadConfig},
@@ -153,6 +158,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel {
     this.isSingleOrg = Boolean(window.gristConfig && window.gristConfig.singleOrg);
     this.productFlavor = getFlavor(window.gristConfig && window.gristConfig.org);
     this._gristConfig = window.gristConfig;
+    this._widgets = new AsyncCreate<ICustomWidget[]>(() => this.api.getWidgets());
 
     // Initially, and on any change to subdomain, call initialize() to get the full Organization
     // and the FullUser to use for it (the user may change when switching orgs).
@@ -175,6 +181,10 @@ export class TopAppModelImpl extends Disposable implements TopAppModel {
     }
   }
 
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    return this._widgets.get();
+  }
+
   public getUntrustedContentOrigin() {
     if (G.window.isRunningUnderElectron) {
       // when loaded within webviews it is safe to serve plugin's content from the same domain

app/client/models/entities/ViewSectionRec.ts

@@ -21,7 +21,7 @@ import {removeRule, RuleOwner} from 'app/client/models/RuleOwner';
 import {LinkConfig} from 'app/client/ui/selectBy';
 import {getWidgetTypes} from "app/client/ui/widgetTypesMap";
 import {FilterColValues} from "app/common/ActiveDocAPI";
-import {AccessLevel, ICustomWidget} from 'app/common/CustomWidget';
+import {AccessLevel} from 'app/common/CustomWidget';
 import {UserAction} from 'app/common/DocActions';
 import {arrayRepeat} from 'app/common/gutil';
 import {Sort} from 'app/common/SortSpec';
@@ -245,10 +245,19 @@ export interface CustomViewSectionDef {
    * The url.
    */
   url: modelUtil.KoSaveableObservable<string|null>;
+  /**
+   * A widgetId, if available. Preferred to url.
+   * For bundled custom widgets, it is important to refer
+   * to them by something other than url, since url will
+   * vary with deployment, and it should be possible to move
+   * documents between deployments if they have compatible
+   * widgets available.
+   */
+  widgetId: modelUtil.KoSaveableObservable<string|null>;
    /**
    * Custom widget information.
    */
-  widgetDef: modelUtil.KoSaveableObservable<ICustomWidget|null>;
+  // widgetDef: modelUtil.KoSaveableObservable<ICustomWidget|null>;
    /**
    * Custom widget options.
    */
@@ -324,7 +333,7 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
   const customViewDefaults = {
     mode: 'url',
     url: null,
-    widgetDef: null,
+    // widgetDef: null,
     access: '',
     pluginId: '',
     sectionId: '',
@@ -336,7 +345,8 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
   this.customDef = {
     mode: customDefObj.prop('mode'),
     url: customDefObj.prop('url'),
-    widgetDef: customDefObj.prop('widgetDef'),
+    widgetId: customDefObj.prop('widgetId'),
+    // widgetDef: customDefObj.prop('widgetDef'),
     widgetOptions: customDefObj.prop('widgetOptions'),
     columnsMapping: customDefObj.prop('columnsMapping'),
     access: customDefObj.prop('access'),

app/client/ui/CustomSectionConfig.ts

@@ -15,7 +15,7 @@ import {IconName} from 'app/client/ui2018/IconList';
 import {icon} from 'app/client/ui2018/icons';
 import {cssLink} from 'app/client/ui2018/links';
 import {IOptionFull, menu, menuItem, menuText, select} from 'app/client/ui2018/menus';
-import {AccessLevel, ICustomWidget, isSatisfied} from 'app/common/CustomWidget';
+import { AccessLevel, ICustomWidget, isSatisfied, matchWidget } from 'app/common/CustomWidget';
 import {GristLoadConfig} from 'app/common/gristUrls';
 import {unwrap} from 'app/common/gutil';
 import {
@@ -322,7 +322,8 @@ export class CustomSectionConfig extends Disposable {
 
     // Test if we can offer widget list.
     const gristConfig: GristLoadConfig = (window as any).gristConfig || {};
-    this._canSelect = gristConfig.enableWidgetRepository ?? true;
+    console.log("Ignoring gristConfig now", {gristConfig});
+    this._canSelect = true; // gristConfig.enableWidgetRepository ?? true;
 
     // Array of available widgets - will be updated asynchronously.
     this._widgets = Observable.create(this, []);
@@ -331,12 +332,16 @@ export class CustomSectionConfig extends Disposable {
 
     // Selected value from the dropdown (contains widgetId or "custom" string for Custom URL)
     this._selectedId = Computed.create(this, use => {
-      if (use(_section.customDef.widgetDef)) {
+      const widgetId = use(_section.customDef.widgetId);
-        return _section.customDef.widgetDef.peek()!.widgetId;
+      const pluginId = use(_section.customDef.pluginId);
+      if (widgetId) {
+        console.log("_selectedId", {widgetId, pluginId});
+        return (pluginId||'') + ':' + widgetId;
       }
       return CUSTOM_ID;
     });
     this._selectedId.onWrite(async value => {
+      console.log("_selectedId onWrite", {value});
       if (value === CUSTOM_ID) {
         // Select Custom URL
         bundleChanges(() => {
@@ -344,8 +349,11 @@ export class CustomSectionConfig extends Disposable {
           _section.customDef.renderAfterReady(false);
           // Clear url.
           _section.customDef.url(null);
+          // Clear widgetId
+          _section.customDef.widgetId(null);
+          _section.customDef.pluginId('');
           // Clear widget definition.
-          _section.customDef.widgetDef(null);
+          // _section.customDef.widgetDef(null);
           // Reset access level to none.
           _section.customDef.access(AccessLevel.none);
           // Clear all saved options.
@@ -359,27 +367,50 @@ export class CustomSectionConfig extends Disposable {
         });
         await _section.saveCustomDef();
       } else {
+        const [pluginId, widgetId] = value?.split(':') || [];
         // Select Widget
-        const selectedWidget = this._widgets.get().find(w => w.widgetId === value);
+        console.log("Start match");
+        const selectedWidget = matchWidget(this._widgets.get(), {
+          widgetId,
+          pluginId,
+        });
+        console.log("Started match");
+        console.log("SETTING", {pluginId, widgetId, selectedWidget});
         if (!selectedWidget) {
           // should not happen
           throw new Error('Error accessing widget from the list');
         }
         // If user selected the same one, do nothing.
-        if (_section.customDef.widgetDef.peek()?.widgetId === value) {
+        if (_section.customDef.widgetId.peek() === widgetId &&
+            _section.customDef.pluginId.peek() === pluginId) {
+          console.log("DO NOTHING", {
+            widgetId,
+            pluginId,
+            owidgetId: _section.customDef.widgetId.peek(),
+            opluginId: _section.customDef.pluginId.peek(),
+          });
           return;
         }
         bundleChanges(() => {
           // Reset whether widget should render after `grist.ready()`.
-          _section.customDef.renderAfterReady(false);
+          _section.customDef.renderAfterReady(selectedWidget.renderAfterReady ?? false);
           // Clear access level
           _section.customDef.access(AccessLevel.none);
           // When widget wants some access, set desired access level.
           this._desiredAccess.set(selectedWidget.accessLevel || AccessLevel.none);
           // Update widget definition.
-          _section.customDef.widgetDef(selectedWidget);
+          // _section.customDef.widgetDef(selectedWidget);
+          // Update widgetId.
+          _section.customDef.widgetId(selectedWidget.widgetId);
+          _section.customDef.pluginId(selectedWidget.fromPlugin || '');
+          console.log({
+            setty: 1,
+            widgetId: selectedWidget.widgetId,
+            pluginId: selectedWidget.fromPlugin || '',
+            selectedWidget
+          });
           // Update widget URL.
-          _section.customDef.url(selectedWidget.url);
+          _section.customDef.url(null);
           // Clear options.
           _section.customDef.widgetOptions(null);
           // Clear has custom configuration.
@@ -389,6 +420,7 @@ export class CustomSectionConfig extends Disposable {
           _section.columnsToMap(null);
         });
         await _section.saveCustomDef();
+        console.log("CustomSectionConfig saved");
       }
     });
 
@@ -398,7 +430,12 @@ export class CustomSectionConfig extends Disposable {
     this._url.onWrite(async newUrl => {
       bundleChanges(() => {
         _section.customDef.renderAfterReady(false);
-        _section.customDef.url(newUrl);
+        if (newUrl) {
+          console.log("ZAP widgetId and pluginId");
+          _section.customDef.widgetId(null);
+          _section.customDef.pluginId('');
+        }
+        //_section.customDef.url(newUrl);
       });
       await _section.saveCustomDef();
     });
@@ -423,6 +460,11 @@ export class CustomSectionConfig extends Disposable {
     const holder = new MultiHolder();
 
     // Show prompt, when desired access level is different from actual one.
+    function makeLabel(widget: ICustomWidget) {
+      if (!widget.fromPlugin) { return widget.name; }
+      const group = widget.fromPlugin.replace('builtIn/', '');
+      return `${widget.name} (${group})`;
+    }
     const prompt = Computed.create(holder, use =>
       use(this._desiredAccess)
       && !isSatisfied(use(this._currentAccess), use(this._desiredAccess)!));
@@ -433,7 +475,9 @@ export class CustomSectionConfig extends Disposable {
     // Options for the select-box (all widgets definitions and Custom URL)
     const options = Computed.create(holder, use => [
       {label: 'Custom URL', value: 'custom'},
-      ...use(this._widgets).map(w => ({label: w.name, value: w.widgetId})),
+      ...use(this._widgets).map(w => ({
+        label: makeLabel(w), value: ((w.fromPlugin||'') + ':' + w.widgetId)
+      })),
     ]);
     function buildPrompt(level: AccessLevel|null) {
       if (!level) {
@@ -469,7 +513,7 @@ export class CustomSectionConfig extends Disposable {
           testId('select')
         )
         : null,
-      dom.maybe(isCustom && this.shouldRenderWidgetSelector(), () => [
+      dom.maybe((use) => use(isCustom) && this.shouldRenderWidgetSelector(), () => [
         cssRow(
           cssTextInput(
             this._url,
@@ -538,17 +582,23 @@ export class CustomSectionConfig extends Disposable {
   }
 
   protected async _getWidgets() {
-    const api = this._gristDoc.app.topAppModel.api;
+    const widgets = await this._gristDoc.app.topAppModel.getWidgets();
-    const wigets = await api.getWidgets();
+    /*
+    const widgets = filterWidgets(widgets1, {
+      keepWidgetIdUnique: true,
+      preferPlugin: false,
+    });
+    */
+    // const wigets = await api.getWidgets();
     // Request for rest of the widgets.
     if (this._canSelect) {
       // From the start we will provide single widget definition
       // that was chosen previously.
-      if (this._section.customDef.widgetDef.peek()) {
+      // if (this._section.customDef.widgetDef.peek()) {
-        wigets.push(this._section.customDef.widgetDef.peek()!);
+      // wigets.push(this._section.customDef.widgetDef.peek()!);
-      }
+      // }
     }
-    this._widgets.set(wigets);
+    this._widgets.set(widgets);
   }
 
 

app/client/ui2018/menus.ts

@@ -220,6 +220,7 @@ export function multiSelect<T>(selectedOptions: MutableObsArray<T>,
       },
       dom.domComputed(selectedOptionsSet, selectedOpts => {
         return dom.forEach(availableOptions, option => {
+          console.log(">>> option", {availableOptions});
           const fullOption = weasel.getOptionFull(option);
           return cssCheckboxLabel(
             cssCheckboxSquare(

app/common/CustomWidget.ts

@@ -1,3 +1,5 @@
+import sortBy = require('lodash/sortBy');
+
 /**
  * Custom widget manifest definition.
  */
@@ -8,6 +10,9 @@ export interface ICustomWidget {
   name: string;
   /**
    * Widget unique id, probably in npm package format @gristlabs/custom-widget-name.
+   *
+   * There could be multiple versions of the same widget with the
+   * same id, e.g. a bundled version and an external version.
    */
   widgetId: string;
   /**
@@ -25,6 +30,8 @@ export interface ICustomWidget {
    * a chance to apply the Grist theme.
    */
   renderAfterReady?: boolean;
+
+  fromPlugin?: string;
 }
 
 /**
@@ -56,3 +63,65 @@ export function isSatisfied(current: AccessLevel, minimum: AccessLevel) {
   }
   return ordered(current) >= ordered(minimum);
 }
+
+export function matchWidget(widgets: ICustomWidget[], options: {
+  widgetId?: string,
+  pluginId?: string,
+}): ICustomWidget|undefined {
+  console.log("MATCHING", {
+    widgets,
+    options,
+  });
+  const prefs = sortBy(widgets, (w) => {
+    return [w.widgetId !== options.widgetId,
+            (w.fromPlugin||'') !== options.pluginId]
+  });
+  if (prefs.length === 0) { return; }
+  if (options.widgetId && prefs[0].widgetId !== options.widgetId) {
+    return;
+  }
+  console.log("ORDERED", prefs);
+  console.log("MATCHED", prefs[0]);
+  return prefs[0];
+}
+
+export function filterWidgets(widgets: ICustomWidget[], options: {
+  preferPlugin?: boolean,
+  keepWidgetIdUnique?: boolean,
+}) {
+  const folders = new Map<string, ICustomWidget[]>();
+  for (const widget of widgets) {
+    const widgetId = widget.widgetId;
+    if (!folders.has(widgetId)) { folders.set(widgetId, []); }
+    const widgetFolder = folders.get(widgetId)!;
+    widgetFolder.push(widget);
+  }
+  let finalResults: ICustomWidget[] = widgets;
+  if (options.preferPlugin !== undefined) {
+    const results = [];
+    const seen = new Set<string>();
+    for (const widget of widgets) {
+      const folder = folders.get(widget.widgetId)!;
+      if (folder.length === 1) {
+        results.push(widget);
+        continue;
+      }
+      if (seen.has(widget.widgetId)) { continue; }
+      seen.add(widget.widgetId);
+      const folderSorted = sortBy(folder, (w) => Boolean(w.fromPlugin) !== options.preferPlugin);
+      results.push(folderSorted[0]!);
+    }
+    finalResults = results;
+  }
+  if (options.keepWidgetIdUnique) {
+    const results = [];
+    const seen = new Set<string>();
+    for (const widget of widgets) {
+      if (seen.has(widget.widgetId)) { continue; }
+      seen.add(widget.widgetId);
+      results.push(widget);
+    }
+    finalResults = results;
+  }
+  return finalResults;
+}

app/plugin/PluginManifest-ti.ts

@@ -14,6 +14,7 @@ export const BarePlugin = t.iface([], {
     "safeBrowser": t.opt("string"),
     "safePython": t.opt("string"),
     "unsafeNode": t.opt("string"),
+    "widgets": t.opt("string"),
     "deactivate": t.opt(t.iface([], {
       "inactivitySec": t.opt("number"),
     })),

app/plugin/PluginManifest.ts

@@ -82,6 +82,8 @@ export interface BarePlugin {
      */
     unsafeNode?: string;
 
+    widgets?: string;
+
     /**
      * Options for when to deactivate the plugin, i.e. when to stop any plugin processes. (Note
      * that we may in the future also add options for when to activate the plugin, which is for

app/server/devServerMain.ts

@@ -34,22 +34,6 @@ function getPort(envVarName: string, fallbackPort: number): number {
   return val ? parseInt(val, 10) : fallbackPort;
 }
 
-// Checks whether to serve user content on same domain but on different port
-function checkUserContentPort(): number | null {
-  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
-    const homeUrl = new URL(process.env.APP_HOME_URL);
-    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
-    // If the hostname of both home and plugin url are the same,
-    // but the ports are different
-    if (homeUrl.hostname === pluginUrl.hostname &&
-        homeUrl.port !== pluginUrl.port) {
-      const port = parseInt(pluginUrl.port || '80', 10);
-      return port;
-    }
-  }
-  return null;
-}
-
 export async function main() {
   log.info("==========================================================================");
   log.info("== devServer");
@@ -114,14 +98,6 @@ export async function main() {
     }
     const server = await mergedServerMain(port, ["home", "docs", "static"]);
     await server.addTestingHooks();
-    // If plugin content is served from same host but on different port,
-    // run webserver on that port
-    const userPort = checkUserContentPort();
-    if (userPort !== null) {
-      log.info("==========================================================================");
-      log.info("== userContent");
-      await server.startCopy('pluginServer', userPort);
-    }
     return;
   }
 
@@ -155,15 +131,6 @@ export async function main() {
     await home.startCopy('webServer', webServerPort);
   }
 
-  // If plugin content is served from same host but on different port,
-  // run webserver on that port
-  const userPort = checkUserContentPort();
-  if (userPort !== null) {
-    log.info("==========================================================================");
-    log.info("== userContent");
-    await home.startCopy('pluginServer', userPort);
-  }
-
   // Bring up the docWorker(s)
   log.info("==========================================================================");
   log.info("== docWorker");

app/server/lib/FlexServer.ts

@@ -1,4 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
+import { ICustomWidget } from 'app/common/CustomWidget';
 import {delay} from 'app/common/delay';
 import {DocCreationInfo} from 'app/common/DocListAPI';
 import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
@@ -65,7 +66,7 @@ import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {addUploadRoute} from 'app/server/lib/uploads';
-import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
+import { buildWidgetRepository, getWidgetPlaces, IWidgetRepository} from 'app/server/lib/WidgetRepository';
 import {setupLocale} from 'app/server/localization';
 import axios from 'axios';
 import * as bodyParser from 'body-parser';
@@ -128,6 +129,9 @@ export class FlexServer implements GristServer {
   private _dbManager: HomeDBManager;
   private _defaultBaseDomain: string|undefined;
   private _pluginUrl: string|undefined;
+  private _pluginUrlSet: boolean = false;
+  private _willServePlugins?: boolean;
+  private _bundledWidgets?: ICustomWidget[];
   private _billing: IBilling;
   private _instanceRoot: string;
   private _docManager: DocManager;
@@ -220,7 +224,6 @@ export class FlexServer implements GristServer {
     }
     this.info.push(['defaultBaseDomain', this._defaultBaseDomain]);
     this._pluginUrl = options.pluginUrl || process.env.APP_UNTRUSTED_URL;
-    this.info.push(['pluginUrl', this._pluginUrl]);
 
     // The electron build is not supported at this time, but this stub
     // implementation of electronServerMethods is present to allow kicking
@@ -551,19 +554,36 @@ export class FlexServer implements GristServer {
     this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) =>
       res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
     // Plugins get access to static resources without a tag
-    this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'static'))));
+    this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'static'))));
-    this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'bower_components'))));
+    this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'bower_components'))));
     // Serve custom-widget.html message for anyone.
     this.app.use(/^\/(custom-widget.html)$/, expressWrap(async (req, res) =>
       res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
     this.addOrg();
     addPluginEndpoints(this, await this._addPluginManager());
+    const places = getWidgetPlaces(this, 'wotnot');
+    // Allow static files to be requested from any origin.
+    const options: serveStatic.ServeStaticOptions = {
+      setHeaders: (res, filepath, stat) => {
+        res.setHeader("Access-Control-Allow-Origin", "*");
+      }
+    };
+     for (const place of places) {
+       this.app.use(
+         '/widgets/' + place.name,
+         this.tagChecker.withTag(
+           limitToPlugins(this,
+             express.static(place.fileDir, options)
+           )
+         )
+       );
+    }
   }
 
   // Prepare cache for managing org-to-host relationship.
   public addHosts() {
     if (this._check('hosts', 'homedb')) { return; }
-    this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this._pluginUrl);
+    this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this);
   }
 
   public async initHomeDBManager() {
@@ -671,7 +691,7 @@ export class FlexServer implements GristServer {
 
     // ApiServer's constructor adds endpoints to the app.
     // tslint:disable-next-line:no-unused-expression
-    new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository());
+    new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository(this));
   }
 
   public addBillingApi() {
@@ -1425,6 +1445,56 @@ export class FlexServer implements GristServer {
     });
   }
 
+  public setPluginPort(port: number) {
+    const url = new URL(this.getOwnUrl());
+    url.port = String(port);
+    this._pluginUrl = url.href;
+  }
+
+  public willServePlugins() {
+    if (this._willServePlugins === undefined) {
+      throw new Error('do not know if will serve plugins');
+    }
+    return this._willServePlugins;
+  }
+  
+  public setWillServePlugins(flag: boolean) {
+    this._willServePlugins = flag;
+  }
+
+  public getPluginUrl() {
+    if (!this._pluginUrlSet) {
+      throw new Error('looked at plugin url too early');
+    }
+    return this._pluginUrl;
+  }
+
+  public getPlugins() {
+    if (!this._pluginManager) {
+      throw new Error('plugin manager not available');
+    }
+    return this._pluginManager.getPlugins();
+  }
+
+  public async prepareSummary() {
+    // Add some information that isn't guaranteed set until the end.
+
+    this.info.push(['pluginUrl', this._pluginUrl]);
+    // plugin url should be finalized by now.
+    this._pluginUrlSet = true;
+    this.info.push(['willServePlugins', this._willServePlugins]);
+
+    const repo = buildWidgetRepository(this, { localOnly: true });
+    this._bundledWidgets = await repo.getWidgets();
+  }
+
+  public getBundledWidgets(): ICustomWidget[] {
+    if (!this._bundledWidgets) {
+      throw new Error('bundled widgets accessed too early');
+    }
+    return this._bundledWidgets;
+  }
+
   public summary() {
     for (const [label, value] of this.info) {
       log.info("== %s: %s", label, value);
@@ -1538,9 +1608,12 @@ export class FlexServer implements GristServer {
     await this.housekeeper.start();
   }
 
-  public async startCopy(name2: string, port2: number) {
+  public async startCopy(name2: string, port2: number): Promise<{
+    serverPort: number,
+    httpsServerPort?: number,
+  }>{
     const servers = this._createServers();
-    await this._startServers(servers.server, servers.httpsServer, name2, port2, true);
+    return this._startServers(servers.server, servers.httpsServer, name2, port2, true);
   }
 
   /**
@@ -1606,6 +1679,9 @@ export class FlexServer implements GristServer {
   }
 
   public getTag(): string {
+    if (!this.tag) {
+      throw new Error('getTag called too early');
+    }
     return this.tag;
   }
 
@@ -1913,6 +1989,10 @@ export class FlexServer implements GristServer {
       await listenPromise(httpsServer.listen(httpsPort, this.host));
       if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); }
     }
+    return {
+      serverPort: (server.address() as AddressInfo).port,
+      httpsServerPort: (server.address() as AddressInfo)?.port,
+    };
   }
 
   private async _recordNewUserInfo(row: object) {

app/server/lib/GristServer.ts

@@ -1,4 +1,6 @@
+import { ICustomWidget } from 'app/common/CustomWidget';
 import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
+import { LocalPlugin } from 'app/common/plugin';
 import { FullUser, UserProfile } from 'app/common/UserAPI';
 import { Document } from 'app/gen-server/entity/Document';
 import { Organization } from 'app/gen-server/entity/Organization';
@@ -53,6 +55,10 @@ export interface GristServer {
   sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void>;
   getAccessTokens(): IAccessTokens;
   resolveLoginSystem(): Promise<GristLoginSystem>;
+  getPluginUrl(): string|undefined;
+  getPlugins(): LocalPlugin[];
+  willServePlugins(): boolean;
+  getBundledWidgets(): ICustomWidget[];
 }
 
 export interface GristLoginSystem {
@@ -135,6 +141,10 @@ export function createDummyGristServer(): GristServer {
     sendAppPage() { return Promise.resolve(); },
     getAccessTokens() { throw new Error('no access tokens'); },
     resolveLoginSystem() { throw new Error('no login system'); },
+    getPluginUrl() { return undefined; },
+    willServePlugins() { return false; },
+    getPlugins() { return []; },
+    getBundledWidgets() { return []; },
   };
 }
 

app/server/lib/PluginEndpoint.ts

@@ -1,34 +1,36 @@
 import {FlexServer} from 'app/server/lib/FlexServer';
+import {GristServer} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
 import {PluginManager} from 'app/server/lib/PluginManager';
 import * as express from 'express';
 import * as mimeTypes from 'mime-types';
 import * as path from 'path';
 
-// Get the url where plugin material should be served from.
-export function getUntrustedContentOrigin(): string|undefined {
-  return process.env.APP_UNTRUSTED_URL;
-}
-
 // Get the host serving plugin material
-export function getUntrustedContentHost(): string|undefined {
+export function getUntrustedContentHost(origin: string|undefined): string|undefined {
-  const origin = getUntrustedContentOrigin();
   if (!origin) { return; }
   return new URL(origin).host;
 }
 
 // Add plugin endpoints to be served on untrusted host
 export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) {
-  const host = getUntrustedContentHost();
+  if (server.willServePlugins()) {
-  if (host) {
     server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) =>
-                   servePluginContent(req, res, pluginManager, host));
+                   servePluginContent(req, res, pluginManager, server));
   }
 }
 
 // Serve content for plugins with various checks that it is being accessed as we expect.
 function servePluginContent(req: express.Request, res: express.Response,
-                            pluginManager: PluginManager, untrustedContentHost: string) {
+                            pluginManager: PluginManager,
+                            gristServer: GristServer) {
+  const pluginUrl = gristServer.getPluginUrl();
+  const untrustedContentHost = getUntrustedContentHost(pluginUrl);
+  if (!untrustedContentHost) {
+    // not expected
+    throw new Error('plugin host unexpectedly not set');
+  }
+
   const pluginKind = req.params[0];
   const pluginId = req.params[1];
   const pluginPath = req.params[2];
@@ -56,9 +58,11 @@ function servePluginContent(req: express.Request, res: express.Response,
 }
 
 // Middleware to restrict some assets to untrusted host.
-export function limitToPlugins(handler: express.RequestHandler) {
+export function limitToPlugins(gristServer: GristServer,
-  const host = getUntrustedContentHost();
+                               handler: express.RequestHandler) {
   return function(req: express.Request, resp: express.Response, next: express.NextFunction) {
+    const pluginUrl = gristServer.getPluginUrl();
+    const host = getUntrustedContentHost(pluginUrl);
     if (!host) { return next(); }
     if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") {
       return handler(req, resp, next);

app/server/lib/PluginManager.ts

@@ -131,6 +131,7 @@ export class PluginManager {
 
 
 async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise<DirectoryScanEntry[]> {
+  console.log("SCAN", {dir, kind});
   const plugins: DirectoryScanEntry[] = [];
   let listDir;
 

app/server/lib/WidgetRepository.ts

@@ -1,8 +1,14 @@
 import {ICustomWidget} from 'app/common/CustomWidget';
 import log from 'app/server/lib/log';
+import * as fse from 'fs-extra';
 import fetch from 'node-fetch';
+import * as path from 'path';
 import {ApiError} from 'app/common/ApiError';
 import LRUCache from 'lru-cache';
+import * as url from 'url';
+import { removeTrailingSlash } from 'app/common/gutil';
+import { GristServer } from './GristServer';
+// import { LocalPlugin } from 'app/common/plugin';
 
 /**
  * Widget Repository returns list of available Custom Widgets.
@@ -14,25 +20,89 @@ export interface IWidgetRepository {
 // Static url for StaticWidgetRepository
 const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
 
-/**
+export class FileWidgetRepository implements IWidgetRepository {
- * Default repository that gets list of available widgets from a static URL.
+  constructor(private _widgetFileName: string,
- */
+              private _widgetBaseUrl: string,
-export class WidgetRepositoryImpl implements IWidgetRepository {
+              private _pluginId?: string) {}
-  constructor(protected _staticUrl = STATIC_URL) {}
 
-  /**
+  public async getWidgets(): Promise<ICustomWidget[]> {
-   * Method exposed for testing, overrides widget url.
+    const txt = await fse.readFile(this._widgetFileName, {
-   */
+      encoding: 'utf8',
-  public testOverrideUrl(url: string) {
+    });
-    this._staticUrl = url;
+    const widgets: ICustomWidget[] = JSON.parse(txt);
+    fixUrls(widgets, this._widgetBaseUrl);
+    if (this._pluginId) {
+      for (const widget of widgets) {
+        widget.fromPlugin = this._pluginId;
+      }
+    }
+    console.log("FileWidget", {widgets});
+    return widgets;
   }
+}
+
+/*
+export class NestedWidgetRepository implements IWidgetRepository {
+  constructor(private _widgetDir: string,
+              private _widgetBaseUrl: string) {}
+
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    const listDir = await fse.readdir(this._widgetDir,
+                                      { withFileTypes: true });
+    const fileName = 'manifest.json';
+    const allWidgets: ICustomWidget[] = [];
+    for (const dir of listDir) {
+      if (!dir.isDirectory()) { continue; }
+      const fullPath = path.join(this._widgetDir, dir.name, fileName);
+      if (!await fse.pathExists(fullPath)) { continue; }
+      const txt = await fse.readFile(fullPath, 'utf8');
+      const widgets = JSON.parse(txt);
+      fixUrls(
+        widgets,
+        removeTrailingSlash(this._widgetBaseUrl) + '/' + dir.name + '/'
+      );
+      allWidgets.push(...widgets);
+    }
+    return allWidgets;
+  }
+}
+*/
+
+export class DelayedWidgetRepository implements IWidgetRepository {
+  constructor(private _makeRepo: () => Promise<IWidgetRepository|undefined>) {}
+
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    const repo = await this._makeRepo();
+    if (!repo) { return []; }
+    return repo.getWidgets();
+  }
+}
+
+export class CombinedWidgetRepository implements IWidgetRepository {
+  constructor(private _repos: IWidgetRepository[]) {}
+
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    const allWidgets: ICustomWidget[] = [];
+    for (const repo of this._repos) {
+      allWidgets.push(...await repo.getWidgets());
+    }
+    console.log("COMBINED", {allWidgets});
+    return allWidgets;
+  }
+}
+
+/**
+ * Repository that gets list of available widgets from a static URL.
+ */
+export class UrlWidgetRepository implements IWidgetRepository {
+  constructor(private _staticUrl = STATIC_URL) {}
 
   public async getWidgets(): Promise<ICustomWidget[]> {
     if (!this._staticUrl) {
       log.warn(
-        'WidgetRepository: Widget repository is not configured.' + !STATIC_URL
+        'WidgetRepository: Widget repository is not configured.' + (!STATIC_URL
           ? ' Missing GRIST_WIDGET_LIST_URL environmental variable.'
-          : ''
+          : '')
       );
       return [];
     }
@@ -52,6 +122,7 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
       if (!widgets || !Array.isArray(widgets)) {
         throw new ApiError('WidgetRepository: Error reading widget list', 500);
       }
+      fixUrls(widgets, this._staticUrl);
       return widgets;
     } catch (err) {
       if (!(err instanceof ApiError)) {
@@ -62,6 +133,56 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
   }
 }
 
+/**
+ * Default repository that gets list of available widgets from a static URL.
+ */
+export class WidgetRepositoryImpl implements IWidgetRepository {
+  protected _staticUrl: string|undefined;
+  private _urlWidgets: UrlWidgetRepository;
+  private _combinedWidgets: CombinedWidgetRepository;
+  private _dirWidgets?: IWidgetRepository;
+
+  constructor(_options: {
+    staticUrl?: string,
+    gristServer?: GristServer,
+  }) {
+    const {staticUrl, gristServer} = _options;
+    if (gristServer) {
+      this._dirWidgets = new DelayedWidgetRepository(async () => {
+        const places = getWidgetPlaces(gristServer);
+        console.log("PLACES!", places);
+        const files = places.map(place => new FileWidgetRepository(place.fileBase,
+                                                                   place.urlBase,
+                                                                   place.pluginId));
+        return new CombinedWidgetRepository(files);
+      });
+    }
+    this.testSetUrl(staticUrl);
+  }
+
+  /**
+   * Method exposed for testing, overrides widget url.
+   */
+  public testOverrideUrl(url: string|undefined) {
+    this.testSetUrl(url);
+  }
+
+  public testSetUrl(url: string|undefined) {
+    const repos: IWidgetRepository[] = [];
+    this._staticUrl = url ?? STATIC_URL;
+    if (this._staticUrl) {
+      this._urlWidgets = new UrlWidgetRepository(this._staticUrl);
+      repos.push(this._urlWidgets);
+    }
+    if (this._dirWidgets) { repos.push(this._dirWidgets); }
+    this._combinedWidgets = new CombinedWidgetRepository(repos);
+  }
+
+  public async getWidgets(): Promise<ICustomWidget[]> {
+    return this._combinedWidgets.getWidgets();
+  }
+}
+
 /**
  * Version of WidgetRepository that caches successful result for 2 minutes.
  */
@@ -79,6 +200,7 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
     const list = await super.getWidgets();
     // Cache only if there are some widgets.
     if (list.length) { this._cache.set(1, list); }
+    console.log("CACHABLE RESULT", {list});
     return list;
   }
 
@@ -91,6 +213,66 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
 /**
  * Returns widget repository implementation.
  */
-export function buildWidgetRepository() {
+export function buildWidgetRepository(gristServer: GristServer,
-  return new CachedWidgetRepository();
+                                      options?: {
+                                        localOnly: boolean
+                                      }) {
+  if (options?.localOnly) {
+    return new WidgetRepositoryImpl({
+      gristServer,
+      staticUrl: ''
+    });
+  }
+  return new CachedWidgetRepository({
+    gristServer,
+  });
+}
+
+function fixUrls(widgets: ICustomWidget[], baseUrl: string) {
+  // If URLs are relative, make them absolute, interpreting them
+  // relative to the manifest file.
+  for (const widget of widgets) {
+    if (!(url.parse(widget.url).protocol)) {
+      widget.url = new URL(widget.url, baseUrl).href;
+    }
+  }
+}
+
+export interface CustomWidgetPlace {
+  urlBase: string,
+  fileBase: string,
+  fileDir: string,
+  name: string,
+  pluginId: string,
+}
+
+export function getWidgetPlaces(gristServer: GristServer,
+                                pluginUrl?: string) {
+  const places: CustomWidgetPlace[] = [];
+  const plugins = gristServer.getPlugins();
+  console.log("PLUGINS", plugins);
+  pluginUrl = pluginUrl || gristServer.getPluginUrl();
+  if (!pluginUrl) { return []; }
+  for (const plugin of plugins) {
+    console.log("PLUGIN", plugin);
+    const components = plugin.manifest.components;
+    if (!components.widgets) { continue; }
+    console.log("GOT SOMETHING", {
+      name: plugin.id,
+      path: plugin.path,
+      widgets: components.widgets
+    });
+    const urlBase =
+        removeTrailingSlash(pluginUrl) + '/v/' +
+        gristServer.getTag() + '/widgets/' + plugin.id + '/';
+    places.push({
+      urlBase,
+      fileBase: path.join(plugin.path, components.widgets),
+      fileDir: plugin.path,
+      name: plugin.id,
+      pluginId: plugin.id,
+    });
+  }
+  console.log("PLACES", places);
+  return places;
 }

app/server/lib/extractOrg.ts

@@ -6,6 +6,7 @@ import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
 import { getOriginUrl } from 'app/server/lib/requestUtils';
 import { NextFunction, Request, RequestHandler, Response } from 'express';
 import { IncomingMessage } from 'http';
+import { GristServer } from './GristServer';
 
 // How long we cache information about the relationship between
 // orgs and custom hosts.  The higher this is, the fewer requests
@@ -41,7 +42,7 @@ export class Hosts {
 
   // baseDomain should start with ".". It may be undefined for localhost or single-org mode.
   constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager,
-              private _pluginUrl: string|undefined) {
+              private _gristServer: GristServer|undefined) {
   }
 
   /**
@@ -165,6 +166,6 @@ export class Hosts {
   }
 
   private _getHostType(host: string) {
-    return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._pluginUrl});
+    return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._gristServer?.getPluginUrl()});
   }
 }

app/server/lib/sendAppPage.ts

@@ -33,7 +33,7 @@ export interface ISendAppPageOptions {
   googleTagManager?: true | false | 'anon';
 }
 
-export interface MakeGristConfigOptons {
+export interface MakeGristConfigOptions {
   homeUrl: string|null;
   extra: Partial<GristLoadConfig>;
   baseDomain?: string;
@@ -41,7 +41,7 @@ export interface MakeGristConfigOptons {
   server?: GristServer|null;
 }
 
-export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
+export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfig {
   const {homeUrl, extra, baseDomain, req, server} = options;
   // .invalid is a TLD the IETF promises will never exist.
   const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
@@ -78,7 +78,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
     featureComments: isAffirmative(process.env.COMMENTS),
     featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT),
     assistantService: process.env.OPENAI_API_KEY ? 'OpenAI' : undefined,
-    permittedCustomWidgets: getPermittedCustomWidgets(),
+    permittedCustomWidgets: getPermittedCustomWidgets(server),
     supportEmail: SUPPORT_EMAIL,
     userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
     telemetry: server?.getTelemetry().getTelemetryConfig(),
@@ -115,6 +115,7 @@ export function makeSendAppPage(opts: {
 }) {
   const {server, staticDir, tag, testLogin} = opts;
   return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
+      console.log("HERE WE GO");
       const config = makeGristConfig({
         homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
         extra: options.config,
@@ -170,9 +171,30 @@ function getFeatures(): IFeature[] {
   return Features.checkAll(difference(enabledFeatures, disabledFeatures));
 }
 
-function getPermittedCustomWidgets(): IAttachedCustomWidget[] {
+function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCustomWidget[] {
+  if (!process.env.PERMITTED_CUSTOM_WIDGETS && gristServer) {
+    console.log("*****");
+    console.log("*****");
+    console.log("*****");
+    const widgets = gristServer.getBundledWidgets();
+    const names = new Set(AttachedCustomWidgets.values as string[]);
+    console.log({widgets, names});
+    const namesFound: IAttachedCustomWidget[] = [];
+    for (const widget of widgets) {
+      // For some reason in different parts of the code attached custom
+      // widgets are identified by a lot of variants of their name or id
+      // e.g. "Calendar", "calendar", "custom.calendar", "@gristlabs/grist-calendar'...
+      const name = widget.widgetId.replace('@gristlabs/widget-', 'custom.');
+      console.log("CHECK", {name});
+      if (names.has(name)) {
+        console.log("CHECK FOUND", {name});
+        namesFound.push(name as IAttachedCustomWidget);
+      }
+    }
+    return AttachedCustomWidgets.checkAll(namesFound);
+  }
   const widgetsList = process.env.PERMITTED_CUSTOM_WIDGETS?.split(',').map(widgetName=>`custom.${widgetName}`) ?? [];
-  return  AttachedCustomWidgets.checkAll(widgetsList);
+  return AttachedCustomWidgets.checkAll(widgetsList);
 }
 
 function configuredPageTitleSuffix() {

app/server/mergedServerMain.ts

@@ -31,6 +31,26 @@ export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
   return types as ServerType[];
 }
 
+function checkUserContentPort(): number | null {
+  // Check whether a port is explicitly set for user content.
+  if (process.env.GRIST_UNTRUSTED_PORT) {
+    return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
+  }
+  // Checks whether to serve user content on same domain but on different port
+  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
+    const homeUrl = new URL(process.env.APP_HOME_URL);
+    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
+    // If the hostname of both home and plugin url are the same,
+    // but the ports are different
+    if (homeUrl.hostname === pluginUrl.hostname &&
+        homeUrl.port !== pluginUrl.port) {
+      const port = parseInt(pluginUrl.port || '80', 10);
+      return port;
+    }
+  }
+  return null;
+}
+
 interface ServerOptions extends FlexServerOptions {
   logToConsole?: boolean;  // If set, messages logged to console (default: false)
                            //   (but if options are not given at all in call to main,
@@ -52,6 +72,13 @@ export async function main(port: number, serverTypes: ServerType[],
 
   const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
 
+  if (includeHome) {
+    const userPort = checkUserContentPort();
+    server.setWillServePlugins(userPort !== undefined);
+  } else {
+    server.setWillServePlugins(false);
+  }
+
   if (options.loginSystem) {
     server.setLoginSystem(options.loginSystem);
   }
@@ -143,7 +170,26 @@ export async function main(port: number, serverTypes: ServerType[],
 
     server.finalize();
 
+    if (includeHome) {
+      // If plugin content is served from same host but on different port,
+      // run webserver on that port
+      const userPort = checkUserContentPort();
+      if (userPort !== null) {
+        const ports = await server.startCopy('pluginServer', userPort);
+        // If Grist is running on a desktop, directly on the host, it
+        // can be convenient to leave the user port free for the OS to
+        // allocate by using GRIST_UNTRUSTED_PORT=0. But we do need to
+        // remember how to contact it.
+        if (userPort === 0) {
+          server.setPluginPort(ports.serverPort);
+        } else if (process.env.APP_UNTRUSTED_URL === undefined) {
+          server.setPluginPort(userPort);
+        }
+      }
+    }
+
     server.checkOptionCombinations();
+    await server.prepareSummary();
     server.summary();
     return server;
   } catch(e) {

package.json

@@ -51,7 +51,7 @@
     "@types/double-ended-queue": "2.1.0",
     "@types/express": "4.16.0",
     "@types/form-data": "2.2.1",
-    "@types/fs-extra": "5.0.4",
+    "@types/fs-extra": "11.0.2",
     "@types/http-proxy": "1.17.9",
     "@types/i18next-fs-backend": "1.1.2",
     "@types/image-size": "0.0.29",
@@ -139,7 +139,7 @@
     "exceljs": "4.2.1",
     "express": "4.16.4",
     "file-type": "16.5.4",
-    "fs-extra": "7.0.0",
+    "fs-extra": "11.1.1",
     "grain-rpc": "0.1.7",
     "grainjs": "1.0.2",
     "handlebars": "4.7.7",

stubs/app/server/server.ts

@@ -107,6 +107,11 @@ export async function main() {
   // Set directory for uploaded documents.
   setDefaultEnv('GRIST_DATA_DIR', 'docs');
   setDefaultEnv('GRIST_SERVERS', 'home,docs,static');
+  if (process.env.GRIST_SERVERS?.includes('home')) {
+    // By default, we will now start an untrusted port alongside a
+    // home server. Suppress with GRIST_UNTRUSTED_PORT=''
+    setDefaultEnv('GRIST_UNTRUSTED_PORT', '0');
+  }
   const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
 
   await fse.mkdirp(process.env.GRIST_DATA_DIR!);

yarn.lock

@@ -721,11 +721,12 @@
   dependencies:
     "@types/node" "*"
 
-"@types/fs-extra@5.0.4":
+"@types/fs-extra@11.0.2":
-  version "5.0.4"
+  version "11.0.2"
-  resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.4.tgz"
+  resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.2.tgz#23dc1ed7b2eba8ccd75568ac34e7a4e48aa2d087"
-  integrity sha512-DsknoBvD8s+RFfSGjmERJ7ZOP1HI0UZRA3FSI+Zakhrc/Gy26YQsLI+m5V5DHxroHRJqCDLKJp7Hixn8zyaF7g==
+  integrity sha512-c0hrgAOVYr21EX8J0jBMXGLMgJqVf/v6yxi0dLaJboW9aQPh16Id+z6w2Tx1hm+piJOLv8xPfVKZCLfjPw/IMQ==
   dependencies:
+    "@types/jsonfile" "*"
     "@types/node" "*"
 
 "@types/http-cache-semantics@*":
@@ -785,6 +786,13 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3"
   integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==
 
+"@types/jsonfile@*":
+  version "6.1.2"
+  resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.2.tgz#d3b8a3536c5bb272ebee0f784180e456b7691c8f"
+  integrity sha512-8t92P+oeW4d/CRQfJaSqEwXujrhH4OEeHRjGU3v1Q8mUS8GPF3yiX26sw4svv6faL2HfBtGTe2xWIoVgN3dy9w==
+  dependencies:
+    "@types/node" "*"
+
 "@types/jsonwebtoken@7.2.8":
   version "7.2.8"
   resolved "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-7.2.8.tgz"
@@ -857,9 +865,9 @@
     form-data "^3.0.0"
 
 "@types/node@*":
-  version "14.0.1"
+  version "20.7.2"
-  resolved "https://registry.npmjs.org/@types/node/-/node-14.0.1.tgz"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.7.2.tgz#0bdc211f8c2438cfadad26dc8c040a874d478aed"
-  integrity sha512-FAYBGwC+W6F9+huFIDtn43cpy7+SzG+atzRiTfdp3inUKL2hXnd4rG8hylJLIh4+hqrQy1P17kvJByE/z825hA==
+  integrity sha512-RcdC3hOBOauLP+r/kRt27NrByYtDjsXyAuSbR87O6xpsvi763WI+5fbSIvYJrXnt9w4RuxhV6eAXfIs7aaf/FQ==
 
 "@types/node@^14":
   version "14.18.21"
@@ -3909,14 +3917,14 @@ fs-constants@^1.0.0:
   resolved "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz"
   integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
 
-fs-extra@7.0.0:
+fs-extra@11.1.1:
-  version "7.0.0"
+  version "11.1.1"
-  resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.0.tgz"
+  resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d"
-  integrity sha512-EglNDLRpmaTWiD/qraZn6HREAEAHJcJOmxNEYwq6xeMKnVMAy3GUcFB+wXt2C6k4CNvB/mP1y/U3dzvKKj5OtQ==
+  integrity sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==
   dependencies:
-    graceful-fs "^4.1.2"
+    graceful-fs "^4.2.0"
-    jsonfile "^4.0.0"
+    jsonfile "^6.0.1"
-    universalify "^0.1.0"
+    universalify "^2.0.0"
 
 fs-extra@^4.0.2, fs-extra@^4.0.3:
   version "4.0.3"
@@ -4323,12 +4331,12 @@ graceful-fs@^4.0.0, graceful-fs@^4.1.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6,
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c"
   integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==
 
-graceful-fs@^4.1.2:
+graceful-fs@^4.1.2, graceful-fs@^4.1.6:
-  version "4.2.4"
+  version "4.2.11"
-  resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz"
+  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
-  integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==
+  integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
 
-graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2:
+graceful-fs@^4.2.0, graceful-fs@^4.2.2:
   version "4.2.6"
   resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz"
   integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==
@@ -5247,8 +5255,17 @@ json5@^2.2.1:
 
 jsonfile@^4.0.0:
   version "4.0.0"
-  resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
-  integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=
+  integrity sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==
+  optionalDependencies:
+    graceful-fs "^4.1.6"
+
+jsonfile@^6.0.1:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae"
+  integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==
+  dependencies:
+    universalify "^2.0.0"
   optionalDependencies:
     graceful-fs "^4.1.6"
 
@@ -8384,7 +8401,7 @@ unique-string@^2.0.0:
 
 universalify@^0.1.0:
   version "0.1.2"
-  resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
   integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==
 
 universalify@^0.2.0:
@@ -8392,6 +8409,11 @@ universalify@^0.2.0:
   resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz"
   integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==
 
+universalify@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717"
+  integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==
+
 unpipe@1.0.0, unpipe@~1.0.0:
   version "1.0.0"
   resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"