diff --git a/app/client/components/CustomCalendarView.ts b/app/client/components/CustomCalendarView.ts index f666ff30..f7e1f20c 100644 --- a/app/client/components/CustomCalendarView.ts +++ b/app/client/components/CustomCalendarView.ts @@ -1,11 +1,13 @@ -import {AccessLevel} from "app/common/CustomWidget"; -import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec"; -import {CustomView} from "app/client/components/CustomView"; -import {GristDoc} from "app/client/components/GristDoc"; -import {reportError} from 'app/client/models/errors'; +// import {AccessLevel} from "app/common/CustomWidget"; +// import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec"; +import { CustomView, CustomViewSettings } from "app/client/components/CustomView"; +import { AccessLevel } from "app/common/CustomWidget"; +// import {GristDoc} from "app/client/components/GristDoc"; +// import {reportError} from 'app/client/models/errors'; //Abstract class for more future inheritances -abstract class CustomAttachedView extends CustomView { +// abstract class CustomAttachedView extends CustomView { + /* public override create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) { super.create(gristDoc, viewSectionModel); if (viewSectionModel.customDef.access.peek() !== AccessLevel.full) { @@ -18,7 +20,7 @@ abstract class CustomAttachedView extends CustomView { }); } - const widgetsApi = this.gristDoc.app.topAppModel.api; + const widgetsApi = this.gristDoc.app.topAppModel; widgetsApi.getWidgets().then(async result=>{ const widget = result.find(w=>w.name == this.getWidgetName()); if (widget && this.customDef.url.peek() !== widget.url) { @@ -34,13 +36,17 @@ abstract class CustomAttachedView extends CustomView { } }); } + */ - protected abstract getWidgetName(): string; +// protected abstract getWidgetName(): string; -} +// } -export class CustomCalendarView extends CustomAttachedView { - protected getWidgetName(): string { - return "Calendar"; +export class CustomCalendarView extends CustomView { + protected getInitialSettings(): CustomViewSettings { + return { + widgetId: '@gristlabs/widget-calendar', + accessLevel: AccessLevel.full, + }; } } diff --git a/app/client/components/CustomView.ts b/app/client/components/CustomView.ts index 32f1b019..55ac3714 100644 --- a/app/client/components/CustomView.ts +++ b/app/client/components/CustomView.ts @@ -32,6 +32,10 @@ import {dom as grains} from 'grainjs'; import * as ko from 'knockout'; import defaults = require('lodash/defaults'); +export interface CustomViewSettings { + widgetId?: string; + accessLevel?: AccessLevel; +} /** * CustomView components displays arbitrary html. There are two modes available, in the "url" mode @@ -81,11 +85,10 @@ export class CustomView extends Disposable { private _frame: WidgetFrame; // plugin frame (holding external page) - public create(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) { BaseView.call(this as any, gristDoc, viewSectionModel, { 'addNewRow': true }); - this.customDef = this.viewSection.customDef; + this.customDef = this.viewSection.customDef; this.autoDisposeCallback(() => { if (this._customSection) { @@ -103,8 +106,43 @@ export class CustomView extends Disposable { this.viewPane = this.autoDispose(this._buildDom()); this._updatePluginInstance(); + + this.dealWithBundledWidgets(gristDoc, viewSectionModel); } + public dealWithBundledWidgets(gristDoc: GristDoc, viewSectionModel: ViewSectionRec) { + const settings = this.getInitialSettings(); + console.log("dealWith!", {settings}); + if (!settings.widgetId) { return; } + if (viewSectionModel.customDef.access.peek() !== AccessLevel.full) { + void viewSectionModel.customDef.access.setAndSave(AccessLevel.full).catch((err)=>{ + if (err?.code === "ACL_DENY") { + // do nothing, we might be in a readonly mode. + return; + } + reportError(err); + }); + } + + const widgetsApi = this.gristDoc.app.topAppModel; + widgetsApi.getWidgets().then(async result=>{ + const widget = result.find(w => w.widgetId === settings.widgetId); + console.log("FOUND", {widget}); + if (widget && this.customDef.widgetId.peek() !== widget.widgetId) { + console.log("SET!!"); + await this.customDef.widgetId.setAndSave(widget.widgetId); + await this.customDef.pluginId.setAndSave(widget.fromPlugin||''); + } + }).catch((err)=>{ + if (err?.code !== "ACL_DENY") { + // TODO: revisit it later. getWidgets() is async call, and non of the code + // above is checking if we are still alive. + console.error(err); + } else { + // do nothing, we might be in a readonly mode. + } + }); + } public async triggerPrint() { if (!this.isDisposed() && this._frame) { @@ -112,9 +150,14 @@ export class CustomView extends Disposable { } } + protected getInitialSettings(): CustomViewSettings { + return {}; + } + protected getEmptyWidgetPage(): string { return new URL("custom-widget.html", getGristConfig().homeUrl!).href; } + /** * Find a plugin instance that matches the plugin id, update the `found` observables, then tries to * find a matching section. @@ -154,13 +197,16 @@ export class CustomView extends Disposable { } private _buildDom() { - const {mode, url, access, renderAfterReady} = this.customDef; + const {mode, url, access, renderAfterReady, widgetId, pluginId} = this.customDef; const showPlugin = ko.pureComputed(() => this.customDef.mode() === "plugin"); const showAfterReady = () => { // The empty widget page calls `grist.ready()`. + // Pending: URLs set now only when user actually enters a URL, + // so this could be breaking pages without grist.ready() call + // added to manifests. if (!url()) { return true; } - return this.customDef.widgetDef()?.renderAfterReady ?? renderAfterReady(); + return renderAfterReady(); }; // When both plugin and section are not found, let's show only plugin notification. @@ -176,12 +222,14 @@ export class CustomView extends Disposable { dom.autoDispose(showSectionNotification), dom.autoDispose(showPluginContent), // todo: should display content in webview when running electron - kd.scope(() => [mode(), url(), access()], ([_mode, _url, _access]: string[]) => + kd.scope(() => [mode(), url(), access(), widgetId(), pluginId()], ([_mode, _url, _access, _widgetId, _pluginId]: string[]) => _mode === "url" ? this._buildIFrame({ baseUrl: _url, access: (_access as AccessLevel || AccessLevel.none), showAfterReady: showAfterReady(), + widgetId: _widgetId, + pluginId: _pluginId, }) : null ), @@ -211,10 +259,15 @@ export class CustomView extends Disposable { baseUrl: string|null, access: AccessLevel, showAfterReady?: boolean, + widgetId?: string|null, + pluginId?: string, }) { - const {baseUrl, access, showAfterReady} = options; + const {baseUrl, access, showAfterReady, widgetId, pluginId} = options; return grains.create(WidgetFrame, { url: baseUrl || this.getEmptyWidgetPage(), + widgetId, + pluginId, + emptyUrl: this.getEmptyWidgetPage(), access, readonly: this.gristDoc.isReadonly.get(), showAfterReady, @@ -265,7 +318,8 @@ export class CustomView extends Disposable { } // allow menus to close if any closeRegisteredMenu(); - }) + }), + gristDoc: this.gristDoc, }); } diff --git a/app/client/components/WidgetFrame.ts b/app/client/components/WidgetFrame.ts index 6c08d5c7..575e1177 100644 --- a/app/client/components/WidgetFrame.ts +++ b/app/client/components/WidgetFrame.ts @@ -6,7 +6,7 @@ import {hooks} from 'app/client/Hooks'; import {get as getBrowserGlobals} from 'app/client/lib/browserGlobals'; import {makeTestId} from 'app/client/lib/domUtils'; import {ColumnRec, ViewSectionRec} from 'app/client/models/DocModel'; -import {AccessLevel, isSatisfied} from 'app/common/CustomWidget'; +import {AccessLevel, ICustomWidget, isSatisfied, matchWidget } from 'app/common/CustomWidget'; import {DisposableWithEvents} from 'app/common/DisposableWithEvents'; import {BulkColValues, fromTableDataAction, RowRecord} from 'app/common/DocActions'; import {extractInfoFromColType, reencodeAsAny} from 'app/common/gristTypes'; @@ -19,6 +19,7 @@ import noop = require('lodash/noop'); import debounce = require('lodash/debounce'); import isEqual = require('lodash/isEqual'); import flatMap = require('lodash/flatMap'); +import { reportError } from '../models/errors'; const testId = makeTestId('test-custom-widget-'); @@ -43,6 +44,9 @@ export interface WidgetFrameOptions { * Url of external page. Iframe is rebuild each time the URL changes. */ url: string; + widgetId?: string|null; + pluginId?: string; + emptyUrl: string; /** * Assigned access level. Iframe is rebuild each time access level is changed. */ @@ -73,6 +77,8 @@ export interface WidgetFrameOptions { * Optional handler to modify the iframe. */ onElem?: (iframe: HTMLIFrameElement) => void; + + gristDoc: GristDoc; } /** @@ -87,6 +93,7 @@ export class WidgetFrame extends DisposableWithEvents { private _readyCalled = Observable.create(this, false); // Whether the iframe is visible. private _visible = Observable.create(this, !this._options.showAfterReady); + public readonly _widgets = Observable.create(this, []); constructor(private _options: WidgetFrameOptions) { super(); @@ -113,7 +120,10 @@ export class WidgetFrame extends DisposableWithEvents { // Call custom configuration handler. _options.configure?.(this); + + this._fetchWidgets().catch(reportError); } + /** * Attach an EventSource with desired access level. */ @@ -167,6 +177,23 @@ export class WidgetFrame extends DisposableWithEvents { } public buildDom() { + const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el); + return onElem( + (this._iframe = dom( + 'iframe', + dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'), + dom.cls('clipboard_focus'), + dom.cls('custom_view'), + dom.attr('src', use => this._getUrl(use(this._widgets))), + { + ...hooks.iframeAttributes, + }, + testId('ready', this._readyCalled), + )) + ); + } + + private _getUrl(widgets: ICustomWidget[]): string { // Append access level to query string. const urlWithAccess = (url: string) => { if (!url) { @@ -177,19 +204,20 @@ export class WidgetFrame extends DisposableWithEvents { urlObj.searchParams.append('readonly', String(this._options.readonly)); return urlObj.href; }; - const fullUrl = urlWithAccess(this._options.url); - const onElem = this._options.onElem ?? ((el: HTMLIFrameElement) => el); - return onElem( - (this._iframe = dom('iframe', - dom.style('visibility', use => use(this._visible) ? 'visible' : 'hidden'), - dom.cls('clipboard_focus'), - dom.cls('custom_view'), { - src: fullUrl, - ...hooks.iframeAttributes, - }, - testId('ready', this._readyCalled), - )) - ); + const {widgetId, pluginId} = this._options; + let url = this._options.url; + if (widgetId) { + console.log("Iframe match starting"); + const widget = matchWidget(widgets, {widgetId, pluginId}); + console.log("Iframe match done"); + if (widget) { + url = widget.url; + } else { + return 'about:blank'; + } + } + const fullUrl = urlWithAccess(url); + return fullUrl; } private _onMessage(event: MessageEvent) { @@ -216,6 +244,14 @@ export class WidgetFrame extends DisposableWithEvents { this._rpc.receiveMessage(event.data); } } + + private async _fetchWidgets() { + if (this.isDisposed()) { return; } + const widgets = await this._options.gristDoc.app.topAppModel.getWidgets(); + if (this.isDisposed()) { return; } + this._widgets.set(widgets); + console.log("SAVED", {widgets}); + } } const throwError = (access: AccessLevel) => { diff --git a/app/client/models/AppModel.ts b/app/client/models/AppModel.ts index 7dfe341e..9b2d1c42 100644 --- a/app/client/models/AppModel.ts +++ b/app/client/models/AppModel.ts @@ -27,6 +27,8 @@ import {getOrgName, isTemplatesOrg, Organization, OrgError, UserAPI, UserAPIImpl import {getUserPrefObs, getUserPrefsObs, markAsSeen, markAsUnSeen} from 'app/client/models/UserPrefs'; import {bundleChanges, Computed, Disposable, Observable, subscribe} from 'grainjs'; import isEqual from 'lodash/isEqual'; +import { ICustomWidget } from 'app/common/CustomWidget'; +import { AsyncCreate } from 'app/common/AsyncCreate'; const t = makeT('AppModel'); @@ -75,6 +77,8 @@ export interface TopAppModel { * Reloads orgs and accounts for current user. */ fetchUsersAndOrgs(): Promise; + + getWidgets(): Promise; } /** @@ -143,6 +147,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel { public readonly users = Observable.create(this, []); public readonly plugins: LocalPlugin[] = []; private readonly _gristConfig?: GristLoadConfig; + private readonly _widgets: AsyncCreate; constructor( window: {gristConfig?: GristLoadConfig}, @@ -153,6 +158,7 @@ export class TopAppModelImpl extends Disposable implements TopAppModel { this.isSingleOrg = Boolean(window.gristConfig && window.gristConfig.singleOrg); this.productFlavor = getFlavor(window.gristConfig && window.gristConfig.org); this._gristConfig = window.gristConfig; + this._widgets = new AsyncCreate(() => this.api.getWidgets()); // Initially, and on any change to subdomain, call initialize() to get the full Organization // and the FullUser to use for it (the user may change when switching orgs). @@ -175,6 +181,10 @@ export class TopAppModelImpl extends Disposable implements TopAppModel { } } + public async getWidgets(): Promise { + return this._widgets.get(); + } + public getUntrustedContentOrigin() { if (G.window.isRunningUnderElectron) { // when loaded within webviews it is safe to serve plugin's content from the same domain diff --git a/app/client/models/entities/ViewSectionRec.ts b/app/client/models/entities/ViewSectionRec.ts index f7f1072d..57d60ce7 100644 --- a/app/client/models/entities/ViewSectionRec.ts +++ b/app/client/models/entities/ViewSectionRec.ts @@ -21,7 +21,7 @@ import {removeRule, RuleOwner} from 'app/client/models/RuleOwner'; import {LinkConfig} from 'app/client/ui/selectBy'; import {getWidgetTypes} from "app/client/ui/widgetTypesMap"; import {FilterColValues} from "app/common/ActiveDocAPI"; -import {AccessLevel, ICustomWidget} from 'app/common/CustomWidget'; +import {AccessLevel} from 'app/common/CustomWidget'; import {UserAction} from 'app/common/DocActions'; import {arrayRepeat} from 'app/common/gutil'; import {Sort} from 'app/common/SortSpec'; @@ -245,10 +245,19 @@ export interface CustomViewSectionDef { * The url. */ url: modelUtil.KoSaveableObservable; + /** + * A widgetId, if available. Preferred to url. + * For bundled custom widgets, it is important to refer + * to them by something other than url, since url will + * vary with deployment, and it should be possible to move + * documents between deployments if they have compatible + * widgets available. + */ + widgetId: modelUtil.KoSaveableObservable; /** * Custom widget information. */ - widgetDef: modelUtil.KoSaveableObservable; + // widgetDef: modelUtil.KoSaveableObservable; /** * Custom widget options. */ @@ -324,7 +333,7 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel): const customViewDefaults = { mode: 'url', url: null, - widgetDef: null, + // widgetDef: null, access: '', pluginId: '', sectionId: '', @@ -336,7 +345,8 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel): this.customDef = { mode: customDefObj.prop('mode'), url: customDefObj.prop('url'), - widgetDef: customDefObj.prop('widgetDef'), + widgetId: customDefObj.prop('widgetId'), + // widgetDef: customDefObj.prop('widgetDef'), widgetOptions: customDefObj.prop('widgetOptions'), columnsMapping: customDefObj.prop('columnsMapping'), access: customDefObj.prop('access'), diff --git a/app/client/ui/CustomSectionConfig.ts b/app/client/ui/CustomSectionConfig.ts index 8efff470..2acb4f69 100644 --- a/app/client/ui/CustomSectionConfig.ts +++ b/app/client/ui/CustomSectionConfig.ts @@ -15,7 +15,7 @@ import {IconName} from 'app/client/ui2018/IconList'; import {icon} from 'app/client/ui2018/icons'; import {cssLink} from 'app/client/ui2018/links'; import {IOptionFull, menu, menuItem, menuText, select} from 'app/client/ui2018/menus'; -import {AccessLevel, ICustomWidget, isSatisfied} from 'app/common/CustomWidget'; +import { AccessLevel, ICustomWidget, isSatisfied, matchWidget } from 'app/common/CustomWidget'; import {GristLoadConfig} from 'app/common/gristUrls'; import {unwrap} from 'app/common/gutil'; import { @@ -322,7 +322,8 @@ export class CustomSectionConfig extends Disposable { // Test if we can offer widget list. const gristConfig: GristLoadConfig = (window as any).gristConfig || {}; - this._canSelect = gristConfig.enableWidgetRepository ?? true; + console.log("Ignoring gristConfig now", {gristConfig}); + this._canSelect = true; // gristConfig.enableWidgetRepository ?? true; // Array of available widgets - will be updated asynchronously. this._widgets = Observable.create(this, []); @@ -331,12 +332,16 @@ export class CustomSectionConfig extends Disposable { // Selected value from the dropdown (contains widgetId or "custom" string for Custom URL) this._selectedId = Computed.create(this, use => { - if (use(_section.customDef.widgetDef)) { - return _section.customDef.widgetDef.peek()!.widgetId; + const widgetId = use(_section.customDef.widgetId); + const pluginId = use(_section.customDef.pluginId); + if (widgetId) { + console.log("_selectedId", {widgetId, pluginId}); + return (pluginId||'') + ':' + widgetId; } return CUSTOM_ID; }); this._selectedId.onWrite(async value => { + console.log("_selectedId onWrite", {value}); if (value === CUSTOM_ID) { // Select Custom URL bundleChanges(() => { @@ -344,8 +349,11 @@ export class CustomSectionConfig extends Disposable { _section.customDef.renderAfterReady(false); // Clear url. _section.customDef.url(null); + // Clear widgetId + _section.customDef.widgetId(null); + _section.customDef.pluginId(''); // Clear widget definition. - _section.customDef.widgetDef(null); + // _section.customDef.widgetDef(null); // Reset access level to none. _section.customDef.access(AccessLevel.none); // Clear all saved options. @@ -359,27 +367,50 @@ export class CustomSectionConfig extends Disposable { }); await _section.saveCustomDef(); } else { + const [pluginId, widgetId] = value?.split(':') || []; // Select Widget - const selectedWidget = this._widgets.get().find(w => w.widgetId === value); + console.log("Start match"); + const selectedWidget = matchWidget(this._widgets.get(), { + widgetId, + pluginId, + }); + console.log("Started match"); + console.log("SETTING", {pluginId, widgetId, selectedWidget}); if (!selectedWidget) { // should not happen throw new Error('Error accessing widget from the list'); } // If user selected the same one, do nothing. - if (_section.customDef.widgetDef.peek()?.widgetId === value) { + if (_section.customDef.widgetId.peek() === widgetId && + _section.customDef.pluginId.peek() === pluginId) { + console.log("DO NOTHING", { + widgetId, + pluginId, + owidgetId: _section.customDef.widgetId.peek(), + opluginId: _section.customDef.pluginId.peek(), + }); return; } bundleChanges(() => { // Reset whether widget should render after `grist.ready()`. - _section.customDef.renderAfterReady(false); + _section.customDef.renderAfterReady(selectedWidget.renderAfterReady ?? false); // Clear access level _section.customDef.access(AccessLevel.none); // When widget wants some access, set desired access level. this._desiredAccess.set(selectedWidget.accessLevel || AccessLevel.none); // Update widget definition. - _section.customDef.widgetDef(selectedWidget); + // _section.customDef.widgetDef(selectedWidget); + // Update widgetId. + _section.customDef.widgetId(selectedWidget.widgetId); + _section.customDef.pluginId(selectedWidget.fromPlugin || ''); + console.log({ + setty: 1, + widgetId: selectedWidget.widgetId, + pluginId: selectedWidget.fromPlugin || '', + selectedWidget + }); // Update widget URL. - _section.customDef.url(selectedWidget.url); + _section.customDef.url(null); // Clear options. _section.customDef.widgetOptions(null); // Clear has custom configuration. @@ -389,6 +420,7 @@ export class CustomSectionConfig extends Disposable { _section.columnsToMap(null); }); await _section.saveCustomDef(); + console.log("CustomSectionConfig saved"); } }); @@ -398,7 +430,12 @@ export class CustomSectionConfig extends Disposable { this._url.onWrite(async newUrl => { bundleChanges(() => { _section.customDef.renderAfterReady(false); - _section.customDef.url(newUrl); + if (newUrl) { + console.log("ZAP widgetId and pluginId"); + _section.customDef.widgetId(null); + _section.customDef.pluginId(''); + } + //_section.customDef.url(newUrl); }); await _section.saveCustomDef(); }); @@ -423,6 +460,11 @@ export class CustomSectionConfig extends Disposable { const holder = new MultiHolder(); // Show prompt, when desired access level is different from actual one. + function makeLabel(widget: ICustomWidget) { + if (!widget.fromPlugin) { return widget.name; } + const group = widget.fromPlugin.replace('builtIn/', ''); + return `${widget.name} (${group})`; + } const prompt = Computed.create(holder, use => use(this._desiredAccess) && !isSatisfied(use(this._currentAccess), use(this._desiredAccess)!)); @@ -433,7 +475,9 @@ export class CustomSectionConfig extends Disposable { // Options for the select-box (all widgets definitions and Custom URL) const options = Computed.create(holder, use => [ {label: 'Custom URL', value: 'custom'}, - ...use(this._widgets).map(w => ({label: w.name, value: w.widgetId})), + ...use(this._widgets).map(w => ({ + label: makeLabel(w), value: ((w.fromPlugin||'') + ':' + w.widgetId) + })), ]); function buildPrompt(level: AccessLevel|null) { if (!level) { @@ -469,7 +513,7 @@ export class CustomSectionConfig extends Disposable { testId('select') ) : null, - dom.maybe(isCustom && this.shouldRenderWidgetSelector(), () => [ + dom.maybe((use) => use(isCustom) && this.shouldRenderWidgetSelector(), () => [ cssRow( cssTextInput( this._url, @@ -538,17 +582,23 @@ export class CustomSectionConfig extends Disposable { } protected async _getWidgets() { - const api = this._gristDoc.app.topAppModel.api; - const wigets = await api.getWidgets(); + const widgets = await this._gristDoc.app.topAppModel.getWidgets(); + /* + const widgets = filterWidgets(widgets1, { + keepWidgetIdUnique: true, + preferPlugin: false, + }); + */ + // const wigets = await api.getWidgets(); // Request for rest of the widgets. if (this._canSelect) { // From the start we will provide single widget definition // that was chosen previously. - if (this._section.customDef.widgetDef.peek()) { - wigets.push(this._section.customDef.widgetDef.peek()!); - } + // if (this._section.customDef.widgetDef.peek()) { + // wigets.push(this._section.customDef.widgetDef.peek()!); + // } } - this._widgets.set(wigets); + this._widgets.set(widgets); } diff --git a/app/client/ui2018/menus.ts b/app/client/ui2018/menus.ts index 88d6f0c1..4eafce6d 100644 --- a/app/client/ui2018/menus.ts +++ b/app/client/ui2018/menus.ts @@ -220,6 +220,7 @@ export function multiSelect(selectedOptions: MutableObsArray, }, dom.domComputed(selectedOptionsSet, selectedOpts => { return dom.forEach(availableOptions, option => { + console.log(">>> option", {availableOptions}); const fullOption = weasel.getOptionFull(option); return cssCheckboxLabel( cssCheckboxSquare( diff --git a/app/common/CustomWidget.ts b/app/common/CustomWidget.ts index 115e0a2b..9b1a9b4f 100644 --- a/app/common/CustomWidget.ts +++ b/app/common/CustomWidget.ts @@ -1,3 +1,5 @@ +import sortBy = require('lodash/sortBy'); + /** * Custom widget manifest definition. */ @@ -8,6 +10,9 @@ export interface ICustomWidget { name: string; /** * Widget unique id, probably in npm package format @gristlabs/custom-widget-name. + * + * There could be multiple versions of the same widget with the + * same id, e.g. a bundled version and an external version. */ widgetId: string; /** @@ -25,6 +30,8 @@ export interface ICustomWidget { * a chance to apply the Grist theme. */ renderAfterReady?: boolean; + + fromPlugin?: string; } /** @@ -56,3 +63,65 @@ export function isSatisfied(current: AccessLevel, minimum: AccessLevel) { } return ordered(current) >= ordered(minimum); } + +export function matchWidget(widgets: ICustomWidget[], options: { + widgetId?: string, + pluginId?: string, +}): ICustomWidget|undefined { + console.log("MATCHING", { + widgets, + options, + }); + const prefs = sortBy(widgets, (w) => { + return [w.widgetId !== options.widgetId, + (w.fromPlugin||'') !== options.pluginId] + }); + if (prefs.length === 0) { return; } + if (options.widgetId && prefs[0].widgetId !== options.widgetId) { + return; + } + console.log("ORDERED", prefs); + console.log("MATCHED", prefs[0]); + return prefs[0]; +} + +export function filterWidgets(widgets: ICustomWidget[], options: { + preferPlugin?: boolean, + keepWidgetIdUnique?: boolean, +}) { + const folders = new Map(); + for (const widget of widgets) { + const widgetId = widget.widgetId; + if (!folders.has(widgetId)) { folders.set(widgetId, []); } + const widgetFolder = folders.get(widgetId)!; + widgetFolder.push(widget); + } + let finalResults: ICustomWidget[] = widgets; + if (options.preferPlugin !== undefined) { + const results = []; + const seen = new Set(); + for (const widget of widgets) { + const folder = folders.get(widget.widgetId)!; + if (folder.length === 1) { + results.push(widget); + continue; + } + if (seen.has(widget.widgetId)) { continue; } + seen.add(widget.widgetId); + const folderSorted = sortBy(folder, (w) => Boolean(w.fromPlugin) !== options.preferPlugin); + results.push(folderSorted[0]!); + } + finalResults = results; + } + if (options.keepWidgetIdUnique) { + const results = []; + const seen = new Set(); + for (const widget of widgets) { + if (seen.has(widget.widgetId)) { continue; } + seen.add(widget.widgetId); + results.push(widget); + } + finalResults = results; + } + return finalResults; +} diff --git a/app/plugin/PluginManifest-ti.ts b/app/plugin/PluginManifest-ti.ts index 04ae64f8..bd459d3d 100644 --- a/app/plugin/PluginManifest-ti.ts +++ b/app/plugin/PluginManifest-ti.ts @@ -14,6 +14,7 @@ export const BarePlugin = t.iface([], { "safeBrowser": t.opt("string"), "safePython": t.opt("string"), "unsafeNode": t.opt("string"), + "widgets": t.opt("string"), "deactivate": t.opt(t.iface([], { "inactivitySec": t.opt("number"), })), diff --git a/app/plugin/PluginManifest.ts b/app/plugin/PluginManifest.ts index c4d60aba..fd6a94a7 100644 --- a/app/plugin/PluginManifest.ts +++ b/app/plugin/PluginManifest.ts @@ -82,6 +82,8 @@ export interface BarePlugin { */ unsafeNode?: string; + widgets?: string; + /** * Options for when to deactivate the plugin, i.e. when to stop any plugin processes. (Note * that we may in the future also add options for when to activate the plugin, which is for diff --git a/app/server/devServerMain.ts b/app/server/devServerMain.ts index 7e7012a5..296f24b7 100644 --- a/app/server/devServerMain.ts +++ b/app/server/devServerMain.ts @@ -34,22 +34,6 @@ function getPort(envVarName: string, fallbackPort: number): number { return val ? parseInt(val, 10) : fallbackPort; } -// Checks whether to serve user content on same domain but on different port -function checkUserContentPort(): number | null { - if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) { - const homeUrl = new URL(process.env.APP_HOME_URL); - const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL); - // If the hostname of both home and plugin url are the same, - // but the ports are different - if (homeUrl.hostname === pluginUrl.hostname && - homeUrl.port !== pluginUrl.port) { - const port = parseInt(pluginUrl.port || '80', 10); - return port; - } - } - return null; -} - export async function main() { log.info("=========================================================================="); log.info("== devServer"); @@ -114,14 +98,6 @@ export async function main() { } const server = await mergedServerMain(port, ["home", "docs", "static"]); await server.addTestingHooks(); - // If plugin content is served from same host but on different port, - // run webserver on that port - const userPort = checkUserContentPort(); - if (userPort !== null) { - log.info("=========================================================================="); - log.info("== userContent"); - await server.startCopy('pluginServer', userPort); - } return; } @@ -155,15 +131,6 @@ export async function main() { await home.startCopy('webServer', webServerPort); } - // If plugin content is served from same host but on different port, - // run webserver on that port - const userPort = checkUserContentPort(); - if (userPort !== null) { - log.info("=========================================================================="); - log.info("== userContent"); - await home.startCopy('pluginServer', userPort); - } - // Bring up the docWorker(s) log.info("=========================================================================="); log.info("== docWorker"); diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 2f7674f9..940d7dda 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1,4 +1,5 @@ import {ApiError} from 'app/common/ApiError'; +import { ICustomWidget } from 'app/common/CustomWidget'; import {delay} from 'app/common/delay'; import {DocCreationInfo} from 'app/common/DocListAPI'; import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes, @@ -65,7 +66,7 @@ import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry'; import {startTestingHooks} from 'app/server/lib/TestingHooks'; import {getTestLoginSystem} from 'app/server/lib/TestLogin'; import {addUploadRoute} from 'app/server/lib/uploads'; -import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository'; +import { buildWidgetRepository, getWidgetPlaces, IWidgetRepository} from 'app/server/lib/WidgetRepository'; import {setupLocale} from 'app/server/localization'; import axios from 'axios'; import * as bodyParser from 'body-parser'; @@ -128,6 +129,9 @@ export class FlexServer implements GristServer { private _dbManager: HomeDBManager; private _defaultBaseDomain: string|undefined; private _pluginUrl: string|undefined; + private _pluginUrlSet: boolean = false; + private _willServePlugins?: boolean; + private _bundledWidgets?: ICustomWidget[]; private _billing: IBilling; private _instanceRoot: string; private _docManager: DocManager; @@ -220,7 +224,6 @@ export class FlexServer implements GristServer { } this.info.push(['defaultBaseDomain', this._defaultBaseDomain]); this._pluginUrl = options.pluginUrl || process.env.APP_UNTRUSTED_URL; - this.info.push(['pluginUrl', this._pluginUrl]); // The electron build is not supported at this time, but this stub // implementation of electronServerMethods is present to allow kicking @@ -551,19 +554,36 @@ export class FlexServer implements GristServer { this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) => res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); // Plugins get access to static resources without a tag - this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'static')))); - this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'bower_components')))); + this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'static')))); + this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'bower_components')))); // Serve custom-widget.html message for anyone. this.app.use(/^\/(custom-widget.html)$/, expressWrap(async (req, res) => res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')}))); this.addOrg(); addPluginEndpoints(this, await this._addPluginManager()); + const places = getWidgetPlaces(this, 'wotnot'); + // Allow static files to be requested from any origin. + const options: serveStatic.ServeStaticOptions = { + setHeaders: (res, filepath, stat) => { + res.setHeader("Access-Control-Allow-Origin", "*"); + } + }; + for (const place of places) { + this.app.use( + '/widgets/' + place.name, + this.tagChecker.withTag( + limitToPlugins(this, + express.static(place.fileDir, options) + ) + ) + ); + } } // Prepare cache for managing org-to-host relationship. public addHosts() { if (this._check('hosts', 'homedb')) { return; } - this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this._pluginUrl); + this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this); } public async initHomeDBManager() { @@ -671,7 +691,7 @@ export class FlexServer implements GristServer { // ApiServer's constructor adds endpoints to the app. // tslint:disable-next-line:no-unused-expression - new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository()); + new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository(this)); } public addBillingApi() { @@ -1425,6 +1445,56 @@ export class FlexServer implements GristServer { }); } + public setPluginPort(port: number) { + const url = new URL(this.getOwnUrl()); + url.port = String(port); + this._pluginUrl = url.href; + } + + public willServePlugins() { + if (this._willServePlugins === undefined) { + throw new Error('do not know if will serve plugins'); + } + return this._willServePlugins; + } + + public setWillServePlugins(flag: boolean) { + this._willServePlugins = flag; + } + + public getPluginUrl() { + if (!this._pluginUrlSet) { + throw new Error('looked at plugin url too early'); + } + return this._pluginUrl; + } + + public getPlugins() { + if (!this._pluginManager) { + throw new Error('plugin manager not available'); + } + return this._pluginManager.getPlugins(); + } + + public async prepareSummary() { + // Add some information that isn't guaranteed set until the end. + + this.info.push(['pluginUrl', this._pluginUrl]); + // plugin url should be finalized by now. + this._pluginUrlSet = true; + this.info.push(['willServePlugins', this._willServePlugins]); + + const repo = buildWidgetRepository(this, { localOnly: true }); + this._bundledWidgets = await repo.getWidgets(); + } + + public getBundledWidgets(): ICustomWidget[] { + if (!this._bundledWidgets) { + throw new Error('bundled widgets accessed too early'); + } + return this._bundledWidgets; + } + public summary() { for (const [label, value] of this.info) { log.info("== %s: %s", label, value); @@ -1538,9 +1608,12 @@ export class FlexServer implements GristServer { await this.housekeeper.start(); } - public async startCopy(name2: string, port2: number) { + public async startCopy(name2: string, port2: number): Promise<{ + serverPort: number, + httpsServerPort?: number, + }>{ const servers = this._createServers(); - await this._startServers(servers.server, servers.httpsServer, name2, port2, true); + return this._startServers(servers.server, servers.httpsServer, name2, port2, true); } /** @@ -1606,6 +1679,9 @@ export class FlexServer implements GristServer { } public getTag(): string { + if (!this.tag) { + throw new Error('getTag called too early'); + } return this.tag; } @@ -1913,6 +1989,10 @@ export class FlexServer implements GristServer { await listenPromise(httpsServer.listen(httpsPort, this.host)); if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); } } + return { + serverPort: (server.address() as AddressInfo).port, + httpsServerPort: (server.address() as AddressInfo)?.port, + }; } private async _recordNewUserInfo(row: object) { diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts index 7ba31c38..7629d61b 100644 --- a/app/server/lib/GristServer.ts +++ b/app/server/lib/GristServer.ts @@ -1,4 +1,6 @@ +import { ICustomWidget } from 'app/common/CustomWidget'; import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls'; +import { LocalPlugin } from 'app/common/plugin'; import { FullUser, UserProfile } from 'app/common/UserAPI'; import { Document } from 'app/gen-server/entity/Document'; import { Organization } from 'app/gen-server/entity/Organization'; @@ -53,6 +55,10 @@ export interface GristServer { sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise; getAccessTokens(): IAccessTokens; resolveLoginSystem(): Promise; + getPluginUrl(): string|undefined; + getPlugins(): LocalPlugin[]; + willServePlugins(): boolean; + getBundledWidgets(): ICustomWidget[]; } export interface GristLoginSystem { @@ -135,6 +141,10 @@ export function createDummyGristServer(): GristServer { sendAppPage() { return Promise.resolve(); }, getAccessTokens() { throw new Error('no access tokens'); }, resolveLoginSystem() { throw new Error('no login system'); }, + getPluginUrl() { return undefined; }, + willServePlugins() { return false; }, + getPlugins() { return []; }, + getBundledWidgets() { return []; }, }; } diff --git a/app/server/lib/PluginEndpoint.ts b/app/server/lib/PluginEndpoint.ts index 7afdc6be..fb427fff 100644 --- a/app/server/lib/PluginEndpoint.ts +++ b/app/server/lib/PluginEndpoint.ts @@ -1,34 +1,36 @@ import {FlexServer} from 'app/server/lib/FlexServer'; +import {GristServer} from 'app/server/lib/GristServer'; import log from 'app/server/lib/log'; import {PluginManager} from 'app/server/lib/PluginManager'; import * as express from 'express'; import * as mimeTypes from 'mime-types'; import * as path from 'path'; -// Get the url where plugin material should be served from. -export function getUntrustedContentOrigin(): string|undefined { - return process.env.APP_UNTRUSTED_URL; -} - // Get the host serving plugin material -export function getUntrustedContentHost(): string|undefined { - const origin = getUntrustedContentOrigin(); +export function getUntrustedContentHost(origin: string|undefined): string|undefined { if (!origin) { return; } return new URL(origin).host; } // Add plugin endpoints to be served on untrusted host export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) { - const host = getUntrustedContentHost(); - if (host) { + if (server.willServePlugins()) { server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) => - servePluginContent(req, res, pluginManager, host)); + servePluginContent(req, res, pluginManager, server)); } } // Serve content for plugins with various checks that it is being accessed as we expect. function servePluginContent(req: express.Request, res: express.Response, - pluginManager: PluginManager, untrustedContentHost: string) { + pluginManager: PluginManager, + gristServer: GristServer) { + const pluginUrl = gristServer.getPluginUrl(); + const untrustedContentHost = getUntrustedContentHost(pluginUrl); + if (!untrustedContentHost) { + // not expected + throw new Error('plugin host unexpectedly not set'); + } + const pluginKind = req.params[0]; const pluginId = req.params[1]; const pluginPath = req.params[2]; @@ -56,9 +58,11 @@ function servePluginContent(req: express.Request, res: express.Response, } // Middleware to restrict some assets to untrusted host. -export function limitToPlugins(handler: express.RequestHandler) { - const host = getUntrustedContentHost(); +export function limitToPlugins(gristServer: GristServer, + handler: express.RequestHandler) { return function(req: express.Request, resp: express.Response, next: express.NextFunction) { + const pluginUrl = gristServer.getPluginUrl(); + const host = getUntrustedContentHost(pluginUrl); if (!host) { return next(); } if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") { return handler(req, resp, next); diff --git a/app/server/lib/PluginManager.ts b/app/server/lib/PluginManager.ts index 712838b9..0939e127 100644 --- a/app/server/lib/PluginManager.ts +++ b/app/server/lib/PluginManager.ts @@ -131,6 +131,7 @@ export class PluginManager { async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise { + console.log("SCAN", {dir, kind}); const plugins: DirectoryScanEntry[] = []; let listDir; diff --git a/app/server/lib/WidgetRepository.ts b/app/server/lib/WidgetRepository.ts index 1115fcd2..ed482051 100644 --- a/app/server/lib/WidgetRepository.ts +++ b/app/server/lib/WidgetRepository.ts @@ -1,8 +1,14 @@ import {ICustomWidget} from 'app/common/CustomWidget'; import log from 'app/server/lib/log'; +import * as fse from 'fs-extra'; import fetch from 'node-fetch'; +import * as path from 'path'; import {ApiError} from 'app/common/ApiError'; import LRUCache from 'lru-cache'; +import * as url from 'url'; +import { removeTrailingSlash } from 'app/common/gutil'; +import { GristServer } from './GristServer'; +// import { LocalPlugin } from 'app/common/plugin'; /** * Widget Repository returns list of available Custom Widgets. @@ -14,25 +20,89 @@ export interface IWidgetRepository { // Static url for StaticWidgetRepository const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL; -/** - * Default repository that gets list of available widgets from a static URL. - */ -export class WidgetRepositoryImpl implements IWidgetRepository { - constructor(protected _staticUrl = STATIC_URL) {} +export class FileWidgetRepository implements IWidgetRepository { + constructor(private _widgetFileName: string, + private _widgetBaseUrl: string, + private _pluginId?: string) {} - /** - * Method exposed for testing, overrides widget url. - */ - public testOverrideUrl(url: string) { - this._staticUrl = url; + public async getWidgets(): Promise { + const txt = await fse.readFile(this._widgetFileName, { + encoding: 'utf8', + }); + const widgets: ICustomWidget[] = JSON.parse(txt); + fixUrls(widgets, this._widgetBaseUrl); + if (this._pluginId) { + for (const widget of widgets) { + widget.fromPlugin = this._pluginId; + } + } + console.log("FileWidget", {widgets}); + return widgets; } +} + +/* +export class NestedWidgetRepository implements IWidgetRepository { + constructor(private _widgetDir: string, + private _widgetBaseUrl: string) {} + + public async getWidgets(): Promise { + const listDir = await fse.readdir(this._widgetDir, + { withFileTypes: true }); + const fileName = 'manifest.json'; + const allWidgets: ICustomWidget[] = []; + for (const dir of listDir) { + if (!dir.isDirectory()) { continue; } + const fullPath = path.join(this._widgetDir, dir.name, fileName); + if (!await fse.pathExists(fullPath)) { continue; } + const txt = await fse.readFile(fullPath, 'utf8'); + const widgets = JSON.parse(txt); + fixUrls( + widgets, + removeTrailingSlash(this._widgetBaseUrl) + '/' + dir.name + '/' + ); + allWidgets.push(...widgets); + } + return allWidgets; + } +} +*/ + +export class DelayedWidgetRepository implements IWidgetRepository { + constructor(private _makeRepo: () => Promise) {} + + public async getWidgets(): Promise { + const repo = await this._makeRepo(); + if (!repo) { return []; } + return repo.getWidgets(); + } +} + +export class CombinedWidgetRepository implements IWidgetRepository { + constructor(private _repos: IWidgetRepository[]) {} + + public async getWidgets(): Promise { + const allWidgets: ICustomWidget[] = []; + for (const repo of this._repos) { + allWidgets.push(...await repo.getWidgets()); + } + console.log("COMBINED", {allWidgets}); + return allWidgets; + } +} + +/** + * Repository that gets list of available widgets from a static URL. + */ +export class UrlWidgetRepository implements IWidgetRepository { + constructor(private _staticUrl = STATIC_URL) {} public async getWidgets(): Promise { if (!this._staticUrl) { log.warn( - 'WidgetRepository: Widget repository is not configured.' + !STATIC_URL + 'WidgetRepository: Widget repository is not configured.' + (!STATIC_URL ? ' Missing GRIST_WIDGET_LIST_URL environmental variable.' - : '' + : '') ); return []; } @@ -52,6 +122,7 @@ export class WidgetRepositoryImpl implements IWidgetRepository { if (!widgets || !Array.isArray(widgets)) { throw new ApiError('WidgetRepository: Error reading widget list', 500); } + fixUrls(widgets, this._staticUrl); return widgets; } catch (err) { if (!(err instanceof ApiError)) { @@ -62,6 +133,56 @@ export class WidgetRepositoryImpl implements IWidgetRepository { } } +/** + * Default repository that gets list of available widgets from a static URL. + */ +export class WidgetRepositoryImpl implements IWidgetRepository { + protected _staticUrl: string|undefined; + private _urlWidgets: UrlWidgetRepository; + private _combinedWidgets: CombinedWidgetRepository; + private _dirWidgets?: IWidgetRepository; + + constructor(_options: { + staticUrl?: string, + gristServer?: GristServer, + }) { + const {staticUrl, gristServer} = _options; + if (gristServer) { + this._dirWidgets = new DelayedWidgetRepository(async () => { + const places = getWidgetPlaces(gristServer); + console.log("PLACES!", places); + const files = places.map(place => new FileWidgetRepository(place.fileBase, + place.urlBase, + place.pluginId)); + return new CombinedWidgetRepository(files); + }); + } + this.testSetUrl(staticUrl); + } + + /** + * Method exposed for testing, overrides widget url. + */ + public testOverrideUrl(url: string|undefined) { + this.testSetUrl(url); + } + + public testSetUrl(url: string|undefined) { + const repos: IWidgetRepository[] = []; + this._staticUrl = url ?? STATIC_URL; + if (this._staticUrl) { + this._urlWidgets = new UrlWidgetRepository(this._staticUrl); + repos.push(this._urlWidgets); + } + if (this._dirWidgets) { repos.push(this._dirWidgets); } + this._combinedWidgets = new CombinedWidgetRepository(repos); + } + + public async getWidgets(): Promise { + return this._combinedWidgets.getWidgets(); + } +} + /** * Version of WidgetRepository that caches successful result for 2 minutes. */ @@ -79,6 +200,7 @@ class CachedWidgetRepository extends WidgetRepositoryImpl { const list = await super.getWidgets(); // Cache only if there are some widgets. if (list.length) { this._cache.set(1, list); } + console.log("CACHABLE RESULT", {list}); return list; } @@ -91,6 +213,66 @@ class CachedWidgetRepository extends WidgetRepositoryImpl { /** * Returns widget repository implementation. */ -export function buildWidgetRepository() { - return new CachedWidgetRepository(); +export function buildWidgetRepository(gristServer: GristServer, + options?: { + localOnly: boolean + }) { + if (options?.localOnly) { + return new WidgetRepositoryImpl({ + gristServer, + staticUrl: '' + }); + } + return new CachedWidgetRepository({ + gristServer, + }); +} + +function fixUrls(widgets: ICustomWidget[], baseUrl: string) { + // If URLs are relative, make them absolute, interpreting them + // relative to the manifest file. + for (const widget of widgets) { + if (!(url.parse(widget.url).protocol)) { + widget.url = new URL(widget.url, baseUrl).href; + } + } +} + +export interface CustomWidgetPlace { + urlBase: string, + fileBase: string, + fileDir: string, + name: string, + pluginId: string, +} + +export function getWidgetPlaces(gristServer: GristServer, + pluginUrl?: string) { + const places: CustomWidgetPlace[] = []; + const plugins = gristServer.getPlugins(); + console.log("PLUGINS", plugins); + pluginUrl = pluginUrl || gristServer.getPluginUrl(); + if (!pluginUrl) { return []; } + for (const plugin of plugins) { + console.log("PLUGIN", plugin); + const components = plugin.manifest.components; + if (!components.widgets) { continue; } + console.log("GOT SOMETHING", { + name: plugin.id, + path: plugin.path, + widgets: components.widgets + }); + const urlBase = + removeTrailingSlash(pluginUrl) + '/v/' + + gristServer.getTag() + '/widgets/' + plugin.id + '/'; + places.push({ + urlBase, + fileBase: path.join(plugin.path, components.widgets), + fileDir: plugin.path, + name: plugin.id, + pluginId: plugin.id, + }); + } + console.log("PLACES", places); + return places; } diff --git a/app/server/lib/extractOrg.ts b/app/server/lib/extractOrg.ts index 0d3454b6..dc8783c7 100644 --- a/app/server/lib/extractOrg.ts +++ b/app/server/lib/extractOrg.ts @@ -6,6 +6,7 @@ import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager'; import { getOriginUrl } from 'app/server/lib/requestUtils'; import { NextFunction, Request, RequestHandler, Response } from 'express'; import { IncomingMessage } from 'http'; +import { GristServer } from './GristServer'; // How long we cache information about the relationship between // orgs and custom hosts. The higher this is, the fewer requests @@ -41,7 +42,7 @@ export class Hosts { // baseDomain should start with ".". It may be undefined for localhost or single-org mode. constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager, - private _pluginUrl: string|undefined) { + private _gristServer: GristServer|undefined) { } /** @@ -165,6 +166,6 @@ export class Hosts { } private _getHostType(host: string) { - return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._pluginUrl}); + return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._gristServer?.getPluginUrl()}); } } diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts index cf66c59a..81c3c301 100644 --- a/app/server/lib/sendAppPage.ts +++ b/app/server/lib/sendAppPage.ts @@ -33,7 +33,7 @@ export interface ISendAppPageOptions { googleTagManager?: true | false | 'anon'; } -export interface MakeGristConfigOptons { +export interface MakeGristConfigOptions { homeUrl: string|null; extra: Partial; baseDomain?: string; @@ -41,7 +41,7 @@ export interface MakeGristConfigOptons { server?: GristServer|null; } -export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig { +export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfig { const {homeUrl, extra, baseDomain, req, server} = options; // .invalid is a TLD the IETF promises will never exist. const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid'; @@ -78,7 +78,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig featureComments: isAffirmative(process.env.COMMENTS), featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT), assistantService: process.env.OPENAI_API_KEY ? 'OpenAI' : undefined, - permittedCustomWidgets: getPermittedCustomWidgets(), + permittedCustomWidgets: getPermittedCustomWidgets(server), supportEmail: SUPPORT_EMAIL, userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale, telemetry: server?.getTelemetry().getTelemetryConfig(), @@ -115,6 +115,7 @@ export function makeSendAppPage(opts: { }) { const {server, staticDir, tag, testLogin} = opts; return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => { + console.log("HERE WE GO"); const config = makeGristConfig({ homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null, extra: options.config, @@ -170,9 +171,30 @@ function getFeatures(): IFeature[] { return Features.checkAll(difference(enabledFeatures, disabledFeatures)); } -function getPermittedCustomWidgets(): IAttachedCustomWidget[] { +function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCustomWidget[] { + if (!process.env.PERMITTED_CUSTOM_WIDGETS && gristServer) { + console.log("*****"); + console.log("*****"); + console.log("*****"); + const widgets = gristServer.getBundledWidgets(); + const names = new Set(AttachedCustomWidgets.values as string[]); + console.log({widgets, names}); + const namesFound: IAttachedCustomWidget[] = []; + for (const widget of widgets) { + // For some reason in different parts of the code attached custom + // widgets are identified by a lot of variants of their name or id + // e.g. "Calendar", "calendar", "custom.calendar", "@gristlabs/grist-calendar'... + const name = widget.widgetId.replace('@gristlabs/widget-', 'custom.'); + console.log("CHECK", {name}); + if (names.has(name)) { + console.log("CHECK FOUND", {name}); + namesFound.push(name as IAttachedCustomWidget); + } + } + return AttachedCustomWidgets.checkAll(namesFound); + } const widgetsList = process.env.PERMITTED_CUSTOM_WIDGETS?.split(',').map(widgetName=>`custom.${widgetName}`) ?? []; - return AttachedCustomWidgets.checkAll(widgetsList); + return AttachedCustomWidgets.checkAll(widgetsList); } function configuredPageTitleSuffix() { diff --git a/app/server/mergedServerMain.ts b/app/server/mergedServerMain.ts index 2512931b..3cf7e749 100644 --- a/app/server/mergedServerMain.ts +++ b/app/server/mergedServerMain.ts @@ -31,6 +31,26 @@ export function parseServerTypes(serverTypes: string|undefined): ServerType[] { return types as ServerType[]; } +function checkUserContentPort(): number | null { + // Check whether a port is explicitly set for user content. + if (process.env.GRIST_UNTRUSTED_PORT) { + return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10); + } + // Checks whether to serve user content on same domain but on different port + if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) { + const homeUrl = new URL(process.env.APP_HOME_URL); + const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL); + // If the hostname of both home and plugin url are the same, + // but the ports are different + if (homeUrl.hostname === pluginUrl.hostname && + homeUrl.port !== pluginUrl.port) { + const port = parseInt(pluginUrl.port || '80', 10); + return port; + } + } + return null; +} + interface ServerOptions extends FlexServerOptions { logToConsole?: boolean; // If set, messages logged to console (default: false) // (but if options are not given at all in call to main, @@ -52,6 +72,13 @@ export async function main(port: number, serverTypes: ServerType[], const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options); + if (includeHome) { + const userPort = checkUserContentPort(); + server.setWillServePlugins(userPort !== undefined); + } else { + server.setWillServePlugins(false); + } + if (options.loginSystem) { server.setLoginSystem(options.loginSystem); } @@ -143,7 +170,26 @@ export async function main(port: number, serverTypes: ServerType[], server.finalize(); + if (includeHome) { + // If plugin content is served from same host but on different port, + // run webserver on that port + const userPort = checkUserContentPort(); + if (userPort !== null) { + const ports = await server.startCopy('pluginServer', userPort); + // If Grist is running on a desktop, directly on the host, it + // can be convenient to leave the user port free for the OS to + // allocate by using GRIST_UNTRUSTED_PORT=0. But we do need to + // remember how to contact it. + if (userPort === 0) { + server.setPluginPort(ports.serverPort); + } else if (process.env.APP_UNTRUSTED_URL === undefined) { + server.setPluginPort(userPort); + } + } + } + server.checkOptionCombinations(); + await server.prepareSummary(); server.summary(); return server; } catch(e) { diff --git a/package.json b/package.json index ae4c7f75..aeced43c 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "@types/double-ended-queue": "2.1.0", "@types/express": "4.16.0", "@types/form-data": "2.2.1", - "@types/fs-extra": "5.0.4", + "@types/fs-extra": "11.0.2", "@types/http-proxy": "1.17.9", "@types/i18next-fs-backend": "1.1.2", "@types/image-size": "0.0.29", @@ -139,7 +139,7 @@ "exceljs": "4.2.1", "express": "4.16.4", "file-type": "16.5.4", - "fs-extra": "7.0.0", + "fs-extra": "11.1.1", "grain-rpc": "0.1.7", "grainjs": "1.0.2", "handlebars": "4.7.7", diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts index 3ddaa422..694fa06d 100644 --- a/stubs/app/server/server.ts +++ b/stubs/app/server/server.ts @@ -107,6 +107,11 @@ export async function main() { // Set directory for uploaded documents. setDefaultEnv('GRIST_DATA_DIR', 'docs'); setDefaultEnv('GRIST_SERVERS', 'home,docs,static'); + if (process.env.GRIST_SERVERS?.includes('home')) { + // By default, we will now start an untrusted port alongside a + // home server. Suppress with GRIST_UNTRUSTED_PORT='' + setDefaultEnv('GRIST_UNTRUSTED_PORT', '0'); + } const serverTypes = parseServerTypes(process.env.GRIST_SERVERS); await fse.mkdirp(process.env.GRIST_DATA_DIR!); diff --git a/yarn.lock b/yarn.lock index 3f82821d..3a6867c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -721,11 +721,12 @@ dependencies: "@types/node" "*" -"@types/fs-extra@5.0.4": - version "5.0.4" - resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-5.0.4.tgz" - integrity sha512-DsknoBvD8s+RFfSGjmERJ7ZOP1HI0UZRA3FSI+Zakhrc/Gy26YQsLI+m5V5DHxroHRJqCDLKJp7Hixn8zyaF7g== +"@types/fs-extra@11.0.2": + version "11.0.2" + resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-11.0.2.tgz#23dc1ed7b2eba8ccd75568ac34e7a4e48aa2d087" + integrity sha512-c0hrgAOVYr21EX8J0jBMXGLMgJqVf/v6yxi0dLaJboW9aQPh16Id+z6w2Tx1hm+piJOLv8xPfVKZCLfjPw/IMQ== dependencies: + "@types/jsonfile" "*" "@types/node" "*" "@types/http-cache-semantics@*": @@ -785,6 +786,13 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" integrity sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ== +"@types/jsonfile@*": + version "6.1.2" + resolved "https://registry.yarnpkg.com/@types/jsonfile/-/jsonfile-6.1.2.tgz#d3b8a3536c5bb272ebee0f784180e456b7691c8f" + integrity sha512-8t92P+oeW4d/CRQfJaSqEwXujrhH4OEeHRjGU3v1Q8mUS8GPF3yiX26sw4svv6faL2HfBtGTe2xWIoVgN3dy9w== + dependencies: + "@types/node" "*" + "@types/jsonwebtoken@7.2.8": version "7.2.8" resolved "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-7.2.8.tgz" @@ -857,9 +865,9 @@ form-data "^3.0.0" "@types/node@*": - version "14.0.1" - resolved "https://registry.npmjs.org/@types/node/-/node-14.0.1.tgz" - integrity sha512-FAYBGwC+W6F9+huFIDtn43cpy7+SzG+atzRiTfdp3inUKL2hXnd4rG8hylJLIh4+hqrQy1P17kvJByE/z825hA== + version "20.7.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.7.2.tgz#0bdc211f8c2438cfadad26dc8c040a874d478aed" + integrity sha512-RcdC3hOBOauLP+r/kRt27NrByYtDjsXyAuSbR87O6xpsvi763WI+5fbSIvYJrXnt9w4RuxhV6eAXfIs7aaf/FQ== "@types/node@^14": version "14.18.21" @@ -3909,14 +3917,14 @@ fs-constants@^1.0.0: resolved "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== -fs-extra@7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.0.tgz" - integrity sha512-EglNDLRpmaTWiD/qraZn6HREAEAHJcJOmxNEYwq6xeMKnVMAy3GUcFB+wXt2C6k4CNvB/mP1y/U3dzvKKj5OtQ== +fs-extra@11.1.1: + version "11.1.1" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.1.1.tgz#da69f7c39f3b002378b0954bb6ae7efdc0876e2d" + integrity sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ== dependencies: - graceful-fs "^4.1.2" - jsonfile "^4.0.0" - universalify "^0.1.0" + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" fs-extra@^4.0.2, fs-extra@^4.0.3: version "4.0.3" @@ -4323,12 +4331,12 @@ graceful-fs@^4.0.0, graceful-fs@^4.1.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.10.tgz#147d3a006da4ca3ce14728c7aefc287c367d7a6c" integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== -graceful-fs@^4.1.2: - version "4.2.4" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz" - integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== +graceful-fs@^4.1.2, graceful-fs@^4.1.6: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== -graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2: +graceful-fs@^4.2.0, graceful-fs@^4.2.2: version "4.2.6" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz" integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== @@ -5247,8 +5255,17 @@ json5@^2.2.1: jsonfile@^4.0.0: version "4.0.0" - resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz" - integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss= + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" + integrity sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg== + optionalDependencies: + graceful-fs "^4.1.6" + +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" optionalDependencies: graceful-fs "^4.1.6" @@ -8384,7 +8401,7 @@ unique-string@^2.0.0: universalify@^0.1.0: version "0.1.2" - resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== universalify@^0.2.0: @@ -8392,6 +8409,11 @@ universalify@^0.2.0: resolved "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz" integrity sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg== +universalify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"