bundling experiments (WIP)

Looking at ways to bundle custom widgets with Grist. WIP,
experimental, everything will need rewrite.
This commit is contained in:
Paul Fitzpatrick
2023-10-03 15:20:57 -04:00
parent 97a84ce6ee
commit fd1734de69
22 changed files with 734 additions and 155 deletions

View File

@@ -1,4 +1,5 @@
import {ApiError} from 'app/common/ApiError';
import { ICustomWidget } from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
@@ -65,7 +66,7 @@ import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {addUploadRoute} from 'app/server/lib/uploads';
import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import { buildWidgetRepository, getWidgetPlaces, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {setupLocale} from 'app/server/localization';
import axios from 'axios';
import * as bodyParser from 'body-parser';
@@ -128,6 +129,9 @@ export class FlexServer implements GristServer {
private _dbManager: HomeDBManager;
private _defaultBaseDomain: string|undefined;
private _pluginUrl: string|undefined;
private _pluginUrlSet: boolean = false;
private _willServePlugins?: boolean;
private _bundledWidgets?: ICustomWidget[];
private _billing: IBilling;
private _instanceRoot: string;
private _docManager: DocManager;
@@ -220,7 +224,6 @@ export class FlexServer implements GristServer {
}
this.info.push(['defaultBaseDomain', this._defaultBaseDomain]);
this._pluginUrl = options.pluginUrl || process.env.APP_UNTRUSTED_URL;
this.info.push(['pluginUrl', this._pluginUrl]);
// The electron build is not supported at this time, but this stub
// implementation of electronServerMethods is present to allow kicking
@@ -551,19 +554,36 @@ export class FlexServer implements GristServer {
this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) =>
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
// Plugins get access to static resources without a tag
this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'static'))));
this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'bower_components'))));
this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'static'))));
this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'bower_components'))));
// Serve custom-widget.html message for anyone.
this.app.use(/^\/(custom-widget.html)$/, expressWrap(async (req, res) =>
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
this.addOrg();
addPluginEndpoints(this, await this._addPluginManager());
const places = getWidgetPlaces(this, 'wotnot');
// Allow static files to be requested from any origin.
const options: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};
for (const place of places) {
this.app.use(
'/widgets/' + place.name,
this.tagChecker.withTag(
limitToPlugins(this,
express.static(place.fileDir, options)
)
)
);
}
}
// Prepare cache for managing org-to-host relationship.
public addHosts() {
if (this._check('hosts', 'homedb')) { return; }
this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this._pluginUrl);
this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this);
}
public async initHomeDBManager() {
@@ -671,7 +691,7 @@ export class FlexServer implements GristServer {
// ApiServer's constructor adds endpoints to the app.
// tslint:disable-next-line:no-unused-expression
new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository());
new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository(this));
}
public addBillingApi() {
@@ -1425,6 +1445,56 @@ export class FlexServer implements GristServer {
});
}
public setPluginPort(port: number) {
const url = new URL(this.getOwnUrl());
url.port = String(port);
this._pluginUrl = url.href;
}
public willServePlugins() {
if (this._willServePlugins === undefined) {
throw new Error('do not know if will serve plugins');
}
return this._willServePlugins;
}
public setWillServePlugins(flag: boolean) {
this._willServePlugins = flag;
}
public getPluginUrl() {
if (!this._pluginUrlSet) {
throw new Error('looked at plugin url too early');
}
return this._pluginUrl;
}
public getPlugins() {
if (!this._pluginManager) {
throw new Error('plugin manager not available');
}
return this._pluginManager.getPlugins();
}
public async prepareSummary() {
// Add some information that isn't guaranteed set until the end.
this.info.push(['pluginUrl', this._pluginUrl]);
// plugin url should be finalized by now.
this._pluginUrlSet = true;
this.info.push(['willServePlugins', this._willServePlugins]);
const repo = buildWidgetRepository(this, { localOnly: true });
this._bundledWidgets = await repo.getWidgets();
}
public getBundledWidgets(): ICustomWidget[] {
if (!this._bundledWidgets) {
throw new Error('bundled widgets accessed too early');
}
return this._bundledWidgets;
}
public summary() {
for (const [label, value] of this.info) {
log.info("== %s: %s", label, value);
@@ -1538,9 +1608,12 @@ export class FlexServer implements GristServer {
await this.housekeeper.start();
}
public async startCopy(name2: string, port2: number) {
public async startCopy(name2: string, port2: number): Promise<{
serverPort: number,
httpsServerPort?: number,
}>{
const servers = this._createServers();
await this._startServers(servers.server, servers.httpsServer, name2, port2, true);
return this._startServers(servers.server, servers.httpsServer, name2, port2, true);
}
/**
@@ -1606,6 +1679,9 @@ export class FlexServer implements GristServer {
}
public getTag(): string {
if (!this.tag) {
throw new Error('getTag called too early');
}
return this.tag;
}
@@ -1913,6 +1989,10 @@ export class FlexServer implements GristServer {
await listenPromise(httpsServer.listen(httpsPort, this.host));
if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); }
}
return {
serverPort: (server.address() as AddressInfo).port,
httpsServerPort: (server.address() as AddressInfo)?.port,
};
}
private async _recordNewUserInfo(row: object) {

View File

@@ -1,4 +1,6 @@
import { ICustomWidget } from 'app/common/CustomWidget';
import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
import { LocalPlugin } from 'app/common/plugin';
import { FullUser, UserProfile } from 'app/common/UserAPI';
import { Document } from 'app/gen-server/entity/Document';
import { Organization } from 'app/gen-server/entity/Organization';
@@ -53,6 +55,10 @@ export interface GristServer {
sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void>;
getAccessTokens(): IAccessTokens;
resolveLoginSystem(): Promise<GristLoginSystem>;
getPluginUrl(): string|undefined;
getPlugins(): LocalPlugin[];
willServePlugins(): boolean;
getBundledWidgets(): ICustomWidget[];
}
export interface GristLoginSystem {
@@ -135,6 +141,10 @@ export function createDummyGristServer(): GristServer {
sendAppPage() { return Promise.resolve(); },
getAccessTokens() { throw new Error('no access tokens'); },
resolveLoginSystem() { throw new Error('no login system'); },
getPluginUrl() { return undefined; },
willServePlugins() { return false; },
getPlugins() { return []; },
getBundledWidgets() { return []; },
};
}

View File

@@ -1,34 +1,36 @@
import {FlexServer} from 'app/server/lib/FlexServer';
import {GristServer} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {PluginManager} from 'app/server/lib/PluginManager';
import * as express from 'express';
import * as mimeTypes from 'mime-types';
import * as path from 'path';
// Get the url where plugin material should be served from.
export function getUntrustedContentOrigin(): string|undefined {
return process.env.APP_UNTRUSTED_URL;
}
// Get the host serving plugin material
export function getUntrustedContentHost(): string|undefined {
const origin = getUntrustedContentOrigin();
export function getUntrustedContentHost(origin: string|undefined): string|undefined {
if (!origin) { return; }
return new URL(origin).host;
}
// Add plugin endpoints to be served on untrusted host
export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) {
const host = getUntrustedContentHost();
if (host) {
if (server.willServePlugins()) {
server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) =>
servePluginContent(req, res, pluginManager, host));
servePluginContent(req, res, pluginManager, server));
}
}
// Serve content for plugins with various checks that it is being accessed as we expect.
function servePluginContent(req: express.Request, res: express.Response,
pluginManager: PluginManager, untrustedContentHost: string) {
pluginManager: PluginManager,
gristServer: GristServer) {
const pluginUrl = gristServer.getPluginUrl();
const untrustedContentHost = getUntrustedContentHost(pluginUrl);
if (!untrustedContentHost) {
// not expected
throw new Error('plugin host unexpectedly not set');
}
const pluginKind = req.params[0];
const pluginId = req.params[1];
const pluginPath = req.params[2];
@@ -56,9 +58,11 @@ function servePluginContent(req: express.Request, res: express.Response,
}
// Middleware to restrict some assets to untrusted host.
export function limitToPlugins(handler: express.RequestHandler) {
const host = getUntrustedContentHost();
export function limitToPlugins(gristServer: GristServer,
handler: express.RequestHandler) {
return function(req: express.Request, resp: express.Response, next: express.NextFunction) {
const pluginUrl = gristServer.getPluginUrl();
const host = getUntrustedContentHost(pluginUrl);
if (!host) { return next(); }
if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") {
return handler(req, resp, next);

View File

@@ -131,6 +131,7 @@ export class PluginManager {
async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise<DirectoryScanEntry[]> {
console.log("SCAN", {dir, kind});
const plugins: DirectoryScanEntry[] = [];
let listDir;

View File

@@ -1,8 +1,14 @@
import {ICustomWidget} from 'app/common/CustomWidget';
import log from 'app/server/lib/log';
import * as fse from 'fs-extra';
import fetch from 'node-fetch';
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import LRUCache from 'lru-cache';
import * as url from 'url';
import { removeTrailingSlash } from 'app/common/gutil';
import { GristServer } from './GristServer';
// import { LocalPlugin } from 'app/common/plugin';
/**
* Widget Repository returns list of available Custom Widgets.
@@ -14,25 +20,89 @@ export interface IWidgetRepository {
// Static url for StaticWidgetRepository
const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
/**
* Default repository that gets list of available widgets from a static URL.
*/
export class WidgetRepositoryImpl implements IWidgetRepository {
constructor(protected _staticUrl = STATIC_URL) {}
export class FileWidgetRepository implements IWidgetRepository {
constructor(private _widgetFileName: string,
private _widgetBaseUrl: string,
private _pluginId?: string) {}
/**
* Method exposed for testing, overrides widget url.
*/
public testOverrideUrl(url: string) {
this._staticUrl = url;
public async getWidgets(): Promise<ICustomWidget[]> {
const txt = await fse.readFile(this._widgetFileName, {
encoding: 'utf8',
});
const widgets: ICustomWidget[] = JSON.parse(txt);
fixUrls(widgets, this._widgetBaseUrl);
if (this._pluginId) {
for (const widget of widgets) {
widget.fromPlugin = this._pluginId;
}
}
console.log("FileWidget", {widgets});
return widgets;
}
}
/*
export class NestedWidgetRepository implements IWidgetRepository {
constructor(private _widgetDir: string,
private _widgetBaseUrl: string) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const listDir = await fse.readdir(this._widgetDir,
{ withFileTypes: true });
const fileName = 'manifest.json';
const allWidgets: ICustomWidget[] = [];
for (const dir of listDir) {
if (!dir.isDirectory()) { continue; }
const fullPath = path.join(this._widgetDir, dir.name, fileName);
if (!await fse.pathExists(fullPath)) { continue; }
const txt = await fse.readFile(fullPath, 'utf8');
const widgets = JSON.parse(txt);
fixUrls(
widgets,
removeTrailingSlash(this._widgetBaseUrl) + '/' + dir.name + '/'
);
allWidgets.push(...widgets);
}
return allWidgets;
}
}
*/
export class DelayedWidgetRepository implements IWidgetRepository {
constructor(private _makeRepo: () => Promise<IWidgetRepository|undefined>) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const repo = await this._makeRepo();
if (!repo) { return []; }
return repo.getWidgets();
}
}
export class CombinedWidgetRepository implements IWidgetRepository {
constructor(private _repos: IWidgetRepository[]) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const allWidgets: ICustomWidget[] = [];
for (const repo of this._repos) {
allWidgets.push(...await repo.getWidgets());
}
console.log("COMBINED", {allWidgets});
return allWidgets;
}
}
/**
* Repository that gets list of available widgets from a static URL.
*/
export class UrlWidgetRepository implements IWidgetRepository {
constructor(private _staticUrl = STATIC_URL) {}
public async getWidgets(): Promise<ICustomWidget[]> {
if (!this._staticUrl) {
log.warn(
'WidgetRepository: Widget repository is not configured.' + !STATIC_URL
'WidgetRepository: Widget repository is not configured.' + (!STATIC_URL
? ' Missing GRIST_WIDGET_LIST_URL environmental variable.'
: ''
: '')
);
return [];
}
@@ -52,6 +122,7 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
if (!widgets || !Array.isArray(widgets)) {
throw new ApiError('WidgetRepository: Error reading widget list', 500);
}
fixUrls(widgets, this._staticUrl);
return widgets;
} catch (err) {
if (!(err instanceof ApiError)) {
@@ -62,6 +133,56 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
}
}
/**
* Default repository that gets list of available widgets from a static URL.
*/
export class WidgetRepositoryImpl implements IWidgetRepository {
protected _staticUrl: string|undefined;
private _urlWidgets: UrlWidgetRepository;
private _combinedWidgets: CombinedWidgetRepository;
private _dirWidgets?: IWidgetRepository;
constructor(_options: {
staticUrl?: string,
gristServer?: GristServer,
}) {
const {staticUrl, gristServer} = _options;
if (gristServer) {
this._dirWidgets = new DelayedWidgetRepository(async () => {
const places = getWidgetPlaces(gristServer);
console.log("PLACES!", places);
const files = places.map(place => new FileWidgetRepository(place.fileBase,
place.urlBase,
place.pluginId));
return new CombinedWidgetRepository(files);
});
}
this.testSetUrl(staticUrl);
}
/**
* Method exposed for testing, overrides widget url.
*/
public testOverrideUrl(url: string|undefined) {
this.testSetUrl(url);
}
public testSetUrl(url: string|undefined) {
const repos: IWidgetRepository[] = [];
this._staticUrl = url ?? STATIC_URL;
if (this._staticUrl) {
this._urlWidgets = new UrlWidgetRepository(this._staticUrl);
repos.push(this._urlWidgets);
}
if (this._dirWidgets) { repos.push(this._dirWidgets); }
this._combinedWidgets = new CombinedWidgetRepository(repos);
}
public async getWidgets(): Promise<ICustomWidget[]> {
return this._combinedWidgets.getWidgets();
}
}
/**
* Version of WidgetRepository that caches successful result for 2 minutes.
*/
@@ -79,6 +200,7 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
const list = await super.getWidgets();
// Cache only if there are some widgets.
if (list.length) { this._cache.set(1, list); }
console.log("CACHABLE RESULT", {list});
return list;
}
@@ -91,6 +213,66 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
/**
* Returns widget repository implementation.
*/
export function buildWidgetRepository() {
return new CachedWidgetRepository();
export function buildWidgetRepository(gristServer: GristServer,
options?: {
localOnly: boolean
}) {
if (options?.localOnly) {
return new WidgetRepositoryImpl({
gristServer,
staticUrl: ''
});
}
return new CachedWidgetRepository({
gristServer,
});
}
function fixUrls(widgets: ICustomWidget[], baseUrl: string) {
// If URLs are relative, make them absolute, interpreting them
// relative to the manifest file.
for (const widget of widgets) {
if (!(url.parse(widget.url).protocol)) {
widget.url = new URL(widget.url, baseUrl).href;
}
}
}
export interface CustomWidgetPlace {
urlBase: string,
fileBase: string,
fileDir: string,
name: string,
pluginId: string,
}
export function getWidgetPlaces(gristServer: GristServer,
pluginUrl?: string) {
const places: CustomWidgetPlace[] = [];
const plugins = gristServer.getPlugins();
console.log("PLUGINS", plugins);
pluginUrl = pluginUrl || gristServer.getPluginUrl();
if (!pluginUrl) { return []; }
for (const plugin of plugins) {
console.log("PLUGIN", plugin);
const components = plugin.manifest.components;
if (!components.widgets) { continue; }
console.log("GOT SOMETHING", {
name: plugin.id,
path: plugin.path,
widgets: components.widgets
});
const urlBase =
removeTrailingSlash(pluginUrl) + '/v/' +
gristServer.getTag() + '/widgets/' + plugin.id + '/';
places.push({
urlBase,
fileBase: path.join(plugin.path, components.widgets),
fileDir: plugin.path,
name: plugin.id,
pluginId: plugin.id,
});
}
console.log("PLACES", places);
return places;
}

View File

@@ -6,6 +6,7 @@ import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
import { getOriginUrl } from 'app/server/lib/requestUtils';
import { NextFunction, Request, RequestHandler, Response } from 'express';
import { IncomingMessage } from 'http';
import { GristServer } from './GristServer';
// How long we cache information about the relationship between
// orgs and custom hosts. The higher this is, the fewer requests
@@ -41,7 +42,7 @@ export class Hosts {
// baseDomain should start with ".". It may be undefined for localhost or single-org mode.
constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager,
private _pluginUrl: string|undefined) {
private _gristServer: GristServer|undefined) {
}
/**
@@ -165,6 +166,6 @@ export class Hosts {
}
private _getHostType(host: string) {
return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._pluginUrl});
return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._gristServer?.getPluginUrl()});
}
}

View File

@@ -33,7 +33,7 @@ export interface ISendAppPageOptions {
googleTagManager?: true | false | 'anon';
}
export interface MakeGristConfigOptons {
export interface MakeGristConfigOptions {
homeUrl: string|null;
extra: Partial<GristLoadConfig>;
baseDomain?: string;
@@ -41,7 +41,7 @@ export interface MakeGristConfigOptons {
server?: GristServer|null;
}
export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfig {
const {homeUrl, extra, baseDomain, req, server} = options;
// .invalid is a TLD the IETF promises will never exist.
const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
@@ -78,7 +78,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
featureComments: isAffirmative(process.env.COMMENTS),
featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT),
assistantService: process.env.OPENAI_API_KEY ? 'OpenAI' : undefined,
permittedCustomWidgets: getPermittedCustomWidgets(),
permittedCustomWidgets: getPermittedCustomWidgets(server),
supportEmail: SUPPORT_EMAIL,
userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
telemetry: server?.getTelemetry().getTelemetryConfig(),
@@ -115,6 +115,7 @@ export function makeSendAppPage(opts: {
}) {
const {server, staticDir, tag, testLogin} = opts;
return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
console.log("HERE WE GO");
const config = makeGristConfig({
homeUrl: !isSingleUserMode() ? server.getHomeUrl(req) : null,
extra: options.config,
@@ -170,9 +171,30 @@ function getFeatures(): IFeature[] {
return Features.checkAll(difference(enabledFeatures, disabledFeatures));
}
function getPermittedCustomWidgets(): IAttachedCustomWidget[] {
function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCustomWidget[] {
if (!process.env.PERMITTED_CUSTOM_WIDGETS && gristServer) {
console.log("*****");
console.log("*****");
console.log("*****");
const widgets = gristServer.getBundledWidgets();
const names = new Set(AttachedCustomWidgets.values as string[]);
console.log({widgets, names});
const namesFound: IAttachedCustomWidget[] = [];
for (const widget of widgets) {
// For some reason in different parts of the code attached custom
// widgets are identified by a lot of variants of their name or id
// e.g. "Calendar", "calendar", "custom.calendar", "@gristlabs/grist-calendar'...
const name = widget.widgetId.replace('@gristlabs/widget-', 'custom.');
console.log("CHECK", {name});
if (names.has(name)) {
console.log("CHECK FOUND", {name});
namesFound.push(name as IAttachedCustomWidget);
}
}
return AttachedCustomWidgets.checkAll(namesFound);
}
const widgetsList = process.env.PERMITTED_CUSTOM_WIDGETS?.split(',').map(widgetName=>`custom.${widgetName}`) ?? [];
return AttachedCustomWidgets.checkAll(widgetsList);
return AttachedCustomWidgets.checkAll(widgetsList);
}
function configuredPageTitleSuffix() {