Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) support for bundling custom widgets with the Grist app

Browse Source 
Summary:
This adds support for bundling custom widgets with the Grist app, as follows:

 * Adds a new `widgets` component to plugins mechanism.
 * When a set of widgets is provided in a plugin, the html/js/css assets for those widgets are served on the existing untrusted user content port.
 * Any bundled `grist-plugin-api.js` will be served with the Grist app's own version of that file. It is important that bundled widgets not refer to https://docs.getgrist.com for the plugin js, since they must be capable of working offline.
 * The logic for configuring that port is updated a bit.
 * I removed the CustomAttachedView class in favor of applying settings of bundled custom widgets more directly, without modification on view.

Any Grist installation via docker will need an extra step now, since there is an extra port that needs exposing for full functionality. I did add a `GRIST_TRUST_PLUGINS` option for anyone who really doesn't want to do this, and would prefer to trust the plugins and have them served on the same port.

Actually making use of bundling will be another step. It'll be important to mesh it with our SaaS's use of APP_STATIC_URL for serving most static assets.

Design sketch: https://grist.quip.com/bJlWACWzr2R9/Bundled-custom-widgets

Test Plan: added a test

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4069
This commit is contained in:
Paul Fitzpatrick
2023-10-27 15:34:42 -04:00
parent cb0ce9b20f
commit cc9a9ae8c5
26 changed files with 961 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
app/server/devServerMain.ts
View File

@@ -34,22 +34,6 @@ function getPort(envVarName: string, fallbackPort: number): number {
return val ? parseInt(val, 10) : fallbackPort;
}
// Checks whether to serve user content on same domain but on different port
function checkUserContentPort(): number | null {
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
export async function main() {
log.info("==========================================================================");
log.info("== devServer");
@@ -114,14 +98,6 @@ export async function main() {
}
const server = await mergedServerMain(port, ["home", "docs", "static"]);
await server.addTestingHooks();
// If plugin content is served from same host but on different port,
// run webserver on that port
const userPort = checkUserContentPort();
if (userPort !== null) {
log.info("==========================================================================");
log.info("== userContent");
await server.startCopy('pluginServer', userPort);
}
return;
}
@@ -155,15 +131,6 @@ export async function main() {
await home.startCopy('webServer', webServerPort);
}
// If plugin content is served from same host but on different port,
// run webserver on that port
const userPort = checkUserContentPort();
if (userPort !== null) {
log.info("==========================================================================");
log.info("== userContent");
await home.startCopy('pluginServer', userPort);
}
// Bring up the docWorker(s)
log.info("==========================================================================");
log.info("== docWorker");

173
app/server/lib/FlexServer.ts
View File

@@ -1,4 +1,5 @@
import {ApiError} from 'app/common/ApiError';
import {ICustomWidget} from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
@@ -65,7 +66,7 @@ import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {addUploadRoute} from 'app/server/lib/uploads';
import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {setupLocale} from 'app/server/localization';
import axios from 'axios';
import * as cookie from 'cookie';
@@ -127,6 +128,9 @@ export class FlexServer implements GristServer {
private _dbManager: HomeDBManager;
private _defaultBaseDomain: string|undefined;
private _pluginUrl: string|undefined;
private _pluginUrlReady: boolean = false;
private _servesPlugins?: boolean;
private _bundledWidgets?: ICustomWidget[];
private _billing: IBilling;
private _instanceRoot: string;
private _docManager: DocManager;
@@ -169,11 +173,30 @@ export class FlexServer implements GristServer {
private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
private _getLoginSystem?: () => Promise<GristLoginSystem>;
// Called by ready() to allow requests to be served.
private _ready: () => void;
// Set once ready() is called
private _isReady: boolean = false;
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
this.app = express();
this.app.set('port', port);
// Before doing anything, we pause any request handling to wait
// for the server being entirely ready. The specific reason to do
// so is because, if we are serving plugins, and using an
// OS-assigned port to do so, we won't know the URL to use for
// plugins until quite late. But it seems a nice thing to
// guarantee in general.
const readyPromise = new Promise(resolve => {
this._ready = () => resolve(undefined);
});
this.app.use(async (_req, _res, next) => {
await readyPromise;
next();
});
this.appRoot = getAppRoot();
this.host = process.env.GRIST_HOST || "localhost";
log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`);
@@ -219,7 +242,6 @@ export class FlexServer implements GristServer {
}
this.info.push(['defaultBaseDomain', this._defaultBaseDomain]);
this._pluginUrl = options.pluginUrl || process.env.APP_UNTRUSTED_URL;
this.info.push(['pluginUrl', this._pluginUrl]);
// The electron build is not supported at this time, but this stub
// implementation of electronServerMethods is present to allow kicking
@@ -540,12 +562,6 @@ export class FlexServer implements GristServer {
public addStaticAndBowerDirectories() {
if (this._check('static_and_bower', 'dir')) { return; }
this.addTagChecker();
// Allow static files to be requested from any origin.
const options: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};
// Grist has static help files, which may be useful for standalone app,
// but for hosted grist the latest help is at support.getgrist.com. Redirect
// to this page for the benefit of crawlers which currently rank the static help
@@ -558,11 +574,11 @@ export class FlexServer implements GristServer {
// as an Electron app.
const staticExtDir = getAppPathTo(this.appRoot, 'static') + '_ext';
const staticExtApp = fse.existsSync(staticExtDir) ?
express.static(staticExtDir, options) : null;
const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options);
const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options);
express.static(staticExtDir, serveAnyOrigin) : null;
const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), serveAnyOrigin);
const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), serveAnyOrigin);
if (process.env.GRIST_LOCALES_DIR) {
const locales = express.static(process.env.GRIST_LOCALES_DIR, options);
const locales = express.static(process.env.GRIST_LOCALES_DIR, serveAnyOrigin);
this.app.use("/locales", this.tagChecker.withTag(locales));
}
if (staticExtApp) { this.app.use(this.tagChecker.withTag(staticExtApp)); }
@@ -586,19 +602,38 @@ export class FlexServer implements GristServer {
this.app.use(/^\/(grist-plugin-api.js)$/, expressWrap(async (req, res) =>
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
// Plugins get access to static resources without a tag
this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'static'))));
this.app.use(limitToPlugins(express.static(getAppPathTo(this.appRoot, 'bower_components'))));
this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'static'))));
this.app.use(limitToPlugins(this, express.static(getAppPathTo(this.appRoot, 'bower_components'))));
// Serve custom-widget.html message for anyone.
this.app.use(/^\/(custom-widget.html)$/, expressWrap(async (req, res) =>
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
this.addOrg();
addPluginEndpoints(this, await this._addPluginManager());
// Serve bundled custom widgets on the plugin endpoint.
const places = getWidgetsInPlugins(this, '');
if (places.length > 0) {
// For all widgets served in place, replace any copies of
// grist-plugin-api.js with this app's version of it.
// This is perhaps a bit rude, but beats the alternative
// of either using inconsistent bundled versions, or
// requiring network access.
this.app.use(/^\/widgets\/.*\/(grist-plugin-api.js)$/, expressWrap(async (req, res) =>
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
}
for (const place of places) {
this.app.use(
'/widgets/' + place.pluginId, this.tagChecker.withTag(
limitToPlugins(this, express.static(place.dir, serveAnyOrigin))
)
);
}
}
// Prepare cache for managing org-to-host relationship.
public addHosts() {
if (this._check('hosts', 'homedb')) { return; }
this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this._pluginUrl);
this._hosts = new Hosts(this._defaultBaseDomain, this._dbManager, this);
}
public async initHomeDBManager() {
@@ -706,7 +741,7 @@ export class FlexServer implements GristServer {
// ApiServer's constructor adds endpoints to the app.
// tslint:disable-next-line:no-unused-expression
new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository());
new ApiServer(this, this.app, this._dbManager, this._widgetRepository = buildWidgetRepository(this));
}
public addBillingApi() {
@@ -1420,7 +1455,7 @@ export class FlexServer implements GristServer {
}), jsonErrorHandler); // Add a final error handler that reports errors as JSON.
}
public finalize() {
public finalizeEndpoints() {
this.addApiErrorHandlers();
// add a final non-found handler for other content.
@@ -1452,6 +1487,72 @@ export class FlexServer implements GristServer {
});
}
/**
* Check whether there's a local plugin port.
*/
public servesPlugins() {
if (this._servesPlugins === undefined) {
throw new Error('do not know if server will serve plugins');
}
return this._servesPlugins;
}
/**
* Declare that there will be a local plugin port.
*/
public setServesPlugins(flag: boolean) {
this._servesPlugins = flag;
}
/**
* Get the base URL for plugins. Throws an error if the URL is not
* yet available.
*/
public getPluginUrl() {
if (!this._pluginUrlReady) {
throw new Error('looked at plugin url too early');
}
return this._pluginUrl;
}
public getPlugins() {
if (!this._pluginManager) {
throw new Error('plugin manager not available');
}
return this._pluginManager.getPlugins();
}
public async finalizePlugins(userPort: number|null) {
if (isAffirmative(process.env.GRIST_TRUST_PLUGINS)) {
this._pluginUrl = this.getDefaultHomeUrl();
} else if (userPort !== null) {
// If plugin content is served from same host but on different port,
// run webserver on that port
const ports = await this.startCopy('pluginServer', userPort);
// If Grist is running on a desktop, directly on the host, it
// can be convenient to leave the user port free for the OS to
// allocate by using GRIST_UNTRUSTED_PORT=0. But we do need to
// remember how to contact it.
if (process.env.APP_UNTRUSTED_URL === undefined) {
const url = new URL(this.getOwnUrl());
url.port = String(userPort || ports.serverPort);
this._pluginUrl = url.href;
}
}
this.info.push(['pluginUrl', this._pluginUrl]);
this.info.push(['willServePlugins', this._servesPlugins]);
this._pluginUrlReady = true;
const repo = buildWidgetRepository(this, { localOnly: true });
this._bundledWidgets = await repo.getWidgets();
}
public getBundledWidgets(): ICustomWidget[] {
if (!this._bundledWidgets) {
throw new Error('bundled widgets accessed too early');
}
return this._bundledWidgets;
}
public summary() {
for (const [label, value] of this.info) {
log.info("== %s: %s", label, value);
@@ -1466,6 +1567,12 @@ export class FlexServer implements GristServer {
}
}
public ready() {
if (this._isReady) { return; }
this._isReady = true;
this._ready();
}
public checkOptionCombinations() {
// Check for some bad combinations we should warn about.
const allowedWebhookDomains = appSettings.section('integrations').flag('allowedWebhookDomains').readString({
@@ -1565,9 +1672,12 @@ export class FlexServer implements GristServer {
await this.housekeeper.start();
}
public async startCopy(name2: string, port2: number) {
public async startCopy(name2: string, port2: number): Promise<{
serverPort: number,
httpsServerPort?: number,
}>{
const servers = this._createServers();
await this._startServers(servers.server, servers.httpsServer, name2, port2, true);
return this._startServers(servers.server, servers.httpsServer, name2, port2, true);
}
/**
@@ -1633,6 +1743,9 @@ export class FlexServer implements GristServer {
}
public getTag(): string {
if (!this.tag) {
throw new Error('getTag called too early');
}
return this.tag;
}
@@ -1934,12 +2047,19 @@ export class FlexServer implements GristServer {
private async _startServers(server: http.Server, httpsServer: https.Server|undefined,
name: string, port: number, verbose: boolean) {
await listenPromise(server.listen(port, this.host));
if (verbose) { log.info(`${name} available at ${this.host}:${port}`); }
const serverPort = (server.address() as AddressInfo).port;
if (verbose) { log.info(`${name} available at ${this.host}:${serverPort}`); }
let httpsServerPort: number|undefined;
if (TEST_HTTPS_OFFSET && httpsServer) {
const httpsPort = port + TEST_HTTPS_OFFSET;
await listenPromise(httpsServer.listen(httpsPort, this.host));
if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); }
if (port === 0) { throw new Error('cannot use https with OS-assigned port'); }
httpsServerPort = port + TEST_HTTPS_OFFSET;
await listenPromise(httpsServer.listen(httpsServerPort, this.host));
if (verbose) { log.info(`${name} available at https://${this.host}:${httpsServerPort}`); }
}
return {
serverPort,
httpsServerPort,
};
}
private async _recordNewUserInfo(row: object) {
@@ -2194,3 +2314,10 @@ export interface ElectronServerMethods {
updateUserConfig(obj: any): Promise<void>;
onBackupMade(cb: () => void): void;
}
// Allow static files to be requested from any origin.
const serveAnyOrigin: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};

10
app/server/lib/GristServer.ts
View File

@@ -1,4 +1,6 @@
import { ICustomWidget } from 'app/common/CustomWidget';
import { GristDeploymentType, GristLoadConfig } from 'app/common/gristUrls';
import { LocalPlugin } from 'app/common/plugin';
import { UserProfile } from 'app/common/UserAPI';
import { Document } from 'app/gen-server/entity/Document';
import { Organization } from 'app/gen-server/entity/Organization';
@@ -54,6 +56,10 @@ export interface GristServer {
sendAppPage(req: express.Request, resp: express.Response, options: ISendAppPageOptions): Promise<void>;
getAccessTokens(): IAccessTokens;
resolveLoginSystem(): Promise<GristLoginSystem>;
getPluginUrl(): string|undefined;
getPlugins(): LocalPlugin[];
servesPlugins(): boolean;
getBundledWidgets(): ICustomWidget[];
}
export interface GristLoginSystem {
@@ -136,6 +142,10 @@ export function createDummyGristServer(): GristServer {
sendAppPage() { return Promise.resolve(); },
getAccessTokens() { throw new Error('no access tokens'); },
resolveLoginSystem() { throw new Error('no login system'); },
getPluginUrl() { return undefined; },
servesPlugins() { return false; },
getPlugins() { return []; },
getBundledWidgets() { return []; },
};
}

30
app/server/lib/PluginEndpoint.ts
View File

@@ -1,34 +1,36 @@
import {FlexServer} from 'app/server/lib/FlexServer';
import {GristServer} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {PluginManager} from 'app/server/lib/PluginManager';
import * as express from 'express';
import * as mimeTypes from 'mime-types';
import * as path from 'path';
// Get the url where plugin material should be served from.
export function getUntrustedContentOrigin(): string|undefined {
return process.env.APP_UNTRUSTED_URL;
}
// Get the host serving plugin material
export function getUntrustedContentHost(): string|undefined {
const origin = getUntrustedContentOrigin();
export function getUntrustedContentHost(origin: string|undefined): string|undefined {
if (!origin) { return; }
return new URL(origin).host;
}
// Add plugin endpoints to be served on untrusted host
export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) {
const host = getUntrustedContentHost();
if (host) {
if (server.servesPlugins()) {
server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) =>
servePluginContent(req, res, pluginManager, host));
servePluginContent(req, res, pluginManager, server));
}
}
// Serve content for plugins with various checks that it is being accessed as we expect.
function servePluginContent(req: express.Request, res: express.Response,
pluginManager: PluginManager, untrustedContentHost: string) {
pluginManager: PluginManager,
gristServer: GristServer) {
const pluginUrl = gristServer.getPluginUrl();
const untrustedContentHost = getUntrustedContentHost(pluginUrl);
if (!untrustedContentHost) {
// not expected
throw new Error('plugin host unexpectedly not set');
}
const pluginKind = req.params[0];
const pluginId = req.params[1];
const pluginPath = req.params[2];
@@ -56,9 +58,11 @@ function servePluginContent(req: express.Request, res: express.Response,
}
// Middleware to restrict some assets to untrusted host.
export function limitToPlugins(handler: express.RequestHandler) {
const host = getUntrustedContentHost();
export function limitToPlugins(gristServer: GristServer,
handler: express.RequestHandler) {
return function(req: express.Request, resp: express.Response, next: express.NextFunction) {
const pluginUrl = gristServer.getPluginUrl();
const host = getUntrustedContentHost(pluginUrl);
if (!host) { return next(); }
if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") {
return handler(req, resp, next);

6
app/server/lib/PluginManager.ts
View File

@@ -137,8 +137,10 @@ async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise<
try {
listDir = await fse.readdir(dir);
} catch (e) {
// non existing dir is treated as an empty dir
log.info(`No plugins directory: ${e.message}`);
// Non existing dir is treated as an empty dir.
// It is hard for user to avoid Grist checking a dir,
// so phrase the message as information rather than error.
log.info(`No plugins found in directory: ${dir}`);
return [];
}

209
app/server/lib/WidgetRepository.ts
View File

@@ -1,8 +1,17 @@
import {ICustomWidget} from 'app/common/CustomWidget';
import log from 'app/server/lib/log';
import * as fse from 'fs-extra';
import fetch from 'node-fetch';
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import {removeTrailingSlash} from 'app/common/gutil';
import {GristServer} from 'app/server/lib/GristServer';
import LRUCache from 'lru-cache';
import * as url from 'url';
import { AsyncCreate } from 'app/common/AsyncCreate';
// Static url for UrlWidgetRepository
const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
/**
* Widget Repository returns list of available Custom Widgets.
@@ -11,28 +20,90 @@ export interface IWidgetRepository {
getWidgets(): Promise<ICustomWidget[]>;
}
// Static url for StaticWidgetRepository
const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
/**
*
* A widget repository that lives on disk.
*
* The _widgetFile should point to a json file containing a
* list of custom widgets, in the format used by the grist-widget
* repo:
* https://github.com/gristlabs/grist-widget
*
* The file can use relative URLs. The URLs will be interpreted
* as relative to the _widgetBaseUrl.
*
* If a _source is provided, it will be passed along in the
* widget listings.
*
*/
export class DiskWidgetRepository implements IWidgetRepository {
constructor(private _widgetFile: string,
private _widgetBaseUrl: string,
private _source?: any) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const txt = await fse.readFile(this._widgetFile, { encoding: 'utf8' });
const widgets: ICustomWidget[] = JSON.parse(txt);
fixUrls(widgets, this._widgetBaseUrl);
if (this._source) {
for (const widget of widgets) {
widget.source = this._source;
}
}
return widgets;
}
}
/**
* Default repository that gets list of available widgets from a static URL.
*
* A wrapper around a widget repository that delays creating it
* until the first call to getWidgets().
*
*/
export class WidgetRepositoryImpl implements IWidgetRepository {
constructor(protected _staticUrl = STATIC_URL) {}
export class DelayedWidgetRepository implements IWidgetRepository {
private _repo: AsyncCreate<IWidgetRepository|undefined>;
/**
* Method exposed for testing, overrides widget url.
*/
public testOverrideUrl(url: string) {
this._staticUrl = url;
constructor(_makeRepo: () => Promise<IWidgetRepository|undefined>) {
this._repo = new AsyncCreate(_makeRepo);
}
public async getWidgets(): Promise<ICustomWidget[]> {
const repo = await this._repo.get();
if (!repo) { return []; }
return repo.getWidgets();
}
}
/**
*
* A wrapper around a list of widget repositories that concatenates
* their results.
*
*/
export class CombinedWidgetRepository implements IWidgetRepository {
constructor(private _repos: IWidgetRepository[]) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const allWidgets: ICustomWidget[] = [];
for (const repo of this._repos) {
allWidgets.push(...await repo.getWidgets());
}
return allWidgets;
}
}
/**
* Repository that gets a list of widgets from a URL.
*/
export class UrlWidgetRepository implements IWidgetRepository {
constructor(private _staticUrl = STATIC_URL) {}
public async getWidgets(): Promise<ICustomWidget[]> {
if (!this._staticUrl) {
log.warn(
'WidgetRepository: Widget repository is not configured.' + !STATIC_URL
'WidgetRepository: Widget repository is not configured.' + (!STATIC_URL
? ' Missing GRIST_WIDGET_LIST_URL environmental variable.'
: ''
: '')
);
return [];
}
@@ -52,6 +123,7 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
if (!widgets || !Array.isArray(widgets)) {
throw new ApiError('WidgetRepository: Error reading widget list', 500);
}
fixUrls(widgets, this._staticUrl);
return widgets;
} catch (err) {
if (!(err instanceof ApiError)) {
@@ -62,6 +134,61 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
}
}
/**
* Default repository that gets list of available widgets from multiple
* sources.
*/
export class WidgetRepositoryImpl implements IWidgetRepository {
protected _staticUrl: string|undefined;
private _diskWidgets?: IWidgetRepository;
private _urlWidgets: UrlWidgetRepository;
private _combinedWidgets: CombinedWidgetRepository;
constructor(_options: {
staticUrl?: string,
gristServer?: GristServer,
}) {
const {staticUrl, gristServer} = _options;
if (gristServer) {
this._diskWidgets = new DelayedWidgetRepository(async () => {
const places = getWidgetsInPlugins(gristServer);
const files = places.map(
place => new DiskWidgetRepository(
place.file,
place.urlBase,
{
pluginId: place.pluginId,
name: place.name
}));
return new CombinedWidgetRepository(files);
});
}
this.testSetUrl(staticUrl);
}
/**
* Method exposed for testing, overrides widget url.
*/
public testOverrideUrl(overrideUrl: string|undefined) {
this.testSetUrl(overrideUrl);
}
public testSetUrl(overrideUrl: string|undefined) {
const repos: IWidgetRepository[] = [];
this._staticUrl = overrideUrl ?? STATIC_URL;
if (this._staticUrl) {
this._urlWidgets = new UrlWidgetRepository(this._staticUrl);
repos.push(this._urlWidgets);
}
if (this._diskWidgets) { repos.push(this._diskWidgets); }
this._combinedWidgets = new CombinedWidgetRepository(repos);
}
public async getWidgets(): Promise<ICustomWidget[]> {
return this._combinedWidgets.getWidgets();
}
}
/**
* Version of WidgetRepository that caches successful result for 2 minutes.
*/
@@ -91,6 +218,60 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
/**
* Returns widget repository implementation.
*/
export function buildWidgetRepository() {
return new CachedWidgetRepository();
export function buildWidgetRepository(gristServer: GristServer,
options?: {
localOnly: boolean
}) {
return new CachedWidgetRepository({
gristServer,
...(options?.localOnly ? { staticUrl: '' } : undefined)
});
}
function fixUrls(widgets: ICustomWidget[], baseUrl: string) {
// If URLs are relative, make them absolute, interpreting them
// relative to the supplied base.
for (const widget of widgets) {
if (!(url.parse(widget.url).protocol)) {
widget.url = new URL(widget.url, baseUrl).href;
}
}
}
/**
* Information about widgets in a plugin. We need to coordinate
* URLs with location on disk.
*/
export interface CustomWidgetsInPlugin {
pluginId: string,
urlBase: string,
dir: string,
file: string,
name: string,
}
/**
* Get a list of widgets available locally via plugins.
*/
export function getWidgetsInPlugins(gristServer: GristServer,
pluginUrl?: string) {
const places: CustomWidgetsInPlugin[] = [];
const plugins = gristServer.getPlugins();
pluginUrl = pluginUrl ?? gristServer.getPluginUrl();
if (pluginUrl === undefined) { return []; }
for (const plugin of plugins) {
const components = plugin.manifest.components;
if (!components.widgets) { continue; }
const urlBase =
removeTrailingSlash(pluginUrl) + '/v/' +
gristServer.getTag() + '/widgets/' + plugin.id + '/';
places.push({
urlBase,
dir: plugin.path,
file: path.join(plugin.path, components.widgets),
name: plugin.manifest.name || plugin.id,
pluginId: plugin.id,
});
}
return places;
}

8
app/server/lib/extractOrg.ts
View File

@@ -1,8 +1,10 @@
import { ApiError } from 'app/common/ApiError';
import { mapGetOrSet, MapWithTTL } from 'app/common/AsyncCreate';
import { extractOrgParts, getHostType, getKnownOrg } from 'app/common/gristUrls';
import { isAffirmative } from 'app/common/gutil';
import { Organization } from 'app/gen-server/entity/Organization';
import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
import { GristServer } from 'app/server/lib/GristServer';
import { getOriginUrl } from 'app/server/lib/requestUtils';
import { NextFunction, Request, RequestHandler, Response } from 'express';
import { IncomingMessage } from 'http';
@@ -41,7 +43,7 @@ export class Hosts {
// baseDomain should start with ".". It may be undefined for localhost or single-org mode.
constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager,
private _pluginUrl: string|undefined) {
private _gristServer: GristServer|undefined) {
}
/**
@@ -165,6 +167,8 @@ export class Hosts {
}
private _getHostType(host: string) {
return getHostType(host, {baseDomain: this._baseDomain, pluginUrl: this._pluginUrl});
const pluginUrl = isAffirmative(process.env.GRIST_TRUST_PLUGINS) ?
undefined : this._gristServer?.getPluginUrl();
return getHostType(host, {baseDomain: this._baseDomain, pluginUrl});
}
}

12
app/server/lib/manifest.ts
View File

@@ -50,7 +50,11 @@ export class ManifestError extends Error {
*/
export async function readManifest(pluginPath: string): Promise<BarePlugin> {
const notices: string[] = [];
const manifest = await _readManifest(pluginPath);
const manifest: any = await _readManifest(pluginPath);
// We allow contributions and components to be omitted as shorthand
// for being the empty object.
if (!manifest.contributions) { manifest.contributions = {}; }
if (!manifest.components) { manifest.components = {}; }
if (isValidManifest(manifest, notices)) {
return manifest as BarePlugin;
}
@@ -58,6 +62,9 @@ export async function readManifest(pluginPath: string): Promise<BarePlugin> {
}
async function _readManifest(pluginPath: string): Promise<object> {
async function readManifestFile(fileExtension: string): Promise<string> {
return await fse.readFile(path.join(pluginPath, "manifest." + fileExtension), "utf8");
}
try {
return yaml.safeLoad(await readManifestFile("yml"));
} catch (e) {
@@ -73,7 +80,4 @@ async function _readManifest(pluginPath: string): Promise<object> {
}
throw new Error('cannot read manifest file: ' + e.message);
}
async function readManifestFile(fileExtension: string): Promise<string> {
return await fse.readFile(path.join(pluginPath, "manifest." + fileExtension), "utf8");
}
}

30
app/server/lib/sendAppPage.ts
View File

@@ -33,7 +33,7 @@ export interface ISendAppPageOptions {
googleTagManager?: true | false | 'anon';
}
export interface MakeGristConfigOptons {
export interface MakeGristConfigOptions {
homeUrl: string|null;
extra: Partial<GristLoadConfig>;
baseDomain?: string;
@@ -41,7 +41,7 @@ export interface MakeGristConfigOptons {
server?: GristServer|null;
}
export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig {
export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfig {
const {homeUrl, extra, baseDomain, req, server} = options;
// .invalid is a TLD the IETF promises will never exist.
const pluginUrl = process.env.APP_UNTRUSTED_URL || 'http://plugins.invalid';
@@ -68,7 +68,8 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,
timestampMs: Date.now(),
enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL),
enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL) ||
((server?.getBundledWidgets().length || 0) > 0),
survey: Boolean(process.env.DOC_ID_NEW_USER_INFO),
tagManagerId: process.env.GOOGLE_TAG_MANAGER_ID,
activation: getActivation(req as RequestWithLogin | undefined),
@@ -78,7 +79,7 @@ export function makeGristConfig(options: MakeGristConfigOptons): GristLoadConfig
featureComments: isAffirmative(process.env.COMMENTS),
featureFormulaAssistant: Boolean(process.env.OPENAI_API_KEY || process.env.ASSISTANT_CHAT_COMPLETION_ENDPOINT),
assistantService: process.env.OPENAI_API_KEY ? 'OpenAI' : undefined,
permittedCustomWidgets: getPermittedCustomWidgets(),
permittedCustomWidgets: getPermittedCustomWidgets(server),
gristNewColumnMenu: isAffirmative(process.env.GRIST_NEW_COLUMN_MENU),
supportEmail: SUPPORT_EMAIL,
userLocale: (req as RequestWithLogin | undefined)?.user?.options?.locale,
@@ -171,9 +172,26 @@ function getFeatures(): IFeature[] {
return Features.checkAll(difference(enabledFeatures, disabledFeatures));
}
function getPermittedCustomWidgets(): IAttachedCustomWidget[] {
function getPermittedCustomWidgets(gristServer?: GristServer|null): IAttachedCustomWidget[] {
if (!process.env.PERMITTED_CUSTOM_WIDGETS && gristServer) {
// The PERMITTED_CUSTOM_WIDGETS environment variable is a bit of
// a drag. If there are bundled widgets that overlap with widgets
// described in the codebase, let's just assume they are permitted.
const widgets = gristServer.getBundledWidgets();
const names = new Set(AttachedCustomWidgets.values as string[]);
const namesFound: IAttachedCustomWidget[] = [];
for (const widget of widgets) {
// Permitted custom widgets are identified so many ways across the
// code! Why? TODO: cut down on identifiers.
const name = widget.widgetId.replace('@gristlabs/widget-', 'custom.');
if (names.has(name)) {
namesFound.push(name as IAttachedCustomWidget);
}
}
return AttachedCustomWidgets.checkAll(namesFound);
}
const widgetsList = process.env.PERMITTED_CUSTOM_WIDGETS?.split(',').map(widgetName=>`custom.${widgetName}`) ?? [];
return AttachedCustomWidgets.checkAll(widgetsList);
return AttachedCustomWidgets.checkAll(widgetsList);
}
function configuredPageTitleSuffix() {

33
app/server/mergedServerMain.ts
View File

@@ -31,6 +31,26 @@ export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
return types as ServerType[];
}
function checkUserContentPort(): number | null {
// Check whether a port is explicitly set for user content.
if (process.env.GRIST_UNTRUSTED_PORT) {
return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
}
// Checks whether to serve user content on same domain but on different port
if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
const homeUrl = new URL(process.env.APP_HOME_URL);
const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
// If the hostname of both home and plugin url are the same,
// but the ports are different
if (homeUrl.hostname === pluginUrl.hostname &&
homeUrl.port !== pluginUrl.port) {
const port = parseInt(pluginUrl.port || '80', 10);
return port;
}
}
return null;
}
interface ServerOptions extends FlexServerOptions {
logToConsole?: boolean; // If set, messages logged to console (default: false)
// (but if options are not given at all in call to main,
@@ -52,6 +72,14 @@ export async function main(port: number, serverTypes: ServerType[],
const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
// We need to know early on whether we will be serving plugins or not.
if (includeHome) {
const userPort = checkUserContentPort();
server.setServesPlugins(userPort !== undefined);
} else {
server.setServesPlugins(false);
}
if (options.loginSystem) {
server.setLoginSystem(options.loginSystem);
}
@@ -141,10 +169,11 @@ export async function main(port: number, serverTypes: ServerType[],
server.addClientSecrets();
}
server.finalize();
server.finalizeEndpoints();
await server.finalizePlugins(includeHome ? checkUserContentPort() : null);
server.checkOptionCombinations();
server.summary();
server.ready();
return server;
} catch(e) {
await server.close();