Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

some cleanup

Browse Source
This commit is contained in:
Paul Fitzpatrick
2023-10-06 21:38:37 -04:00
parent fd1734de69
commit 4f3d0d41a0
18 changed files with 305 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files

148
app/server/lib/FlexServer.ts
View File

@@ -1,5 +1,5 @@
import {ApiError} from 'app/common/ApiError';
import { ICustomWidget } from 'app/common/CustomWidget';
import {ICustomWidget} from 'app/common/CustomWidget';
import {delay} from 'app/common/delay';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
@@ -66,7 +66,7 @@ import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
import {startTestingHooks} from 'app/server/lib/TestingHooks';
import {getTestLoginSystem} from 'app/server/lib/TestLogin';
import {addUploadRoute} from 'app/server/lib/uploads';
import { buildWidgetRepository, getWidgetPlaces, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {buildWidgetRepository, getWidgetsInPlugins, IWidgetRepository} from 'app/server/lib/WidgetRepository';
import {setupLocale} from 'app/server/localization';
import axios from 'axios';
import * as bodyParser from 'body-parser';
@@ -129,8 +129,8 @@ export class FlexServer implements GristServer {
private _dbManager: HomeDBManager;
private _defaultBaseDomain: string|undefined;
private _pluginUrl: string|undefined;
private _pluginUrlSet: boolean = false;
private _willServePlugins?: boolean;
private _pluginUrlReady: boolean = false;
private _servesPlugins?: boolean;
private _bundledWidgets?: ICustomWidget[];
private _billing: IBilling;
private _instanceRoot: string;
@@ -174,11 +174,30 @@ export class FlexServer implements GristServer {
private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
private _getLoginSystem?: () => Promise<GristLoginSystem>;
// Called by ready() to allow requests to be served.
private _ready: () => void;
// Set once ready() is called
private _isReady: boolean = false;
constructor(public port: number, public name: string = 'flexServer',
public readonly options: FlexServerOptions = {}) {
this.app = express();
this.app.set('port', port);
// Before doing anything, we pause any request handling to wait
// for the server being entirely ready. The specific reason to do
// so is because, if we are serving plugins, and using an
// OS-assigned port to do so, we won't know the URL to use for
// plugins until quite late. But it seems a nice thing to
// guarantee in general.
const readyPromise = new Promise(resolve => {
this._ready = () => resolve(undefined);
});
this.app.use(async (_req, _res, next) => {
await readyPromise;
next();
});
this.appRoot = getAppRoot();
this.host = process.env.GRIST_HOST || "localhost";
log.info(`== Grist version is ${version.version} (commit ${version.gitcommit})`);
@@ -508,12 +527,6 @@ export class FlexServer implements GristServer {
public addStaticAndBowerDirectories() {
if (this._check('static_and_bower', 'dir')) { return; }
this.addTagChecker();
// Allow static files to be requested from any origin.
const options: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};
// Grist has static help files, which may be useful for standalone app,
// but for hosted grist the latest help is at support.getgrist.com. Redirect
// to this page for the benefit of crawlers which currently rank the static help
@@ -526,11 +539,11 @@ export class FlexServer implements GristServer {
// as an Electron app.
const staticExtDir = getAppPathTo(this.appRoot, 'static') + '_ext';
const staticExtApp = fse.existsSync(staticExtDir) ?
express.static(staticExtDir, options) : null;
const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), options);
const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), options);
express.static(staticExtDir, serveAnyOrigin) : null;
const staticApp = express.static(getAppPathTo(this.appRoot, 'static'), serveAnyOrigin);
const bowerApp = express.static(getAppPathTo(this.appRoot, 'bower_components'), serveAnyOrigin);
if (process.env.GRIST_LOCALES_DIR) {
const locales = express.static(process.env.GRIST_LOCALES_DIR, options);
const locales = express.static(process.env.GRIST_LOCALES_DIR, serveAnyOrigin);
this.app.use("/locales", this.tagChecker.withTag(locales));
}
if (staticExtApp) { this.app.use(this.tagChecker.withTag(staticExtApp)); }
@@ -561,20 +574,13 @@ export class FlexServer implements GristServer {
res.sendFile(req.params[0], {root: getAppPathTo(this.appRoot, 'static')})));
this.addOrg();
addPluginEndpoints(this, await this._addPluginManager());
const places = getWidgetPlaces(this, 'wotnot');
// Allow static files to be requested from any origin.
const options: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};
for (const place of places) {
this.app.use(
'/widgets/' + place.name,
this.tagChecker.withTag(
limitToPlugins(this,
express.static(place.fileDir, options)
)
// Serve bundled custom widgets on the plugin endpoint.
const places = getWidgetsInPlugins(this, '');
for (const place of places) {
this.app.use(
'/widgets/' + place.pluginId, this.tagChecker.withTag(
limitToPlugins(this, express.static(place.dir, serveAnyOrigin))
)
);
}
@@ -1445,25 +1451,29 @@ export class FlexServer implements GristServer {
});
}
public setPluginPort(port: number) {
const url = new URL(this.getOwnUrl());
url.port = String(port);
this._pluginUrl = url.href;
}
public willServePlugins() {
if (this._willServePlugins === undefined) {
throw new Error('do not know if will serve plugins');
/**
* Check whether there's a local plugin port.
*/
public servesPlugins() {
if (this._servesPlugins === undefined) {
throw new Error('do not know if server will serve plugins');
}
return this._willServePlugins;
}
public setWillServePlugins(flag: boolean) {
this._willServePlugins = flag;
return this._servesPlugins;
}
/**
* Declare that there will be a local plugin port.
*/
public setServesPlugins(flag: boolean) {
this._servesPlugins = flag;
}
/**
* Get the base URL for plugins. Throws an error if the URL is not
* yet available.
*/
public getPluginUrl() {
if (!this._pluginUrlSet) {
if (!this._pluginUrlReady) {
throw new Error('looked at plugin url too early');
}
return this._pluginUrl;
@@ -1476,14 +1486,24 @@ export class FlexServer implements GristServer {
return this._pluginManager.getPlugins();
}
public async prepareSummary() {
// Add some information that isn't guaranteed set until the end.
public async finishPluginSetup(userPort: number|null) {
// If plugin content is served from same host but on different port,
// run webserver on that port
if (userPort !== null) {
const ports = await this.startCopy('pluginServer', userPort);
// If Grist is running on a desktop, directly on the host, it
// can be convenient to leave the user port free for the OS to
// allocate by using GRIST_UNTRUSTED_PORT=0. But we do need to
// remember how to contact it.
if (process.env.APP_UNTRUSTED_URL === undefined) {
const url = new URL(this.getOwnUrl());
url.port = String(userPort || ports.serverPort);
this._pluginUrl = url.href;
}
}
this.info.push(['pluginUrl', this._pluginUrl]);
// plugin url should be finalized by now.
this._pluginUrlSet = true;
this.info.push(['willServePlugins', this._willServePlugins]);
this._pluginUrlReady = true;
this.info.push(['willServePlugins', this._servesPlugins]);
const repo = buildWidgetRepository(this, { localOnly: true });
this._bundledWidgets = await repo.getWidgets();
}
@@ -1509,6 +1529,12 @@ export class FlexServer implements GristServer {
}
}
public ready() {
if (this._isReady) { return; }
this._isReady = true;
this._ready();
}
public checkOptionCombinations() {
// Check for some bad combinations we should warn about.
const allowedWebhookDomains = appSettings.section('integrations').flag('allowedWebhookDomains').readString({
@@ -1983,15 +2009,18 @@ export class FlexServer implements GristServer {
private async _startServers(server: http.Server, httpsServer: https.Server|undefined,
name: string, port: number, verbose: boolean) {
await listenPromise(server.listen(port, this.host));
if (verbose) { log.info(`${name} available at ${this.host}:${port}`); }
const serverPort = (server.address() as AddressInfo).port;
if (verbose) { log.info(`${name} available at ${this.host}:${serverPort}`); }
let httpsServerPort: number|undefined;
if (TEST_HTTPS_OFFSET && httpsServer) {
const httpsPort = port + TEST_HTTPS_OFFSET;
await listenPromise(httpsServer.listen(httpsPort, this.host));
if (verbose) { log.info(`${name} available at https://${this.host}:${httpsPort}`); }
if (port === 0) { throw new Error('cannot use https with OS-assigned port'); }
httpsServerPort = port + TEST_HTTPS_OFFSET;
await listenPromise(httpsServer.listen(httpsServerPort, this.host));
if (verbose) { log.info(`${name} available at https://${this.host}:${httpsServerPort}`); }
}
return {
serverPort: (server.address() as AddressInfo).port,
httpsServerPort: (server.address() as AddressInfo)?.port,
serverPort,
httpsServerPort,
};
}
@@ -2247,3 +2276,10 @@ export interface ElectronServerMethods {
updateUserConfig(obj: any): Promise<void>;
onBackupMade(cb: () => void): void;
}
// Allow static files to be requested from any origin.
const serveAnyOrigin: serveStatic.ServeStaticOptions = {
setHeaders: (res, filepath, stat) => {
res.setHeader("Access-Control-Allow-Origin", "*");
}
};

4
app/server/lib/GristServer.ts
View File

@@ -57,7 +57,7 @@ export interface GristServer {
resolveLoginSystem(): Promise<GristLoginSystem>;
getPluginUrl(): string|undefined;
getPlugins(): LocalPlugin[];
willServePlugins(): boolean;
servesPlugins(): boolean;
getBundledWidgets(): ICustomWidget[];
}
@@ -142,7 +142,7 @@ export function createDummyGristServer(): GristServer {
getAccessTokens() { throw new Error('no access tokens'); },
resolveLoginSystem() { throw new Error('no login system'); },
getPluginUrl() { return undefined; },
willServePlugins() { return false; },
servesPlugins() { return false; },
getPlugins() { return []; },
getBundledWidgets() { return []; },
};

2
app/server/lib/PluginEndpoint.ts
View File

@@ -14,7 +14,7 @@ export function getUntrustedContentHost(origin: string|undefined): string|undefi
// Add plugin endpoints to be served on untrusted host
export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) {
if (server.willServePlugins()) {
if (server.servesPlugins()) {
server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) =>
servePluginContent(req, res, pluginManager, server));
}

1
app/server/lib/PluginManager.ts
View File

@@ -131,7 +131,6 @@ export class PluginManager {
async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise<DirectoryScanEntry[]> {
console.log("SCAN", {dir, kind});
const plugins: DirectoryScanEntry[] = [];
let listDir;

167
app/server/lib/WidgetRepository.ts
View File

@@ -4,11 +4,14 @@ import * as fse from 'fs-extra';
import fetch from 'node-fetch';
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import {removeTrailingSlash} from 'app/common/gutil';
import {GristServer} from 'app/server/lib/GristServer';
import LRUCache from 'lru-cache';
import * as url from 'url';
import { removeTrailingSlash } from 'app/common/gutil';
import { GristServer } from './GristServer';
// import { LocalPlugin } from 'app/common/plugin';
import { AsyncCreate } from 'app/common/AsyncCreate';
// Static url for UrlWidgetRepository
const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
/**
* Widget Repository returns list of available Custom Widgets.
@@ -17,67 +20,66 @@ export interface IWidgetRepository {
getWidgets(): Promise<ICustomWidget[]>;
}
// Static url for StaticWidgetRepository
const STATIC_URL = process.env.GRIST_WIDGET_LIST_URL;
export class FileWidgetRepository implements IWidgetRepository {
constructor(private _widgetFileName: string,
/**
*
* A widget repository that lives on disk.
*
* The _widgetFile should point to a json file containing a
* list of custom widgets, in the format used by the grist-widget
* repo:
* https://github.com/gristlabs/grist-widget
*
* The file can use relative URLs. The URLs will be interpreted
* as relative to the _widgetBaseUrl.
*
* If a _source is provided, it will be passed along in the
* widget listings.
*
*/
export class DiskWidgetRepository implements IWidgetRepository {
constructor(private _widgetFile: string,
private _widgetBaseUrl: string,
private _pluginId?: string) {}
private _source?: any) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const txt = await fse.readFile(this._widgetFileName, {
encoding: 'utf8',
});
const txt = await fse.readFile(this._widgetFile, { encoding: 'utf8' });
const widgets: ICustomWidget[] = JSON.parse(txt);
fixUrls(widgets, this._widgetBaseUrl);
if (this._pluginId) {
if (this._source) {
for (const widget of widgets) {
widget.fromPlugin = this._pluginId;
widget.source = this._source;
}
}
console.log("FileWidget", {widgets});
return widgets;
}
}
/*
export class NestedWidgetRepository implements IWidgetRepository {
constructor(private _widgetDir: string,
private _widgetBaseUrl: string) {}
public async getWidgets(): Promise<ICustomWidget[]> {
const listDir = await fse.readdir(this._widgetDir,
{ withFileTypes: true });
const fileName = 'manifest.json';
const allWidgets: ICustomWidget[] = [];
for (const dir of listDir) {
if (!dir.isDirectory()) { continue; }
const fullPath = path.join(this._widgetDir, dir.name, fileName);
if (!await fse.pathExists(fullPath)) { continue; }
const txt = await fse.readFile(fullPath, 'utf8');
const widgets = JSON.parse(txt);
fixUrls(
widgets,
removeTrailingSlash(this._widgetBaseUrl) + '/' + dir.name + '/'
);
allWidgets.push(...widgets);
}
return allWidgets;
}
}
*/
/**
*
* A wrapper around a widget repository that delays creating it
* until the first call to getWidgets().
*
*/
export class DelayedWidgetRepository implements IWidgetRepository {
constructor(private _makeRepo: () => Promise<IWidgetRepository|undefined>) {}
private _repo: AsyncCreate<IWidgetRepository|undefined>;
constructor(_makeRepo: () => Promise<IWidgetRepository|undefined>) {
this._repo = new AsyncCreate(_makeRepo);
}
public async getWidgets(): Promise<ICustomWidget[]> {
const repo = await this._makeRepo();
const repo = await this._repo.get();
if (!repo) { return []; }
return repo.getWidgets();
}
}
/**
*
* A wrapper around a list of widget repositories that concatenates
* their results.
*
*/
export class CombinedWidgetRepository implements IWidgetRepository {
constructor(private _repos: IWidgetRepository[]) {}
@@ -86,13 +88,12 @@ export class CombinedWidgetRepository implements IWidgetRepository {
for (const repo of this._repos) {
allWidgets.push(...await repo.getWidgets());
}
console.log("COMBINED", {allWidgets});
return allWidgets;
}
}
/**
* Repository that gets list of available widgets from a static URL.
* Repository that gets a list of widgets from a URL.
*/
export class UrlWidgetRepository implements IWidgetRepository {
constructor(private _staticUrl = STATIC_URL) {}
@@ -134,13 +135,14 @@ export class UrlWidgetRepository implements IWidgetRepository {
}
/**
* Default repository that gets list of available widgets from a static URL.
* Default repository that gets list of available widgets from multiple
* sources.
*/
export class WidgetRepositoryImpl implements IWidgetRepository {
protected _staticUrl: string|undefined;
private _diskWidgets?: IWidgetRepository;
private _urlWidgets: UrlWidgetRepository;
private _combinedWidgets: CombinedWidgetRepository;
private _dirWidgets?: IWidgetRepository;
constructor(_options: {
staticUrl?: string,
@@ -148,12 +150,16 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
}) {
const {staticUrl, gristServer} = _options;
if (gristServer) {
this._dirWidgets = new DelayedWidgetRepository(async () => {
const places = getWidgetPlaces(gristServer);
console.log("PLACES!", places);
const files = places.map(place => new FileWidgetRepository(place.fileBase,
place.urlBase,
place.pluginId));
this._diskWidgets = new DelayedWidgetRepository(async () => {
const places = getWidgetsInPlugins(gristServer);
const files = places.map(
place => new DiskWidgetRepository(
place.file,
place.urlBase,
{
pluginId: place.pluginId,
name: place.name
}));
return new CombinedWidgetRepository(files);
});
}
@@ -174,7 +180,7 @@ export class WidgetRepositoryImpl implements IWidgetRepository {
this._urlWidgets = new UrlWidgetRepository(this._staticUrl);
repos.push(this._urlWidgets);
}
if (this._dirWidgets) { repos.push(this._dirWidgets); }
if (this._diskWidgets) { repos.push(this._diskWidgets); }
this._combinedWidgets = new CombinedWidgetRepository(repos);
}
@@ -200,7 +206,6 @@ class CachedWidgetRepository extends WidgetRepositoryImpl {
const list = await super.getWidgets();
// Cache only if there are some widgets.
if (list.length) { this._cache.set(1, list); }
console.log("CACHABLE RESULT", {list});
return list;
}
@@ -217,20 +222,15 @@ export function buildWidgetRepository(gristServer: GristServer,
options?: {
localOnly: boolean
}) {
if (options?.localOnly) {
return new WidgetRepositoryImpl({
gristServer,
staticUrl: ''
});
}
return new CachedWidgetRepository({
gristServer,
...(options?.localOnly ? { staticUrl: '' } : undefined)
});
}
function fixUrls(widgets: ICustomWidget[], baseUrl: string) {
// If URLs are relative, make them absolute, interpreting them
// relative to the manifest file.
// relative to the supplied base.
for (const widget of widgets) {
if (!(url.parse(widget.url).protocol)) {
widget.url = new URL(widget.url, baseUrl).href;
@@ -238,41 +238,40 @@ function fixUrls(widgets: ICustomWidget[], baseUrl: string) {
}
}
export interface CustomWidgetPlace {
urlBase: string,
fileBase: string,
fileDir: string,
name: string,
/**
* Information about widgets in a plugin. We need to coordinate
* URLs with location on disk.
*/
export interface CustomWidgetsInPlugin {
pluginId: string,
urlBase: string,
dir: string,
file: string,
name: string,
}
export function getWidgetPlaces(gristServer: GristServer,
pluginUrl?: string) {
const places: CustomWidgetPlace[] = [];
/**
* Get a list of widgets available locally via plugins.
*/
export function getWidgetsInPlugins(gristServer: GristServer,
pluginUrl?: string) {
const places: CustomWidgetsInPlugin[] = [];
const plugins = gristServer.getPlugins();
console.log("PLUGINS", plugins);
pluginUrl = pluginUrl || gristServer.getPluginUrl();
if (!pluginUrl) { return []; }
pluginUrl = pluginUrl ?? gristServer.getPluginUrl();
if (pluginUrl === undefined) { return []; }
for (const plugin of plugins) {
console.log("PLUGIN", plugin);
const components = plugin.manifest.components;
if (!components.widgets) { continue; }
console.log("GOT SOMETHING", {
name: plugin.id,
path: plugin.path,
widgets: components.widgets
});
const urlBase =
removeTrailingSlash(pluginUrl) + '/v/' +
gristServer.getTag() + '/widgets/' + plugin.id + '/';
places.push({
urlBase,
fileBase: path.join(plugin.path, components.widgets),
fileDir: plugin.path,
name: plugin.id,
dir: plugin.path,
file: path.join(plugin.path, components.widgets),
name: plugin.manifest.name || plugin.id,
pluginId: plugin.id,
});
}
console.log("PLACES", places);
return places;
}

2
app/server/lib/sendAppPage.ts
View File

@@ -68,7 +68,7 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi
maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,
timestampMs: Date.now(),
enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL),
enableWidgetRepository: Boolean(process.env.GRIST_WIDGET_LIST_URL) || ((server?.getBundledWidgets().length || 0) > 0),
survey: Boolean(process.env.DOC_ID_NEW_USER_INFO),
tagManagerId: process.env.GOOGLE_TAG_MANAGER_ID,
activation: getActivation(req as RequestWithLogin | undefined),