From 02cfcee84d9bff5e9dc1e3059d1cc443981277a4 Mon Sep 17 00:00:00 2001 From: Leslie H <142967379+SleepyLeslie@users.noreply.github.com> Date: Tue, 17 Sep 2024 01:01:58 +0000 Subject: [PATCH] Make changes required for Desktop FS updates (#1099) Make a set of changes required for Desktop FS improvements, see https://github.com/gristlabs/grist-desktop/pull/42 --------- Co-authored-by: Spoffy Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com> --- .gitignore | 8 +- .../ui/{HomeImports.ts => CoreHomeImports.ts} | 66 +---- app/client/ui/CoreNewDocMethods.ts | 47 ++++ app/client/ui/HomeIntro.ts | 6 +- app/client/ui/HomeLeftPane.ts | 72 ++---- app/client/ui/ImportProgress.ts | 58 +++++ app/gen-server/lib/homedb/HomeDBManager.ts | 16 +- app/server/MergedServer.ts | 237 ++++++++++++++++++ app/server/devServerMain.ts | 18 +- app/server/lib/ActiveDocImport.ts | 2 +- app/server/lib/DocStorageManager.ts | 5 +- app/server/lib/ExternalStorage.ts | 9 + app/server/lib/FlexServer.ts | 31 ++- app/server/lib/HostedStorageManager.ts | 18 +- app/server/lib/ICreate.ts | 63 ++++- app/server/mergedServerMain.ts | 230 ----------------- app/server/utils/pruneActionHistory.ts | 4 +- sandbox/watch.sh | 9 +- stubs/app/client/ui/HomeImports.ts | 2 + stubs/app/client/ui/NewDocMethods.ts | 2 + stubs/app/server/lib/logins.ts | 6 - stubs/app/server/server.ts | 16 +- test/gen-server/AuthCaching.ts | 11 +- test/gen-server/apiUtils.ts | 11 +- test/gen-server/lib/DocWorkerMap.ts | 22 +- test/gen-server/lib/mergedOrgs.ts | 7 +- test/server/docTools.ts | 4 +- test/server/lib/HostedStorageManager.ts | 17 +- test/server/lib/helpers/TestServer.ts | 2 +- 29 files changed, 552 insertions(+), 447 deletions(-) rename app/client/ui/{HomeImports.ts => CoreHomeImports.ts} (61%) create mode 100644 app/client/ui/CoreNewDocMethods.ts create mode 100644 app/client/ui/ImportProgress.ts create mode 100644 app/server/MergedServer.ts delete mode 100644 app/server/mergedServerMain.ts create mode 100644 stubs/app/client/ui/HomeImports.ts create mode 100644 stubs/app/client/ui/NewDocMethods.ts delete mode 100644 stubs/app/server/lib/logins.ts diff --git a/.gitignore b/.gitignore index 95d698a2..307875b2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,11 @@ /sandbox_venv* /.vscode/ +# Files created by grist-desktop setup +/cpython.tar.gz +/python +/static_ext + # Build helper files. /.build* @@ -82,7 +87,8 @@ xunit.xml **/_build # ext directory can be overwritten -ext/** +/ext +/ext/** # Docker compose examples - persistent values and secrets /docker-compose-examples/*/persist diff --git a/app/client/ui/HomeImports.ts b/app/client/ui/CoreHomeImports.ts similarity index 61% rename from app/client/ui/HomeImports.ts rename to app/client/ui/CoreHomeImports.ts index 466e3e73..fe5ccf20 100644 --- a/app/client/ui/HomeImports.ts +++ b/app/client/ui/CoreHomeImports.ts @@ -1,13 +1,13 @@ +import {AppModel, reportError} from 'app/client/models/AppModel'; +import {AxiosProgressEvent} from 'axios'; import {PluginScreen} from 'app/client/components/PluginScreen'; import {guessTimezone} from 'app/client/lib/guessTimezone'; import {ImportSourceElement} from 'app/client/lib/ImportSourceElement'; -import {IMPORTABLE_EXTENSIONS, uploadFiles} from 'app/client/lib/uploads'; -import {AppModel, reportError} from 'app/client/models/AppModel'; -import {IProgress} from 'app/client/models/NotifyModel'; +import {ImportProgress} from 'app/client/ui/ImportProgress'; +import {IMPORTABLE_EXTENSIONS} from 'app/client/lib/uploads'; import {openFilePicker} from 'app/client/ui/FileDialog'; import {byteString} from 'app/common/gutil'; -import { AxiosProgressEvent } from 'axios'; -import {Disposable} from 'grainjs'; +import {uploadFiles} from 'app/client/lib/uploads'; /** * Imports a document and returns its docId, or null if no files were selected. @@ -66,62 +66,6 @@ export async function fileImport( progressUI.dispose(); } } - -export class ImportProgress extends Disposable { - // Import does upload first, then import. We show a single indicator, estimating which fraction - // of the time should be given to upload (whose progress we can report well), and which to the - // subsequent import (whose progress indicator is mostly faked). - private _uploadFraction: number; - private _estImportSeconds: number; - - private _importTimer: null | ReturnType = null; - private _importStart: number = 0; - - constructor(private _progressUI: IProgress, file: File) { - super(); - // We'll assume that for .grist files, the upload takes 90% of the total time, and for other - // files, 40%. - this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4; - - // TODO: Import step should include a progress callback, to be combined with upload progress. - // Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and - // use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful, - // but does slow down for larger files, and is more comforting than a stuck indicator. - this._estImportSeconds = file.size / 1024 / 1024 * 2; - - this._progressUI.setProgress(0); - this.onDispose(() => this._importTimer && clearInterval(this._importTimer)); - } - - // Once this reaches 100, the import stage begins. - public setUploadProgress(percentage: number) { - this._progressUI.setProgress(percentage * this._uploadFraction); - if (percentage >= 100 && !this._importTimer) { - this._importStart = Date.now(); - this._importTimer = setInterval(() => this._onImportTimer(), 100); - } - } - - public finish() { - if (this._importTimer) { - clearInterval(this._importTimer); - } - this._progressUI.setProgress(100); - } - - /** - * Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically - * approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the - * estimate is good, and to keep showing slowing progress even if it's not. - */ - private _onImportTimer() { - const elapsedSeconds = (Date.now() - this._importStart) / 1000; - const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds); - const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction); - this._progressUI.setProgress(100 * progress); - } -} - /** * Imports document through a plugin from a home/welcome screen. */ diff --git a/app/client/ui/CoreNewDocMethods.ts b/app/client/ui/CoreNewDocMethods.ts new file mode 100644 index 00000000..e84483f8 --- /dev/null +++ b/app/client/ui/CoreNewDocMethods.ts @@ -0,0 +1,47 @@ +import {homeImports} from 'app/client/ui/HomeImports'; +import {docUrl, urlState} from 'app/client/models/gristUrlState'; +import {HomeModel} from 'app/client/models/HomeModel'; +import {ImportSourceElement} from 'app/client/lib/ImportSourceElement'; +import {reportError} from 'app/client/models/AppModel'; + +export async function createDocAndOpen(home: HomeModel) { + const destWS = home.newDocWorkspace.get(); + if (!destWS) { return; } + try { + const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id); + // Fetch doc information including urlId. + // TODO: consider changing API to return same response as a GET when creating an + // object, which is a semi-standard. + const doc = await home.app.api.getDoc(docId); + await urlState().pushUrl(docUrl(doc)); + } catch (err) { + reportError(err); + } +} + +export async function importDocAndOpen(home: HomeModel) { + const destWS = home.newDocWorkspace.get(); + if (!destWS) { return; } + const docId = await homeImports.docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id); + if (docId) { + const doc = await home.app.api.getDoc(docId); + await urlState().pushUrl(docUrl(doc)); + } +} + +export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) { + try { + const destWS = home.newDocWorkspace.get(); + if (!destWS) { return; } + const docId = await homeImports.importFromPlugin( + home.app, + destWS === "unsaved" ? "unsaved" : destWS.id, + source); + if (docId) { + const doc = await home.app.api.getDoc(docId); + await urlState().pushUrl(docUrl(doc)); + } + } catch (err) { + reportError(err); + } +} diff --git a/app/client/ui/HomeIntro.ts b/app/client/ui/HomeIntro.ts index 23ea1cd1..32ceeba1 100644 --- a/app/client/ui/HomeIntro.ts +++ b/app/client/ui/HomeIntro.ts @@ -3,7 +3,7 @@ import {getLoginOrSignupUrl, getLoginUrl, getSignupUrl, urlState} from 'app/clie import {HomeModel} from 'app/client/models/HomeModel'; import {productPill} from 'app/client/ui/AppHeader'; import * as css from 'app/client/ui/DocMenuCss'; -import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/HomeLeftPane'; +import {newDocMethods} from 'app/client/ui/NewDocMethods'; import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager'; import {bigBasicButton, cssButton} from 'app/client/ui2018/buttons'; import {testId, theme, vars} from 'app/client/ui2018/cssVars'; @@ -177,11 +177,11 @@ function buildButtons(homeModel: HomeModel, options: { ), !options.import ? null : cssBtn(cssBtnIcon('Import'), t("Import Document"), testId('intro-import-doc'), - dom.on('click', () => importDocAndOpen(homeModel)), + dom.on('click', () => newDocMethods.importDocAndOpen(homeModel)), ), !options.empty ? null : cssBtn(cssBtnIcon('Page'), t("Create Empty Document"), testId('intro-create-doc'), - dom.on('click', () => createDocAndOpen(homeModel)), + dom.on('click', () => newDocMethods.createDocAndOpen(homeModel)), ), ); } diff --git a/app/client/ui/HomeLeftPane.ts b/app/client/ui/HomeLeftPane.ts index 1489b6ab..054745d3 100644 --- a/app/client/ui/HomeLeftPane.ts +++ b/app/client/ui/HomeLeftPane.ts @@ -1,29 +1,27 @@ import {makeT} from 'app/client/lib/localization'; import {loadUserManager} from 'app/client/lib/imports'; -import {ImportSourceElement} from 'app/client/lib/ImportSourceElement'; -import {reportError} from 'app/client/models/AppModel'; -import {docUrl, urlState} from 'app/client/models/gristUrlState'; +import {urlState} from 'app/client/models/gristUrlState'; import {HomeModel} from 'app/client/models/HomeModel'; import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo'; import {getAdminPanelName} from 'app/client/ui/AdminPanelName'; +import * as roles from 'app/common/roles'; import {addNewButton, cssAddNewButton} from 'app/client/ui/AddNewButton'; -import {docImport, importFromPlugin} from 'app/client/ui/HomeImports'; +import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls'; +import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs'; +import {newDocMethods} from 'app/client/ui/NewDocMethods'; +import {createHelpTools, cssLeftPanel, cssScrollPane, + cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon'; import { cssLinkText, cssMenuTrigger, cssPageEntry, cssPageIcon, cssPageLink, cssSpacer } from 'app/client/ui/LeftPanelCommon'; -import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour'; -import {transientInput} from 'app/client/ui/transientInput'; -import {testId, theme} from 'app/client/ui2018/cssVars'; -import {icon} from 'app/client/ui2018/icons'; import {menu, menuIcon, menuItem, upgradableMenuItem, upgradeText} from 'app/client/ui2018/menus'; +import {testId, theme} from 'app/client/ui2018/cssVars'; import {confirmModal} from 'app/client/ui2018/modals'; -import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls'; -import * as roles from 'app/common/roles'; +import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour'; import {getGristConfig} from 'app/common/urlUtils'; +import {icon} from 'app/client/ui2018/icons'; +import {transientInput} from 'app/client/ui/transientInput'; import {Workspace} from 'app/common/UserAPI'; -import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs'; -import {createHelpTools, cssLeftPanel, cssScrollPane, - cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon'; const t = makeT('HomeLeftPane'); @@ -160,65 +158,23 @@ export function createHomeLeftPane(leftPanelOpen: Observable, home: Hom ); } -export async function createDocAndOpen(home: HomeModel) { - const destWS = home.newDocWorkspace.get(); - if (!destWS) { return; } - try { - const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id); - // Fetch doc information including urlId. - // TODO: consider changing API to return same response as a GET when creating an - // object, which is a semi-standard. - const doc = await home.app.api.getDoc(docId); - await urlState().pushUrl(docUrl(doc)); - } catch (err) { - reportError(err); - } -} - -export async function importDocAndOpen(home: HomeModel) { - const destWS = home.newDocWorkspace.get(); - if (!destWS) { return; } - const docId = await docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id); - if (docId) { - const doc = await home.app.api.getDoc(docId); - await urlState().pushUrl(docUrl(doc)); - } -} - -export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) { - try { - const destWS = home.newDocWorkspace.get(); - if (!destWS) { return; } - const docId = await importFromPlugin( - home.app, - destWS === "unsaved" ? "unsaved" : destWS.id, - source); - if (docId) { - const doc = await home.app.api.getDoc(docId); - await urlState().pushUrl(docUrl(doc)); - } - } catch (err) { - reportError(err); - } -} - function addMenu(home: HomeModel, creating: Observable): DomElementArg[] { const org = home.app.currentOrg; const orgAccess: roles.Role|null = org ? org.access : null; const needUpgrade = home.app.currentFeatures?.maxWorkspacesPerOrg === 1; return [ - menuItem(() => createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"), + menuItem(() => newDocMethods.createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"), dom.cls('disabled', !home.newDocWorkspace.get()), testId("dm-new-doc") ), - menuItem(() => importDocAndOpen(home), menuIcon('Import'), t("Import Document"), + menuItem(() => newDocMethods.importDocAndOpen(home), menuIcon('Import'), t("Import Document"), dom.cls('disabled', !home.newDocWorkspace.get()), testId("dm-import") ), domComputed(home.importSources, importSources => ([ ...importSources.map((source, i) => - menuItem(() => importFromPluginAndOpen(home, source), + menuItem(() => newDocMethods.importFromPluginAndOpen(home, source), menuIcon('Import'), source.importSource.label, dom.cls('disabled', !home.newDocWorkspace.get()), diff --git a/app/client/ui/ImportProgress.ts b/app/client/ui/ImportProgress.ts new file mode 100644 index 00000000..ce6d52a2 --- /dev/null +++ b/app/client/ui/ImportProgress.ts @@ -0,0 +1,58 @@ +import {IProgress} from 'app/client/models/NotifyModel'; +import {Disposable} from 'grainjs'; + +export class ImportProgress extends Disposable { + // Import does upload first, then import. We show a single indicator, estimating which fraction + // of the time should be given to upload (whose progress we can report well), and which to the + // subsequent import (whose progress indicator is mostly faked). + private _uploadFraction: number; + private _estImportSeconds: number; + + private _importTimer: null | ReturnType = null; + private _importStart: number = 0; + + constructor(private _progressUI: IProgress, file: File) { + super(); + // We'll assume that for .grist files, the upload takes 90% of the total time, and for other + // files, 40%. + this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4; + + // TODO: Import step should include a progress callback, to be combined with upload progress. + // Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and + // use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful, + // but does slow down for larger files, and is more comforting than a stuck indicator. + this._estImportSeconds = file.size / 1024 / 1024 * 2; + + this._progressUI.setProgress(0); + this.onDispose(() => this._importTimer && clearInterval(this._importTimer)); + } + + // Once this reaches 100, the import stage begins. + public setUploadProgress(percentage: number) { + this._progressUI.setProgress(percentage * this._uploadFraction); + if (percentage >= 100 && !this._importTimer) { + this._importStart = Date.now(); + this._importTimer = setInterval(() => this._onImportTimer(), 100); + } + } + + public finish() { + if (this._importTimer) { + clearInterval(this._importTimer); + } + this._progressUI.setProgress(100); + } + + /** + * Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically + * approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the + * estimate is good, and to keep showing slowing progress even if it's not. + */ + private _onImportTimer() { + const elapsedSeconds = (Date.now() - this._importStart) / 1000; + const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds); + const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction); + this._progressUI.setProgress(100 * progress); + } +} + diff --git a/app/gen-server/lib/homedb/HomeDBManager.ts b/app/gen-server/lib/homedb/HomeDBManager.ts index 09587870..e28a2c87 100644 --- a/app/gen-server/lib/homedb/HomeDBManager.ts +++ b/app/gen-server/lib/homedb/HomeDBManager.ts @@ -68,12 +68,12 @@ import {Request} from "express"; import {defaultsDeep, flatten, pick} from 'lodash'; import { Brackets, - Connection, DatabaseType, + DataSource, EntityManager, ObjectLiteral, SelectQueryBuilder, - WhereExpression + WhereExpressionBuilder } from "typeorm"; import uuidv4 from "uuid/v4"; @@ -247,7 +247,7 @@ export type BillingOptions = Partial(qb: T, org: string|number, includeSupport = false): T { + private _whereOrg(qb: T, org: string|number, includeSupport = false): T { if (this.isMergedOrg(org)) { // Select from universe of personal orgs. // Don't panic though! While this means that SQL can't use an organization id @@ -3458,7 +3462,7 @@ export class HomeDBManager extends EventEmitter { } } - private _wherePlainOrg(qb: T, org: string|number): T { + private _wherePlainOrg(qb: T, org: string|number): T { if (typeof org === 'number') { return qb.andWhere('orgs.id = :org', {org}); } diff --git a/app/server/MergedServer.ts b/app/server/MergedServer.ts new file mode 100644 index 00000000..ba307e80 --- /dev/null +++ b/app/server/MergedServer.ts @@ -0,0 +1,237 @@ +/** + * + * A version of hosted grist that recombines a home server, + * a doc worker, and a static server on a single port. + * + */ + +import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer'; +import log from 'app/server/lib/log'; +import {getGlobalConfig} from "app/server/lib/globalConfig"; + +// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS +// environment variable. +export type ServerType = "home" | "docs" | "static" | "app"; +const allServerTypes: ServerType[] = ["home", "docs", "static", "app"]; + +// Parse a comma-separate list of server types into an array, with validation. +export function parseServerTypes(serverTypes: string|undefined): ServerType[] { + // Split and filter out empty strings (including the one we get when splitting ""). + const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part)); + + // Check that parts is non-empty and only contains valid options. + if (!types.length) { + throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`); + } + for (const t of types) { + if (!allServerTypes.includes(t as ServerType)) { + throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`); + } + } + return types as ServerType[]; +} + +function checkUserContentPort(): number | null { + // Check whether a port is explicitly set for user content. + if (process.env.GRIST_UNTRUSTED_PORT) { + return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10); + } + // Checks whether to serve user content on same domain but on different port + if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) { + const homeUrl = new URL(process.env.APP_HOME_URL); + const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL); + // If the hostname of both home and plugin url are the same, + // but the ports are different + if (homeUrl.hostname === pluginUrl.hostname && + homeUrl.port !== pluginUrl.port) { + const port = parseInt(pluginUrl.port || '80', 10); + return port; + } + } + return null; +} + +interface ServerOptions extends FlexServerOptions { + // If set, messages logged to console (default: false) + // (but if options are not given at all in call to main, logToConsole is set to true) + logToConsole?: boolean; + + // If set, documents saved to external storage such as s3 (default is to check environment variables, + // which get set in various ways in dev/test entry points) + externalStorage?: boolean; +} + +export class MergedServer { + + public static async create(port: number, serverTypes: ServerType[], options: ServerOptions = {}) { + options.settings ??= getGlobalConfig(); + const ms = new MergedServer(port, serverTypes, options); + // We need to know early on whether we will be serving plugins or not. + if (ms.hasComponent("home")) { + const userPort = checkUserContentPort(); + ms.flexServer.setServesPlugins(userPort !== undefined); + } else { + ms.flexServer.setServesPlugins(false); + } + + ms.flexServer.addCleanup(); + ms.flexServer.setDirectory(); + + if (process.env.GRIST_TEST_ROUTER) { + // Add a mock api for adding/removing doc workers from load balancer. + ms.flexServer.testAddRouter(); + } + + if (ms._options.logToConsole !== false) { ms.flexServer.addLogging(); } + if (ms._options.externalStorage === false) { ms.flexServer.disableExternalStorage(); } + await ms.flexServer.addLoginMiddleware(); + + if (ms.hasComponent("docs")) { + // It is important that /dw and /v prefixes are accepted (if present) by health check + // in ms case, since they are included in the url registered for the doc worker. + ms.flexServer.stripDocWorkerIdPathPrefixIfPresent(); + ms.flexServer.addTagChecker(); + } + + ms.flexServer.addHealthCheck(); + if (ms.hasComponent("home") || ms.hasComponent("app")) { + ms.flexServer.addBootPage(); + } + ms.flexServer.denyRequestsIfNotReady(); + + if (ms.hasComponent("home") || ms.hasComponent("static") || ms.hasComponent("app")) { + ms.flexServer.setDirectory(); + } + + if (ms.hasComponent("home") || ms.hasComponent("static")) { + ms.flexServer.addStaticAndBowerDirectories(); + } + + await ms.flexServer.initHomeDBManager(); + ms.flexServer.addHosts(); + + ms.flexServer.addDocWorkerMap(); + + if (ms.hasComponent("home") || ms.hasComponent("static")) { + await ms.flexServer.addAssetsForPlugins(); + } + + if (ms.hasComponent("home")) { + ms.flexServer.addEarlyWebhooks(); + } + + if (ms.hasComponent("home") || ms.hasComponent("docs") || ms.hasComponent("app")) { + ms.flexServer.addSessions(); + } + + ms.flexServer.addAccessMiddleware(); + ms.flexServer.addApiMiddleware(); + await ms.flexServer.addBillingMiddleware(); + + return ms; + } + + public readonly flexServer: FlexServer; + private readonly _serverTypes: ServerType[]; + private readonly _options: ServerOptions; + + private constructor(port: number, serverTypes: ServerType[], options: ServerOptions = {}) { + this._serverTypes = serverTypes; + this._options = options; + this.flexServer = new FlexServer(port, `server(${serverTypes.join(",")})`, options); + } + + public hasComponent(serverType: ServerType) { + return this._serverTypes.includes(serverType); + } + + + public async run() { + + try { + await this.flexServer.start(); + + if (this.hasComponent("home")) { + this.flexServer.addUsage(); + if (!this.hasComponent("docs")) { + this.flexServer.addDocApiForwarder(); + } + this.flexServer.addJsonSupport(); + this.flexServer.addUpdatesCheck(); + await this.flexServer.addLandingPages(); + // todo: add support for home api to standalone app + this.flexServer.addHomeApi(); + this.flexServer.addBillingApi(); + this.flexServer.addNotifier(); + this.flexServer.addAuditLogger(); + await this.flexServer.addTelemetry(); + await this.flexServer.addHousekeeper(); + await this.flexServer.addLoginRoutes(); + this.flexServer.addAccountPage(); + this.flexServer.addBillingPages(); + this.flexServer.addWelcomePaths(); + this.flexServer.addLogEndpoint(); + this.flexServer.addGoogleAuthEndpoint(); + this.flexServer.addInstallEndpoints(); + this.flexServer.addConfigEndpoints(); + } + + if (this.hasComponent("docs")) { + this.flexServer.addJsonSupport(); + this.flexServer.addAuditLogger(); + await this.flexServer.addTelemetry(); + await this.flexServer.addDoc(); + } + + if (this.hasComponent("home")) { + this.flexServer.addClientSecrets(); + } + + this.flexServer.finalizeEndpoints(); + await this.flexServer.finalizePlugins(this.hasComponent("home") ? checkUserContentPort() : null); + this.flexServer.checkOptionCombinations(); + this.flexServer.summary(); + this.flexServer.ready(); + + // Some tests have their timing perturbed by having this earlier + // TODO: update those tests. + if (this.hasComponent("docs")) { + await this.flexServer.checkSandbox(); + } + } catch(e) { + await this.flexServer.close(); + throw e; + } + } +} + +export async function startMain() { + try { + + const serverTypes = parseServerTypes(process.env.GRIST_SERVERS); + + // No defaults for a port, since this server can serve very different purposes. + if (!process.env.GRIST_PORT) { + throw new Error("GRIST_PORT must be specified"); + } + + const port = parseInt(process.env.GRIST_PORT, 10); + + const server = await MergedServer.create(port, serverTypes); + await server.run(); + + const opt = process.argv[2]; + if (opt === '--testingHooks') { + await server.flexServer.addTestingHooks(); + } + + return server.flexServer; + } catch (e) { + log.error('mergedServer failed to start', e); + process.exit(1); + } +} + +if (require.main === module) { + startMain().catch((e) => log.error('mergedServer failed to start', e)); +} diff --git a/app/server/devServerMain.ts b/app/server/devServerMain.ts index 296f24b7..c2b3960b 100644 --- a/app/server/devServerMain.ts +++ b/app/server/devServerMain.ts @@ -21,7 +21,7 @@ import {updateDb} from 'app/server/lib/dbUtils'; import {FlexServer} from 'app/server/lib/FlexServer'; import log from 'app/server/lib/log'; -import {main as mergedServerMain} from 'app/server/mergedServerMain'; +import {MergedServer} from 'app/server/MergedServer'; import {promisifyAll} from 'bluebird'; import * as fse from 'fs-extra'; import * as path from 'path'; @@ -96,8 +96,9 @@ export async function main() { if (!process.env.APP_HOME_URL) { process.env.APP_HOME_URL = `http://localhost:${port}`; } - const server = await mergedServerMain(port, ["home", "docs", "static"]); - await server.addTestingHooks(); + const mergedServer = await MergedServer.create(port, ["home", "docs", "static"]); + await mergedServer.flexServer.addTestingHooks(); + await mergedServer.run(); return; } @@ -118,17 +119,18 @@ export async function main() { log.info("== staticServer"); const staticPort = getPort("STATIC_PORT", 9001); process.env.APP_STATIC_URL = `http://localhost:${staticPort}`; - await mergedServerMain(staticPort, ["static"]); + await MergedServer.create(staticPort, ["static"]).then((s) => s.run()); // Bring up a home server log.info("=========================================================================="); log.info("== homeServer"); - const home = await mergedServerMain(homeServerPort, ["home"]); + const homeServer = await MergedServer.create(homeServerPort, ["home"]); + await homeServer.run(); // If a distinct webServerPort is specified, we listen also on that port, though serving // exactly the same content. This is handy for testing CORS issues. if (webServerPort !== 0 && webServerPort !== homeServerPort) { - await home.startCopy('webServer', webServerPort); + await homeServer.flexServer.startCopy('webServer', webServerPort); } // Bring up the docWorker(s) @@ -147,10 +149,10 @@ export async function main() { } const workers = new Array(); for (const port of ports) { - workers.push(await mergedServerMain(port, ["docs"])); + workers.push((await MergedServer.create(port, ["docs"])).flexServer); } - await home.addTestingHooks(workers); + await homeServer.flexServer.addTestingHooks(workers); } diff --git a/app/server/lib/ActiveDocImport.ts b/app/server/lib/ActiveDocImport.ts index 038b27fa..7e6c911f 100644 --- a/app/server/lib/ActiveDocImport.ts +++ b/app/server/lib/ActiveDocImport.ts @@ -382,7 +382,7 @@ export class ActiveDocImport { * @param {String} tmpPath: The path from of the original file. * @param {FileImportOptions} importOptions: File import options. * @returns {Promise} with `options` property containing parseOptions as serialized JSON as adjusted - * or guessed by the plugin, and `tables`, which is which is a list of objects with information about + * or guessed by the plugin, and `tables`, which is a list of objects with information about * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`. */ private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string, diff --git a/app/server/lib/DocStorageManager.ts b/app/server/lib/DocStorageManager.ts index 7ec65de4..cde167be 100644 --- a/app/server/lib/DocStorageManager.ts +++ b/app/server/lib/DocStorageManager.ts @@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage'; import * as gutil from 'app/common/gutil'; import {Comm} from 'app/server/lib/Comm'; import * as docUtils from 'app/server/lib/docUtils'; -import {GristServer} from 'app/server/lib/GristServer'; import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager'; import {IShell} from 'app/server/lib/IShell'; import log from 'app/server/lib/log'; @@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager { * The file watcher is created if the optComm argument is given. */ constructor(private _docsRoot: string, private _samplesRoot?: string, - private _comm?: Comm, gristServer?: GristServer) { + private _comm?: Comm, shell?: IShell) { // If we have a way to communicate with clients, watch the docsRoot for changes. this._watcher = null; - this._shell = gristServer?.create.Shell?.() || { + this._shell = shell ?? { trashItem() { throw new Error('Unable to move document to trash'); }, showItemInFolder() { throw new Error('Unable to show item in folder'); } }; diff --git a/app/server/lib/ExternalStorage.ts b/app/server/lib/ExternalStorage.ts index c92fb0bb..74394ba5 100644 --- a/app/server/lib/ExternalStorage.ts +++ b/app/server/lib/ExternalStorage.ts @@ -377,6 +377,15 @@ export interface ExternalStorageSettings { extraPrefix?: string; } +/** + * Function returning the core ExternalStorage implementation, + * which may then be wrapped in additional layer(s) of ExternalStorage. + * See ICreate.ExternalStorage. + * Uses S3 by default in hosted Grist. +*/ +export type ExternalStorageCreator = + (purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined; + /** * The storage mapping we use for our SaaS. A reasonable default, but relies * on appropriate lifecycle rules being set up in the bucket. diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index b29e9ffa..b295ca96 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -1,7 +1,6 @@ import {ApiError} from 'app/common/ApiError'; import {ICustomWidget} from 'app/common/CustomWidget'; import {delay} from 'app/common/delay'; -import {DocCreationInfo} from 'app/common/DocListAPI'; import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes, GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls'; @@ -38,7 +37,6 @@ import {create} from 'app/server/lib/create'; import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect'; import {addDocApiRoutes} from 'app/server/lib/DocApi'; import {DocManager} from 'app/server/lib/DocManager'; -import {DocStorageManager} from 'app/server/lib/DocStorageManager'; import {DocWorker} from 'app/server/lib/DocWorker'; import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap'; @@ -47,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth"; import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer, RequestWithGrist} from 'app/server/lib/GristServer'; import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions'; -import {HostedStorageManager} from 'app/server/lib/HostedStorageManager'; import {IBilling} from 'app/server/lib/IBilling'; import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier'; import {InstallAdmin} from 'app/server/lib/InstallAdmin'; import log from 'app/server/lib/log'; -import {getLoginSystem} from 'app/server/lib/logins'; import {IPermitStore} from 'app/server/lib/Permit'; import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places'; import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'; @@ -185,7 +181,7 @@ export class FlexServer implements GristServer { private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise; private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise; private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise; - private _getLoginSystem?: () => Promise; + private _getLoginSystem: () => Promise; // Set once ready() is called private _isReady: boolean = false; private _updateManager: UpdateManager; @@ -193,6 +189,7 @@ export class FlexServer implements GristServer { constructor(public port: number, public name: string = 'flexServer', public readonly options: FlexServerOptions = {}) { + this._getLoginSystem = create.getLoginSystem; this.settings = options.settings; this.app = express(); this.app.set('port', port); @@ -250,7 +247,6 @@ export class FlexServer implements GristServer { recentItems: [], }; this.electronServerMethods = { - async importDoc() { throw new Error('not implemented'); }, onDocOpen(cb) { // currently only a stub. cb(''); @@ -272,11 +268,6 @@ export class FlexServer implements GristServer { }); } - // Allow overridding the login system. - public setLoginSystem(loginSystem: () => Promise) { - this._getLoginSystem = loginSystem; - } - public getHost(): string { return `${this.host}:${this.getOwnPort()}`; } @@ -405,6 +396,11 @@ export class FlexServer implements GristServer { return this._auditLogger; } + public getDocManager(): DocManager { + if (!this._docManager) { throw new Error('no document manager available'); } + return this._docManager; + } + public getTelemetry(): ITelemetry { if (!this._telemetry) { throw new Error('no telemetry available'); } return this._telemetry; @@ -1341,12 +1337,15 @@ export class FlexServer implements GristServer { const workers = this._docWorkerMap; const docWorkerId = await this._addSelfAsWorker(workers); - const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers, - this._dbManager, this.create); + const storageManager = await this.create.createHostedDocStorageManager( + this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage + ); this._storageManager = storageManager; } else { const samples = getAppPathTo(this.appRoot, 'public_samples'); - const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this); + const storageManager = await this.create.createLocalDocStorageManager( + this.docsRoot, samples, this._comm, this.create.Shell?.() + ); this._storageManager = storageManager; } @@ -2012,8 +2011,7 @@ export class FlexServer implements GristServer { public resolveLoginSystem() { return isTestLoginAllowed() ? - getTestLoginSystem() : - (this._getLoginSystem?.() || getLoginSystem()); + getTestLoginSystem() : this._getLoginSystem(); } public addUpdatesCheck() { @@ -2609,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne // Methods that Electron app relies on. export interface ElectronServerMethods { - importDoc(filepath: string): Promise; onDocOpen(cb: (filePath: string) => void): void; getUserConfig(): Promise; updateUserConfig(obj: any): Promise; diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts index 88e5317f..443ab52b 100644 --- a/app/server/lib/HostedStorageManager.ts +++ b/app/server/lib/HostedStorageManager.ts @@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {checksumFile} from 'app/server/lib/checksumFile'; import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots'; import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; -import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage'; +import { + ChecksummedExternalStorage, + DELETED_TOKEN, + ExternalStorage, + ExternalStorageCreator, ExternalStorageSettings, + Unchanged +} from 'app/server/lib/ExternalStorage'; import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager'; -import {ICreate} from 'app/server/lib/ICreate'; import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager'; import {LogMethods} from "app/server/lib/LogMethods"; import {fromCallback} from 'app/server/lib/serverUtils'; @@ -51,11 +56,6 @@ export interface HostedStorageOptions { secondsBeforePush: number; secondsBeforeFirstRetry: number; pushDocUpdateTimes: boolean; - // A function returning the core ExternalStorage implementation, - // which may then be wrapped in additional layer(s) of ExternalStorage. - // See ICreate.ExternalStorage. - // Uses S3 by default in hosted Grist. - externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage; } const defaultOptions: HostedStorageOptions = { @@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager { private _disableS3: boolean, private _docWorkerMap: IDocWorkerMap, dbManager: HomeDBManager, - create: ICreate, + createExternalStorage: ExternalStorageCreator, options: HostedStorageOptions = defaultOptions ) { - const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, '')); + const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, '')); // We store documents either in a test store, or in an s3 store // at s3:///.grist const externalStoreDoc = this._disableS3 ? undefined : creator('doc'); diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index 1914b335..af234f39 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -1,10 +1,11 @@ import {GristDeploymentType} from 'app/common/gristUrls'; +import {getCoreLoginSystem} from 'app/server/lib/coreLogins'; import {getThemeBackgroundSnippet} from 'app/common/Themes'; import {Document} from 'app/gen-server/entity/Document'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {IAuditLogger} from 'app/server/lib/AuditLogger'; -import {ExternalStorage} from 'app/server/lib/ExternalStorage'; -import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer'; +import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage'; +import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer'; import {IBilling} from 'app/server/lib/IBilling'; import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier'; import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin'; @@ -13,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell'; import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox'; import {SqliteVariant} from 'app/server/lib/SqliteCommon'; import {ITelemetry} from 'app/server/lib/Telemetry'; +import {IDocStorageManager} from './IDocStorageManager'; +import { Comm } from "./Comm"; +import { IDocWorkerMap } from "./DocWorkerMap"; +import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager"; +import { DocStorageManager } from "./DocStorageManager"; // In the past, the session secret was used as an additional // protection passed on to expressjs-session for security when @@ -37,7 +43,30 @@ import {ITelemetry} from 'app/server/lib/Telemetry'; export const DEFAULT_SESSION_SECRET = 'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh'; +export type LocalDocStorageManagerCreator = + (docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise; +export type HostedDocStorageManagerCreator = ( + docsRoot: string, + docWorkerId: string, + disableS3: boolean, + docWorkerMap: IDocWorkerMap, + dbManager: HomeDBManager, + createExternalStorage: ExternalStorageCreator, + options?: HostedStorageOptions + ) => Promise; + export interface ICreate { + // Create a space to store files externally, for storing either: + // - documents. This store should be versioned, and can be eventually consistent. + // - meta. This store need not be versioned, and can be eventually consistent. + // For test purposes an extra prefix may be supplied. Stores with different prefixes + // should not interfere with each other. + ExternalStorage: ExternalStorageCreator; + + // Creates a IDocStorageManager for storing documents on the local machine. + createLocalDocStorageManager: LocalDocStorageManagerCreator; + // Creates a IDocStorageManager for storing documents on an external storage (e.g S3) + createHostedDocStorageManager: HostedDocStorageManagerCreator; Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling; Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier; @@ -45,13 +74,6 @@ export interface ICreate { Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry; Shell?(): IShell; // relevant to electron version of Grist only. - // Create a space to store files externally, for storing either: - // - documents. This store should be versioned, and can be eventually consistent. - // - meta. This store need not be versioned, and can be eventually consistent. - // For test purposes an extra prefix may be supplied. Stores with different prefixes - // should not interfere with each other. - ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined; - NSandbox(options: ISandboxCreationOptions): ISandbox; // Create the logic to determine which users are authorized to manage this Grist installation. @@ -69,6 +91,8 @@ export interface ICreate { getStorageOptions?(name: string): ICreateStorageOptions|undefined; getSqliteVariant?(): SqliteVariant; getSandboxVariants?(): Record; + + getLoginSystem(): Promise; } export interface ICreateActiveDocOptions { @@ -126,6 +150,9 @@ export function makeSimpleCreator(opts: { getSqliteVariant?: () => SqliteVariant, getSandboxVariants?: () => Record, createInstallAdmin?: (dbManager: HomeDBManager) => Promise, + getLoginSystem?: () => Promise, + createHostedDocStorageManager?: HostedDocStorageManagerCreator, + createLocalDocStorageManager?: LocalDocStorageManagerCreator, }): ICreate { const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts; return { @@ -199,5 +226,23 @@ export function makeSimpleCreator(opts: { getSqliteVariant: opts.getSqliteVariant, getSandboxVariants: opts.getSandboxVariants, createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)), + getLoginSystem: opts.getLoginSystem || getCoreLoginSystem, + createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager, + createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager, }; } + +const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async ( + docsRoot, + docWorkerId, + disableS3, + docWorkerMap, + dbManager, + createExternalStorage, options +) => + new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options); + +const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async ( + docsRoot, samplesRoot, comm, shell +) => new DocStorageManager(docsRoot, samplesRoot, comm, shell); + diff --git a/app/server/mergedServerMain.ts b/app/server/mergedServerMain.ts deleted file mode 100644 index ea7039d0..00000000 --- a/app/server/mergedServerMain.ts +++ /dev/null @@ -1,230 +0,0 @@ -/** - * - * A version of hosted grist that recombines a home server, - * a doc worker, and a static server on a single port. - * - */ - -import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer'; -import {GristLoginSystem} from 'app/server/lib/GristServer'; -import log from 'app/server/lib/log'; -import {getGlobalConfig} from "app/server/lib/globalConfig"; - -// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS -// environment variable. -export type ServerType = "home" | "docs" | "static" | "app"; -const allServerTypes: ServerType[] = ["home", "docs", "static", "app"]; - -// Parse a comma-separate list of server types into an array, with validation. -export function parseServerTypes(serverTypes: string|undefined): ServerType[] { - // Split and filter out empty strings (including the one we get when splitting ""). - const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part)); - - // Check that parts is non-empty and only contains valid options. - if (!types.length) { - throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`); - } - for (const t of types) { - if (!allServerTypes.includes(t as ServerType)) { - throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`); - } - } - return types as ServerType[]; -} - -function checkUserContentPort(): number | null { - // Check whether a port is explicitly set for user content. - if (process.env.GRIST_UNTRUSTED_PORT) { - return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10); - } - // Checks whether to serve user content on same domain but on different port - if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) { - const homeUrl = new URL(process.env.APP_HOME_URL); - const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL); - // If the hostname of both home and plugin url are the same, - // but the ports are different - if (homeUrl.hostname === pluginUrl.hostname && - homeUrl.port !== pluginUrl.port) { - const port = parseInt(pluginUrl.port || '80', 10); - return port; - } - } - return null; -} - -interface ServerOptions extends FlexServerOptions { - logToConsole?: boolean; // If set, messages logged to console (default: false) - // (but if options are not given at all in call to main, - // logToConsole is set to true) - externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment - // variables, which get set in various ways in dev/test entry points) - loginSystem?: () => Promise; -} - -/** - * Start a server on the given port, including the functionality specified in serverTypes. - */ -export async function main(port: number, serverTypes: ServerType[], - options: ServerOptions = {}) { - const includeHome = serverTypes.includes("home"); - const includeDocs = serverTypes.includes("docs"); - const includeStatic = serverTypes.includes("static"); - const includeApp = serverTypes.includes("app"); - - options.settings ??= getGlobalConfig(); - - const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options); - - // We need to know early on whether we will be serving plugins or not. - if (includeHome) { - const userPort = checkUserContentPort(); - server.setServesPlugins(userPort !== undefined); - } else { - server.setServesPlugins(false); - } - - if (options.loginSystem) { - server.setLoginSystem(options.loginSystem); - } - - server.addCleanup(); - server.setDirectory(); - - if (process.env.GRIST_TEST_ROUTER) { - // Add a mock api for adding/removing doc workers from load balancer. - server.testAddRouter(); - } - - if (options.logToConsole !== false) { server.addLogging(); } - if (options.externalStorage === false) { server.disableExternalStorage(); } - await server.addLoginMiddleware(); - - if (includeDocs) { - // It is important that /dw and /v prefixes are accepted (if present) by health check - // in this case, since they are included in the url registered for the doc worker. - server.stripDocWorkerIdPathPrefixIfPresent(); - server.addTagChecker(); - } - - server.addHealthCheck(); - if (includeHome || includeApp) { - server.addBootPage(); - } - server.denyRequestsIfNotReady(); - - if (includeHome || includeStatic || includeApp) { - server.setDirectory(); - } - - if (includeHome || includeStatic) { - server.addStaticAndBowerDirectories(); - } - - await server.initHomeDBManager(); - server.addHosts(); - - server.addDocWorkerMap(); - - if (includeHome || includeStatic) { - await server.addAssetsForPlugins(); - } - - if (includeHome) { - server.addEarlyWebhooks(); - } - - if (includeHome || includeDocs || includeApp) { - server.addSessions(); - } - - server.addAccessMiddleware(); - server.addApiMiddleware(); - await server.addBillingMiddleware(); - - try { - await server.start(); - - if (includeHome) { - server.addUsage(); - if (!includeDocs) { - server.addDocApiForwarder(); - } - server.addJsonSupport(); - server.addUpdatesCheck(); - await server.addLandingPages(); - // todo: add support for home api to standalone app - server.addHomeApi(); - server.addBillingApi(); - server.addNotifier(); - server.addAuditLogger(); - await server.addTelemetry(); - await server.addHousekeeper(); - await server.addLoginRoutes(); - server.addAccountPage(); - server.addBillingPages(); - server.addWelcomePaths(); - server.addLogEndpoint(); - server.addGoogleAuthEndpoint(); - server.addInstallEndpoints(); - server.addConfigEndpoints(); - } - - if (includeDocs) { - server.addJsonSupport(); - server.addAuditLogger(); - await server.addTelemetry(); - await server.addDoc(); - } - - if (includeHome) { - server.addClientSecrets(); - } - - server.finalizeEndpoints(); - await server.finalizePlugins(includeHome ? checkUserContentPort() : null); - server.checkOptionCombinations(); - server.summary(); - server.ready(); - - // Some tests have their timing perturbed by having this earlier - // TODO: update those tests. - if (includeDocs) { - await server.checkSandbox(); - } - return server; - } catch(e) { - await server.close(); - throw e; - } -} - - -export async function startMain() { - try { - - const serverTypes = parseServerTypes(process.env.GRIST_SERVERS); - - // No defaults for a port, since this server can serve very different purposes. - if (!process.env.GRIST_PORT) { - throw new Error("GRIST_PORT must be specified"); - } - - const port = parseInt(process.env.GRIST_PORT, 10); - - const server = await main(port, serverTypes); - - const opt = process.argv[2]; - if (opt === '--testingHooks') { - await server.addTestingHooks(); - } - - return server; - } catch (e) { - log.error('mergedServer failed to start', e); - process.exit(1); - } -} - -if (require.main === module) { - startMain().catch((e) => log.error('mergedServer failed to start', e)); -} diff --git a/app/server/utils/pruneActionHistory.ts b/app/server/utils/pruneActionHistory.ts index 818454e9..c66c2b41 100644 --- a/app/server/utils/pruneActionHistory.ts +++ b/app/server/utils/pruneActionHistory.ts @@ -1,9 +1,9 @@ import * as gutil from 'app/common/gutil'; import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl'; import {DocStorage} from 'app/server/lib/DocStorage'; -import {DocStorageManager} from 'app/server/lib/DocStorageManager'; import * as docUtils from 'app/server/lib/docUtils'; import log from 'app/server/lib/log'; +import {create} from "app/server/lib/create"; /** * A utility script for cleaning up the action log. @@ -18,7 +18,7 @@ export async function pruneActionHistory(docPath: string, keepN: number) { throw new Error('Invalid document: Document should be a valid .grist file'); } - const storageManager = new DocStorageManager(".", "."); + const storageManager = await create.createLocalDocStorageManager(".", "."); const docStorage = new DocStorage(storageManager, docPath); const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist"; diff --git a/sandbox/watch.sh b/sandbox/watch.sh index 531c97f6..200268d2 100755 --- a/sandbox/watch.sh +++ b/sandbox/watch.sh @@ -2,6 +2,13 @@ set -x +NO_NODEMON=false +for arg in $@; do + if [[ $arg == "--no-nodemon" ]]; then + NO_NODEMON=true + fi +done + PROJECT="" if [[ -e ext/app ]]; then PROJECT="tsconfig-ext.json" @@ -19,6 +26,6 @@ tsc --build -w --preserveWatchOutput $PROJECT & css_files="app/client/**/*.css" chokidar "${css_files}" -c "bash -O globstar -c 'cat ${css_files} > static/bundle.css'" & webpack --config $WEBPACK_CONFIG --mode development --watch & -NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js & +! $NO_NODEMON && NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js & wait diff --git a/stubs/app/client/ui/HomeImports.ts b/stubs/app/client/ui/HomeImports.ts new file mode 100644 index 00000000..d995aa62 --- /dev/null +++ b/stubs/app/client/ui/HomeImports.ts @@ -0,0 +1,2 @@ +import * as coreHomeImports from "app/client/ui/CoreHomeImports"; +export const homeImports = coreHomeImports; diff --git a/stubs/app/client/ui/NewDocMethods.ts b/stubs/app/client/ui/NewDocMethods.ts new file mode 100644 index 00000000..130e6029 --- /dev/null +++ b/stubs/app/client/ui/NewDocMethods.ts @@ -0,0 +1,2 @@ +import * as coreNewDocMethods from "app/client/ui/CoreNewDocMethods"; +export const newDocMethods = coreNewDocMethods; diff --git a/stubs/app/server/lib/logins.ts b/stubs/app/server/lib/logins.ts deleted file mode 100644 index d038ebcb..00000000 --- a/stubs/app/server/lib/logins.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { getCoreLoginSystem } from "app/server/lib/coreLogins"; -import { GristLoginSystem } from "app/server/lib/GristServer"; - -export async function getLoginSystem(): Promise { - return getCoreLoginSystem(); -} diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts index 921def02..c7e5b9eb 100644 --- a/stubs/app/server/server.ts +++ b/stubs/app/server/server.ts @@ -34,7 +34,7 @@ setDefaultEnv('GRIST_UI_FEATURES', 'helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,createSite,supportGrist'); setDefaultEnv('GRIST_WIDGET_LIST_URL', commonUrls.gristLabsWidgetRepository); import {updateDb} from 'app/server/lib/dbUtils'; -import {main as mergedServerMain, parseServerTypes} from 'app/server/mergedServerMain'; +import {MergedServer, parseServerTypes} from 'app/server/MergedServer'; import * as fse from 'fs-extra'; import {runPrometheusExporter} from './prometheus-exporter'; @@ -124,20 +124,20 @@ export async function main() { } // Launch single-port, self-contained version of Grist. - const server = await mergedServerMain(G.port, serverTypes); + const mergedServer = await MergedServer.create(G.port, serverTypes); + await mergedServer.run(); if (process.env.GRIST_TESTING_SOCKET) { - await server.addTestingHooks(); + await mergedServer.flexServer.addTestingHooks(); } if (process.env.GRIST_SERVE_PLUGINS_PORT) { - await server.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10)); + await mergedServer.flexServer.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10)); } - await fixSiteProducts({ - deploymentType: server.getDeploymentType(), - db: server.getHomeDBManager() + deploymentType: mergedServer.flexServer.getDeploymentType(), + db: mergedServer.flexServer.getHomeDBManager() }); - return server; + return mergedServer.flexServer; } if (require.main === module) { diff --git a/test/gen-server/AuthCaching.ts b/test/gen-server/AuthCaching.ts index d28f7382..0f02f1f9 100644 --- a/test/gen-server/AuthCaching.ts +++ b/test/gen-server/AuthCaching.ts @@ -2,7 +2,7 @@ import {delay} from 'app/common/delay'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {FlexServer} from 'app/server/lib/FlexServer'; import log from 'app/server/lib/log'; -import {main as mergedServerMain} from 'app/server/mergedServerMain'; +import {MergedServer} from 'app/server/MergedServer'; import axios from 'axios'; import {assert} from 'chai'; import * as fse from 'fs-extra'; @@ -50,12 +50,17 @@ describe('AuthCaching', function() { setUpDB(); await createInitialDb(); process.env.GRIST_DATA_DIR = testDocDir; - homeServer = await mergedServerMain(0, ['home'], + + const homeMS = await MergedServer.create(0, ['home'], {logToConsole: false, externalStorage: false}); + await homeMS.run(); + homeServer = homeMS.flexServer; homeUrl = homeServer.getOwnUrl(); process.env.APP_HOME_URL = homeUrl; - docsServer = await mergedServerMain(0, ['docs'], + const docsMS = await MergedServer.create(0, ['docs'], {logToConsole: false, externalStorage: false}); + await docsMS.run(); + docsServer = docsMS.flexServer; // Helpers for getting cookie-based logins. session = new TestSession(homeServer); diff --git a/test/gen-server/apiUtils.ts b/test/gen-server/apiUtils.ts index 2dbaf73c..dac549e0 100644 --- a/test/gen-server/apiUtils.ts +++ b/test/gen-server/apiUtils.ts @@ -14,7 +14,7 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import * as docUtils from 'app/server/lib/docUtils'; import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer'; -import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain'; +import {MergedServer, ServerType} from 'app/server/MergedServer'; import axios from 'axios'; import FormData from 'form-data'; import fetch from 'node-fetch'; @@ -37,9 +37,10 @@ export class TestServer { public async start(servers: ServerType[] = ["home"], options: FlexServerOptions = {}): Promise { await createInitialDb(); - this.server = await mergedServerMain(0, servers, {logToConsole: isAffirmative(process.env.DEBUG), - externalStorage: false, - ...options}); + const mergedServer = await MergedServer.create(0, servers, {logToConsole: isAffirmative(process.env.DEBUG), + externalStorage: false, ...options}); + await mergedServer.run(); + this.server = mergedServer.flexServer; this.serverUrl = this.server.getOwnUrl(); this.dbManager = this.server.getHomeDBManager(); this.defaultSession = new TestSession(this.server); @@ -263,7 +264,7 @@ export class TestSession { if (clearCache) { this.home.getSessions().clearCacheIfNeeded(); } this.headers.Cookie = cookie; return { - validateStatus: (status: number) => true, + validateStatus: (_status: number) => true, headers: { 'Cookie': cookie, 'X-Requested-With': 'XMLHttpRequest', diff --git a/test/gen-server/lib/DocWorkerMap.ts b/test/gen-server/lib/DocWorkerMap.ts index 73b7d21c..9bcf5c4a 100644 --- a/test/gen-server/lib/DocWorkerMap.ts +++ b/test/gen-server/lib/DocWorkerMap.ts @@ -2,7 +2,7 @@ import {DocWorkerMap, getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap'; import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap'; import {FlexServer} from 'app/server/lib/FlexServer'; import {Permit} from 'app/server/lib/Permit'; -import {main as mergedServerMain} from 'app/server/mergedServerMain'; +import {MergedServer} from "app/server/MergedServer"; import {delay, promisifyAll} from 'bluebird'; import {assert, expect} from 'chai'; import {countBy, values} from 'lodash'; @@ -387,24 +387,34 @@ describe('DocWorkerMap', function() { process.env.REDIS_URL = process.env.TEST_REDIS_URL; // Make home server. - const home = await mergedServerMain(0, ['home'], opts); + const homeMergedServer = await MergedServer.create(0, ['home'], opts); + const home = homeMergedServer.flexServer; + await homeMergedServer.run(); // Make a worker, not associated with any group. process.env.GRIST_DOC_WORKER_ID = 'worker1'; - const docs1 = await mergedServerMain(0, ['docs'], opts); + const docs1MergedServer = await MergedServer.create(0, ['docs'], opts); + const docs1 = docs1MergedServer.flexServer; + await docs1MergedServer.run(); // Make a worker in "special" group. process.env.GRIST_DOC_WORKER_ID = 'worker2'; process.env.GRIST_WORKER_GROUP = 'special'; - const docs2 = await mergedServerMain(0, ['docs'], opts); + const docs2MergedServer = await MergedServer.create(0, ['docs'], opts); + const docs2 = docs2MergedServer.flexServer; + await docs2MergedServer.run(); // Make two worker in "other" group. process.env.GRIST_DOC_WORKER_ID = 'worker3'; process.env.GRIST_WORKER_GROUP = 'other'; - const docs3 = await mergedServerMain(0, ['docs'], opts); + const docs3MergedServer = await MergedServer.create(0, ['docs'], opts); + const docs3 = docs3MergedServer.flexServer; + await docs3MergedServer.run(); process.env.GRIST_DOC_WORKER_ID = 'worker4'; process.env.GRIST_WORKER_GROUP = 'other'; - const docs4 = await mergedServerMain(0, ['docs'], opts); + const docs4MergedServer = await MergedServer.create(0, ['docs'], opts); + const docs4 = docs4MergedServer.flexServer; + await docs4MergedServer.run(); servers = {home, docs1, docs2, docs3, docs4}; workers = getDocWorkerMap(); diff --git a/test/gen-server/lib/mergedOrgs.ts b/test/gen-server/lib/mergedOrgs.ts index b66860fc..eb71fd61 100644 --- a/test/gen-server/lib/mergedOrgs.ts +++ b/test/gen-server/lib/mergedOrgs.ts @@ -1,7 +1,7 @@ import {Workspace} from 'app/common/UserAPI'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {FlexServer} from 'app/server/lib/FlexServer'; -import {main as mergedServerMain} from 'app/server/mergedServerMain'; +import {MergedServer} from "app/server/MergedServer"; import axios from 'axios'; import {assert} from 'chai'; import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed'; @@ -9,6 +9,7 @@ import {configForUser, createUser, setPlan} from 'test/gen-server/testUtils'; import * as testUtils from 'test/server/testUtils'; describe('mergedOrgs', function() { + let mergedServer: MergedServer; let home: FlexServer; let dbManager: HomeDBManager; let homeUrl: string; @@ -20,8 +21,10 @@ describe('mergedOrgs', function() { before(async function() { setUpDB(this); await createInitialDb(); - home = await mergedServerMain(0, ["home", "docs"], + mergedServer = await MergedServer.create(0, ["home", "docs"], {logToConsole: false, externalStorage: false}); + home = mergedServer.flexServer; + await mergedServer.run(); dbManager = home.getHomeDBManager(); homeUrl = home.getOwnUrl(); }); diff --git a/test/server/docTools.ts b/test/server/docTools.ts index 2b8cadfe..72043ff8 100644 --- a/test/server/docTools.ts +++ b/test/server/docTools.ts @@ -4,7 +4,6 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {DummyAuthorizer} from 'app/server/lib/Authorizer'; import {DocManager} from 'app/server/lib/DocManager'; import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession'; -import {DocStorageManager} from 'app/server/lib/DocStorageManager'; import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer'; import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import {getAppRoot} from 'app/server/lib/places'; @@ -17,6 +16,7 @@ import * as fse from 'fs-extra'; import {tmpdir} from 'os'; import * as path from 'path'; import * as tmp from 'tmp'; +import {create} from "app/server/lib/create"; tmp.setGracefulCleanup(); @@ -138,7 +138,7 @@ export async function createDocManager( server?: GristServer} = {}): Promise { // Set Grist home to a temporary directory, and wipe it out on exit. const tmpDir = options.tmpDir || await createTmpDir(); - const docStorageManager = options.storageManager || new DocStorageManager(tmpDir); + const docStorageManager = options.storageManager || await create.createLocalDocStorageManager(tmpDir); const pluginManager = options.pluginManager || await getGlobalPluginManager(); const store = getDocWorkerMap(); const internalPermitStore = store.getPermitStore('1'); diff --git a/test/server/lib/HostedStorageManager.ts b/test/server/lib/HostedStorageManager.ts index aa164892..285bea29 100644 --- a/test/server/lib/HostedStorageManager.ts +++ b/test/server/lib/HostedStorageManager.ts @@ -7,7 +7,12 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {create} from 'app/server/lib/create'; import {DocManager} from 'app/server/lib/DocManager'; import {makeExceptionalDocSession} from 'app/server/lib/DocSession'; -import {DELETED_TOKEN, ExternalStorage, wrapWithKeyMappedStorage} from 'app/server/lib/ExternalStorage'; +import { + DELETED_TOKEN, + ExternalStorage, ExternalStorageCreator, + ExternalStorageSettings, + wrapWithKeyMappedStorage +} from 'app/server/lib/ExternalStorage'; import {createDummyGristServer} from 'app/server/lib/GristServer'; import { BackupEvent, @@ -270,7 +275,7 @@ class TestStore { private _localDirectory: string, private _workerId: string, private _workers: DocWorkerMap, - private _externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined) { + private _externalStorageCreate: ExternalStorageCreator) { } public async run(fn: () => Promise): Promise { @@ -296,18 +301,20 @@ class TestStore { secondsBeforeFirstRetry: 3, // rumors online suggest delays of 10-11 secs // are not super-unusual. pushDocUpdateTimes: false, - externalStorageCreator: (purpose) => { + + }; + const externalStorageCreator = (purpose: ExternalStorageSettings["purpose"]) => { const result = this._externalStorageCreate(purpose, this._extraPrefix); if (!result) { throw new Error('no storage'); } return result; - } }; + const storageManager = new HostedStorageManager(this._localDirectory, this._workerId, false, this._workers, dbManager, - create, + externalStorageCreator, options); this.storageManager = storageManager; this.docManager = new DocManager(storageManager, await getGlobalPluginManager(), diff --git a/test/server/lib/helpers/TestServer.ts b/test/server/lib/helpers/TestServer.ts index 080abb1e..53cba45a 100644 --- a/test/server/lib/helpers/TestServer.ts +++ b/test/server/lib/helpers/TestServer.ts @@ -91,7 +91,7 @@ export class TestServer { ...this._defaultEnv, ...customEnv }; - const main = await testUtils.getBuildFile('app/server/mergedServerMain.js'); + const main = await testUtils.getBuildFile('app/server/MergedServer.js'); this._server = spawn('node', [main, '--testingHooks'], { env, stdio: ['inherit', serverLog, serverLog]