Make changes required for Desktop FS updates (#1099)

Make a set of changes required for Desktop FS improvements, see
https://github.com/gristlabs/grist-desktop/pull/42

---------

Co-authored-by: Spoffy <contact@spoffy.net>
Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com>

.gitignore

@@ -12,6 +12,11 @@
 /sandbox_venv*
 /.vscode/
 
+# Files created by grist-desktop setup
+/cpython.tar.gz
+/python
+/static_ext
+
 # Build helper files.
 /.build*
 
@@ -82,7 +87,8 @@ xunit.xml
 **/_build
 
 # ext directory can be overwritten
-ext/**
+/ext
+/ext/**
 
 # Docker compose examples - persistent values and secrets
 /docker-compose-examples/*/persist

app/client/ui/CoreHomeImports.ts

@@ -1,13 +1,13 @@
+import {AppModel, reportError} from 'app/client/models/AppModel';
+import {AxiosProgressEvent} from 'axios';
 import {PluginScreen} from 'app/client/components/PluginScreen';
 import {guessTimezone} from 'app/client/lib/guessTimezone';
 import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
-import {IMPORTABLE_EXTENSIONS, uploadFiles} from 'app/client/lib/uploads';
+import {ImportProgress} from 'app/client/ui/ImportProgress';
-import {AppModel, reportError} from 'app/client/models/AppModel';
+import {IMPORTABLE_EXTENSIONS} from 'app/client/lib/uploads';
-import {IProgress} from 'app/client/models/NotifyModel';
 import {openFilePicker} from 'app/client/ui/FileDialog';
 import {byteString} from 'app/common/gutil';
-import { AxiosProgressEvent } from 'axios';
+import {uploadFiles} from 'app/client/lib/uploads';
-import {Disposable} from 'grainjs';
 
 /**
  * Imports a document and returns its docId, or null if no files were selected.
@@ -66,62 +66,6 @@ export async function fileImport(
     progressUI.dispose();
   }
 }
-
-export class ImportProgress extends Disposable {
-  // Import does upload first, then import. We show a single indicator, estimating which fraction
-  // of the time should be given to upload (whose progress we can report well), and which to the
-  // subsequent import (whose progress indicator is mostly faked).
-  private _uploadFraction: number;
-  private _estImportSeconds: number;
-
-  private _importTimer: null | ReturnType<typeof setInterval> = null;
-  private _importStart: number = 0;
-
-  constructor(private _progressUI: IProgress, file: File) {
-    super();
-    // We'll assume that for .grist files, the upload takes 90% of the total time, and for other
-    // files, 40%.
-    this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
-
-    // TODO: Import step should include a progress callback, to be combined with upload progress.
-    // Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
-    // use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
-    // but does slow down for larger files, and is more comforting than a stuck indicator.
-    this._estImportSeconds = file.size / 1024 / 1024 * 2;
-
-    this._progressUI.setProgress(0);
-    this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
-  }
-
-  // Once this reaches 100, the import stage begins.
-  public setUploadProgress(percentage: number) {
-    this._progressUI.setProgress(percentage * this._uploadFraction);
-    if (percentage >= 100 && !this._importTimer) {
-      this._importStart = Date.now();
-      this._importTimer = setInterval(() => this._onImportTimer(), 100);
-    }
-  }
-
-  public finish() {
-    if (this._importTimer) {
-      clearInterval(this._importTimer);
-    }
-    this._progressUI.setProgress(100);
-  }
-
-  /**
-   * Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
-   * approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
-   * estimate is good, and to keep showing slowing progress even if it's not.
-   */
-  private _onImportTimer() {
-    const elapsedSeconds = (Date.now() - this._importStart) / 1000;
-    const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
-    const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
-    this._progressUI.setProgress(100 * progress);
-  }
-}
-
 /**
  * Imports document through a plugin from a home/welcome screen.
  */

app/client/ui/CoreNewDocMethods.ts

@@ -0,0 +1,47 @@
+import {homeImports} from 'app/client/ui/HomeImports';
+import {docUrl, urlState} from 'app/client/models/gristUrlState';
+import {HomeModel} from 'app/client/models/HomeModel';
+import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
+import {reportError} from 'app/client/models/AppModel';
+
+export async function createDocAndOpen(home: HomeModel) {
+  const destWS = home.newDocWorkspace.get();
+  if (!destWS) { return; }
+  try {
+    const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
+    // Fetch doc information including urlId.
+    // TODO: consider changing API to return same response as a GET when creating an
+    // object, which is a semi-standard.
+    const doc = await home.app.api.getDoc(docId);
+    await urlState().pushUrl(docUrl(doc));
+  } catch (err) {
+    reportError(err);
+  }
+}
+
+export async function importDocAndOpen(home: HomeModel) {
+  const destWS = home.newDocWorkspace.get();
+  if (!destWS) { return; }
+  const docId = await homeImports.docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
+  if (docId) {
+    const doc = await home.app.api.getDoc(docId);
+    await urlState().pushUrl(docUrl(doc));
+  }
+}
+
+export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
+  try {
+    const destWS = home.newDocWorkspace.get();
+    if (!destWS) { return; }
+    const docId = await homeImports.importFromPlugin(
+      home.app,
+      destWS === "unsaved" ? "unsaved" : destWS.id,
+      source);
+    if (docId) {
+      const doc = await home.app.api.getDoc(docId);
+      await urlState().pushUrl(docUrl(doc));
+    }
+  } catch (err) {
+    reportError(err);
+  }
+}

app/client/ui/HomeIntro.ts

@@ -3,7 +3,7 @@ import {getLoginOrSignupUrl, getLoginUrl, getSignupUrl, urlState} from 'app/clie
 import {HomeModel} from 'app/client/models/HomeModel';
 import {productPill} from 'app/client/ui/AppHeader';
 import * as css from 'app/client/ui/DocMenuCss';
-import {createDocAndOpen, importDocAndOpen} from 'app/client/ui/HomeLeftPane';
+import {newDocMethods} from 'app/client/ui/NewDocMethods';
 import {manageTeamUsersApp} from 'app/client/ui/OpenUserManager';
 import {bigBasicButton, cssButton} from 'app/client/ui2018/buttons';
 import {testId, theme, vars} from 'app/client/ui2018/cssVars';
@@ -177,11 +177,11 @@ function buildButtons(homeModel: HomeModel, options: {
     ),
     !options.import ? null :
     cssBtn(cssBtnIcon('Import'), t("Import Document"), testId('intro-import-doc'),
-      dom.on('click', () => importDocAndOpen(homeModel)),
+      dom.on('click', () => newDocMethods.importDocAndOpen(homeModel)),
     ),
     !options.empty ? null :
     cssBtn(cssBtnIcon('Page'), t("Create Empty Document"), testId('intro-create-doc'),
-      dom.on('click', () => createDocAndOpen(homeModel)),
+      dom.on('click', () => newDocMethods.createDocAndOpen(homeModel)),
     ),
   );
 }

app/client/ui/HomeLeftPane.ts

@@ -1,29 +1,27 @@
 import {makeT} from 'app/client/lib/localization';
 import {loadUserManager} from 'app/client/lib/imports';
-import {ImportSourceElement} from 'app/client/lib/ImportSourceElement';
+import {urlState} from 'app/client/models/gristUrlState';
-import {reportError} from 'app/client/models/AppModel';
-import {docUrl, urlState} from 'app/client/models/gristUrlState';
 import {HomeModel} from 'app/client/models/HomeModel';
 import {getWorkspaceInfo, workspaceName} from 'app/client/models/WorkspaceInfo';
 import {getAdminPanelName} from 'app/client/ui/AdminPanelName';
+import * as roles from 'app/common/roles';
 import {addNewButton, cssAddNewButton} from 'app/client/ui/AddNewButton';
-import {docImport, importFromPlugin} from 'app/client/ui/HomeImports';
+import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
+import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
+import {newDocMethods} from 'app/client/ui/NewDocMethods';
+import {createHelpTools, cssLeftPanel, cssScrollPane,
+  cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
 import {
   cssLinkText, cssMenuTrigger, cssPageEntry, cssPageIcon, cssPageLink, cssSpacer
 } from 'app/client/ui/LeftPanelCommon';
-import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
-import {transientInput} from 'app/client/ui/transientInput';
-import {testId, theme} from 'app/client/ui2018/cssVars';
-import {icon} from 'app/client/ui2018/icons';
 import {menu, menuIcon, menuItem, upgradableMenuItem, upgradeText} from 'app/client/ui2018/menus';
+import {testId, theme} from 'app/client/ui2018/cssVars';
 import {confirmModal} from 'app/client/ui2018/modals';
-import {commonUrls, isFeatureEnabled} from 'app/common/gristUrls';
+import {createVideoTourToolsButton} from 'app/client/ui/OpenVideoTour';
-import * as roles from 'app/common/roles';
 import {getGristConfig} from 'app/common/urlUtils';
+import {icon} from 'app/client/ui2018/icons';
+import {transientInput} from 'app/client/ui/transientInput';
 import {Workspace} from 'app/common/UserAPI';
-import {computed, dom, domComputed, DomElementArg, observable, Observable, styled} from 'grainjs';
-import {createHelpTools, cssLeftPanel, cssScrollPane,
-        cssSectionHeader, cssTools} from 'app/client/ui/LeftPanelCommon';
 
 const t = makeT('HomeLeftPane');
 
@@ -160,65 +158,23 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
   );
 }
 
-export async function createDocAndOpen(home: HomeModel) {
-  const destWS = home.newDocWorkspace.get();
-  if (!destWS) { return; }
-  try {
-    const docId = await home.createDoc("Untitled document", destWS === "unsaved" ? "unsaved" : destWS.id);
-    // Fetch doc information including urlId.
-    // TODO: consider changing API to return same response as a GET when creating an
-    // object, which is a semi-standard.
-    const doc = await home.app.api.getDoc(docId);
-    await urlState().pushUrl(docUrl(doc));
-  } catch (err) {
-    reportError(err);
-  }
-}
-
-export async function importDocAndOpen(home: HomeModel) {
-  const destWS = home.newDocWorkspace.get();
-  if (!destWS) { return; }
-  const docId = await docImport(home.app, destWS === "unsaved" ? "unsaved" : destWS.id);
-  if (docId) {
-    const doc = await home.app.api.getDoc(docId);
-    await urlState().pushUrl(docUrl(doc));
-  }
-}
-
-export async function importFromPluginAndOpen(home: HomeModel, source: ImportSourceElement) {
-  try {
-    const destWS = home.newDocWorkspace.get();
-    if (!destWS) { return; }
-    const docId = await importFromPlugin(
-      home.app,
-      destWS === "unsaved" ? "unsaved" : destWS.id,
-      source);
-    if (docId) {
-      const doc = await home.app.api.getDoc(docId);
-      await urlState().pushUrl(docUrl(doc));
-    }
-  } catch (err) {
-    reportError(err);
-  }
-}
-
 function addMenu(home: HomeModel, creating: Observable<boolean>): DomElementArg[] {
   const org = home.app.currentOrg;
   const orgAccess: roles.Role|null = org ? org.access : null;
   const needUpgrade = home.app.currentFeatures?.maxWorkspacesPerOrg === 1;
 
   return [
-    menuItem(() => createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
+    menuItem(() => newDocMethods.createDocAndOpen(home), menuIcon('Page'), t("Create Empty Document"),
       dom.cls('disabled', !home.newDocWorkspace.get()),
       testId("dm-new-doc")
     ),
-    menuItem(() => importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
+    menuItem(() => newDocMethods.importDocAndOpen(home), menuIcon('Import'), t("Import Document"),
       dom.cls('disabled', !home.newDocWorkspace.get()),
       testId("dm-import")
     ),
     domComputed(home.importSources, importSources => ([
       ...importSources.map((source, i) =>
-      menuItem(() => importFromPluginAndOpen(home, source),
+      menuItem(() => newDocMethods.importFromPluginAndOpen(home, source),
         menuIcon('Import'),
         source.importSource.label,
         dom.cls('disabled', !home.newDocWorkspace.get()),

app/client/ui/ImportProgress.ts

@@ -0,0 +1,58 @@
+import {IProgress} from 'app/client/models/NotifyModel';
+import {Disposable} from 'grainjs';
+
+export class ImportProgress extends Disposable {
+  // Import does upload first, then import. We show a single indicator, estimating which fraction
+  // of the time should be given to upload (whose progress we can report well), and which to the
+  // subsequent import (whose progress indicator is mostly faked).
+  private _uploadFraction: number;
+  private _estImportSeconds: number;
+
+  private _importTimer: null | ReturnType<typeof setInterval> = null;
+  private _importStart: number = 0;
+
+  constructor(private _progressUI: IProgress, file: File) {
+    super();
+    // We'll assume that for .grist files, the upload takes 90% of the total time, and for other
+    // files, 40%.
+    this._uploadFraction = file.name.endsWith(".grist") ? 0.9 : 0.4;
+
+    // TODO: Import step should include a progress callback, to be combined with upload progress.
+    // Without it, we estimate import to take 2s per MB (non-scientific unreliable estimate), and
+    // use an asymptotic indicator which keeps moving without ever finishing. Not terribly useful,
+    // but does slow down for larger files, and is more comforting than a stuck indicator.
+    this._estImportSeconds = file.size / 1024 / 1024 * 2;
+
+    this._progressUI.setProgress(0);
+    this.onDispose(() => this._importTimer && clearInterval(this._importTimer));
+  }
+
+  // Once this reaches 100, the import stage begins.
+  public setUploadProgress(percentage: number) {
+    this._progressUI.setProgress(percentage * this._uploadFraction);
+    if (percentage >= 100 && !this._importTimer) {
+      this._importStart = Date.now();
+      this._importTimer = setInterval(() => this._onImportTimer(), 100);
+    }
+  }
+
+  public finish() {
+    if (this._importTimer) {
+      clearInterval(this._importTimer);
+    }
+    this._progressUI.setProgress(100);
+  }
+
+  /**
+   * Calls _progressUI.setProgress(percent) with percentage increasing from 0 and asymptotically
+   * approaching 100, reaching 50% after estSeconds. It's intended to look reasonable when the
+   * estimate is good, and to keep showing slowing progress even if it's not.
+   */
+  private _onImportTimer() {
+    const elapsedSeconds = (Date.now() - this._importStart) / 1000;
+    const importProgress = elapsedSeconds / (elapsedSeconds + this._estImportSeconds);
+    const progress = this._uploadFraction + importProgress * (1 - this._uploadFraction);
+    this._progressUI.setProgress(100 * progress);
+  }
+}
+

app/gen-server/lib/homedb/HomeDBManager.ts

@@ -68,12 +68,12 @@ import {Request} from "express";
 import {defaultsDeep, flatten, pick} from 'lodash';
 import {
   Brackets,
-  Connection,
   DatabaseType,
+  DataSource,
   EntityManager,
   ObjectLiteral,
   SelectQueryBuilder,
-  WhereExpression
+  WhereExpressionBuilder
 } from "typeorm";
 import uuidv4 from "uuid/v4";
 
@@ -247,7 +247,7 @@ export type BillingOptions = Partial<Pick<BillingAccount,
  */
 export class HomeDBManager extends EventEmitter {
   private _usersManager = new UsersManager(this, this._runInTransaction.bind(this));
-  private _connection: Connection;
+  private _connection: DataSource;
   private _exampleWorkspaceId: number;
   private _exampleOrgId: number;
   private _idPrefix: string = "";  // Place this before ids in subdomains, used in routing to
@@ -353,7 +353,7 @@ export class HomeDBManager extends EventEmitter {
     this._connection = await getOrCreateConnection();
   }
 
-  public connectTo(connection: Connection) {
+  public connectTo(connection: DataSource) {
     this._connection = connection;
   }
 
@@ -987,6 +987,10 @@ export class HomeDBManager extends EventEmitter {
     return doc;
   }
 
+  public async getAllDocs() {
+    return this.connection.getRepository(Document).find();
+  }
+
   public async getRawDocById(docId: string, transaction?: EntityManager) {
     return await this.getDoc({
       urlId: docId,
@@ -3438,7 +3442,7 @@ export class HomeDBManager extends EventEmitter {
   // Adds a where clause to filter orgs by domain or id.
   // If org is null, filter for user's personal org.
   // if includeSupport is true, include the org of the support@ user (for the Samples workspace)
-  private _whereOrg<T extends WhereExpression>(qb: T, org: string|number, includeSupport = false): T {
+  private _whereOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number, includeSupport = false): T {
     if (this.isMergedOrg(org)) {
       // Select from universe of personal orgs.
       // Don't panic though!  While this means that SQL can't use an organization id
@@ -3458,7 +3462,7 @@ export class HomeDBManager extends EventEmitter {
     }
   }
 
-  private _wherePlainOrg<T extends WhereExpression>(qb: T, org: string|number): T {
+  private _wherePlainOrg<T extends WhereExpressionBuilder>(qb: T, org: string|number): T {
     if (typeof org === 'number') {
       return qb.andWhere('orgs.id = :org', {org});
     }

app/server/MergedServer.ts

@@ -0,0 +1,237 @@
+/**
+ *
+ * A version of hosted grist that recombines a home server,
+ * a doc worker, and a static server on a single port.
+ *
+ */
+
+import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
+import log from 'app/server/lib/log';
+import {getGlobalConfig} from "app/server/lib/globalConfig";
+
+// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
+// environment variable.
+export type ServerType = "home" | "docs" | "static" | "app";
+const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
+
+// Parse a comma-separate list of server types into an array, with validation.
+export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
+  // Split and filter out empty strings (including the one we get when splitting "").
+  const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
+
+  // Check that parts is non-empty and only contains valid options.
+  if (!types.length) {
+    throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
+  }
+  for (const t of types) {
+    if (!allServerTypes.includes(t as ServerType)) {
+      throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
+    }
+  }
+  return types as ServerType[];
+}
+
+function checkUserContentPort(): number | null {
+  // Check whether a port is explicitly set for user content.
+  if (process.env.GRIST_UNTRUSTED_PORT) {
+    return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
+  }
+  // Checks whether to serve user content on same domain but on different port
+  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
+    const homeUrl = new URL(process.env.APP_HOME_URL);
+    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
+    // If the hostname of both home and plugin url are the same,
+    // but the ports are different
+    if (homeUrl.hostname === pluginUrl.hostname &&
+        homeUrl.port !== pluginUrl.port) {
+      const port = parseInt(pluginUrl.port || '80', 10);
+      return port;
+    }
+  }
+  return null;
+}
+
+interface ServerOptions extends FlexServerOptions {
+  // If set, messages logged to console (default: false)
+  // (but if options are not given at all in call to main, logToConsole is set to true)
+  logToConsole?: boolean;
+
+  // If set, documents saved to external storage such as s3 (default is to check environment variables,
+  // which get set in various ways in dev/test entry points)
+  externalStorage?: boolean;
+}
+
+export class MergedServer {
+
+  public static async create(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
+    options.settings ??= getGlobalConfig();
+    const ms = new MergedServer(port, serverTypes, options);
+    // We need to know early on whether we will be serving plugins or not.
+    if (ms.hasComponent("home")) {
+      const userPort = checkUserContentPort();
+      ms.flexServer.setServesPlugins(userPort !== undefined);
+    } else {
+      ms.flexServer.setServesPlugins(false);
+    }
+
+    ms.flexServer.addCleanup();
+    ms.flexServer.setDirectory();
+
+    if (process.env.GRIST_TEST_ROUTER) {
+      // Add a mock api for adding/removing doc workers from load balancer.
+      ms.flexServer.testAddRouter();
+    }
+
+    if (ms._options.logToConsole !== false) { ms.flexServer.addLogging(); }
+    if (ms._options.externalStorage === false) { ms.flexServer.disableExternalStorage(); }
+    await ms.flexServer.addLoginMiddleware();
+
+    if (ms.hasComponent("docs")) {
+      // It is important that /dw and /v prefixes are accepted (if present) by health check
+      // in ms case, since they are included in the url registered for the doc worker.
+      ms.flexServer.stripDocWorkerIdPathPrefixIfPresent();
+      ms.flexServer.addTagChecker();
+    }
+
+    ms.flexServer.addHealthCheck();
+    if (ms.hasComponent("home") || ms.hasComponent("app")) {
+      ms.flexServer.addBootPage();
+    }
+    ms.flexServer.denyRequestsIfNotReady();
+
+    if (ms.hasComponent("home") || ms.hasComponent("static") || ms.hasComponent("app")) {
+      ms.flexServer.setDirectory();
+    }
+
+    if (ms.hasComponent("home") || ms.hasComponent("static")) {
+      ms.flexServer.addStaticAndBowerDirectories();
+    }
+
+    await ms.flexServer.initHomeDBManager();
+    ms.flexServer.addHosts();
+
+    ms.flexServer.addDocWorkerMap();
+
+    if (ms.hasComponent("home") || ms.hasComponent("static")) {
+      await ms.flexServer.addAssetsForPlugins();
+    }
+
+    if (ms.hasComponent("home")) {
+      ms.flexServer.addEarlyWebhooks();
+    }
+
+    if (ms.hasComponent("home") || ms.hasComponent("docs") || ms.hasComponent("app")) {
+      ms.flexServer.addSessions();
+    }
+
+    ms.flexServer.addAccessMiddleware();
+    ms.flexServer.addApiMiddleware();
+    await ms.flexServer.addBillingMiddleware();
+
+    return ms;
+  }
+
+  public readonly flexServer: FlexServer;
+  private readonly _serverTypes: ServerType[];
+  private readonly _options: ServerOptions;
+
+  private constructor(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
+    this._serverTypes = serverTypes;
+    this._options = options;
+    this.flexServer = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
+  }
+
+  public hasComponent(serverType: ServerType) {
+    return this._serverTypes.includes(serverType);
+  }
+
+
+  public async run() {
+
+    try {
+      await this.flexServer.start();
+
+      if (this.hasComponent("home")) {
+        this.flexServer.addUsage();
+        if (!this.hasComponent("docs")) {
+          this.flexServer.addDocApiForwarder();
+        }
+        this.flexServer.addJsonSupport();
+        this.flexServer.addUpdatesCheck();
+        await this.flexServer.addLandingPages();
+        // todo: add support for home api to standalone app
+        this.flexServer.addHomeApi();
+        this.flexServer.addBillingApi();
+        this.flexServer.addNotifier();
+        this.flexServer.addAuditLogger();
+        await this.flexServer.addTelemetry();
+        await this.flexServer.addHousekeeper();
+        await this.flexServer.addLoginRoutes();
+        this.flexServer.addAccountPage();
+        this.flexServer.addBillingPages();
+        this.flexServer.addWelcomePaths();
+        this.flexServer.addLogEndpoint();
+        this.flexServer.addGoogleAuthEndpoint();
+        this.flexServer.addInstallEndpoints();
+        this.flexServer.addConfigEndpoints();
+      }
+
+      if (this.hasComponent("docs")) {
+        this.flexServer.addJsonSupport();
+        this.flexServer.addAuditLogger();
+        await this.flexServer.addTelemetry();
+        await this.flexServer.addDoc();
+      }
+
+      if (this.hasComponent("home")) {
+        this.flexServer.addClientSecrets();
+      }
+
+      this.flexServer.finalizeEndpoints();
+      await this.flexServer.finalizePlugins(this.hasComponent("home") ? checkUserContentPort() : null);
+      this.flexServer.checkOptionCombinations();
+      this.flexServer.summary();
+      this.flexServer.ready();
+
+      // Some tests have their timing perturbed by having this earlier
+      // TODO: update those tests.
+      if (this.hasComponent("docs")) {
+        await this.flexServer.checkSandbox();
+      }
+    } catch(e) {
+      await this.flexServer.close();
+      throw e;
+    }
+  }
+}
+
+export async function startMain() {
+  try {
+
+    const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
+
+    // No defaults for a port, since this server can serve very different purposes.
+    if (!process.env.GRIST_PORT) {
+      throw new Error("GRIST_PORT must be specified");
+    }
+
+    const port = parseInt(process.env.GRIST_PORT, 10);
+
+    const server = await MergedServer.create(port, serverTypes);
+    await server.run();
+
+    const opt = process.argv[2];
+    if (opt === '--testingHooks') {
+      await server.flexServer.addTestingHooks();
+    }
+
+    return server.flexServer;
+  } catch (e) {
+    log.error('mergedServer failed to start', e);
+    process.exit(1);
+  }
+}
+
+if (require.main === module) {
+  startMain().catch((e) => log.error('mergedServer failed to start', e));
+}

app/server/devServerMain.ts

@@ -21,7 +21,7 @@
 import {updateDb} from 'app/server/lib/dbUtils';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import log from 'app/server/lib/log';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from 'app/server/MergedServer';
 import {promisifyAll} from 'bluebird';
 import * as fse from 'fs-extra';
 import * as path from 'path';
@@ -96,8 +96,9 @@ export async function main() {
     if (!process.env.APP_HOME_URL) {
       process.env.APP_HOME_URL = `http://localhost:${port}`;
     }
-    const server = await mergedServerMain(port, ["home", "docs", "static"]);
+    const mergedServer = await MergedServer.create(port, ["home", "docs", "static"]);
-    await server.addTestingHooks();
+    await mergedServer.flexServer.addTestingHooks();
+    await mergedServer.run();
     return;
   }
 
@@ -118,17 +119,18 @@ export async function main() {
   log.info("== staticServer");
   const staticPort = getPort("STATIC_PORT", 9001);
   process.env.APP_STATIC_URL = `http://localhost:${staticPort}`;
-  await mergedServerMain(staticPort, ["static"]);
+  await MergedServer.create(staticPort, ["static"]).then((s) => s.run());
 
   // Bring up a home server
   log.info("==========================================================================");
   log.info("== homeServer");
-  const home = await mergedServerMain(homeServerPort, ["home"]);
+  const homeServer = await MergedServer.create(homeServerPort, ["home"]);
+  await homeServer.run();
 
   // If a distinct webServerPort is specified, we listen also on that port, though serving
   // exactly the same content.  This is handy for testing CORS issues.
   if (webServerPort !== 0 && webServerPort !== homeServerPort) {
-    await home.startCopy('webServer', webServerPort);
+    await homeServer.flexServer.startCopy('webServer', webServerPort);
   }
 
   // Bring up the docWorker(s)
@@ -147,10 +149,10 @@ export async function main() {
   }
   const workers = new Array<FlexServer>();
   for (const port of ports) {
-    workers.push(await mergedServerMain(port, ["docs"]));
+    workers.push((await MergedServer.create(port, ["docs"])).flexServer);
   }
 
-  await home.addTestingHooks(workers);
+  await homeServer.flexServer.addTestingHooks(workers);
 }
 
 

app/server/lib/ActiveDocImport.ts

@@ -382,7 +382,7 @@ export class ActiveDocImport {
    * @param {String} tmpPath: The path from of the original file.
    * @param {FileImportOptions} importOptions: File import options.
    * @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
-   * or guessed by the plugin, and `tables`, which is which is a list of objects with information about
+   * or guessed by the plugin, and `tables`, which is a list of objects with information about
    * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
    */
   private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,

app/server/lib/DocStorageManager.ts

@@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
 import * as gutil from 'app/common/gutil';
 import {Comm} from 'app/server/lib/Comm';
 import * as docUtils from 'app/server/lib/docUtils';
-import {GristServer} from 'app/server/lib/GristServer';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {IShell} from 'app/server/lib/IShell';
 import log from 'app/server/lib/log';
@@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager {
    * The file watcher is created if the optComm argument is given.
    */
   constructor(private _docsRoot: string, private _samplesRoot?: string,
-              private _comm?: Comm, gristServer?: GristServer) {
+              private _comm?: Comm, shell?: IShell) {
     // If we have a way to communicate with clients, watch the docsRoot for changes.
     this._watcher = null;
-    this._shell = gristServer?.create.Shell?.() || {
+    this._shell = shell ?? {
       trashItem() { throw new Error('Unable to move document to trash'); },
       showItemInFolder() { throw new Error('Unable to show item in folder'); }
     };

app/server/lib/ExternalStorage.ts

@@ -377,6 +377,15 @@ export interface ExternalStorageSettings {
   extraPrefix?: string;
 }
 
+/**
+ * Function returning the core ExternalStorage implementation,
+ * which may then be wrapped in additional layer(s) of ExternalStorage.
+ * See ICreate.ExternalStorage.
+ * Uses S3 by default in hosted Grist.
+*/
+export type ExternalStorageCreator =
+  (purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined;
+
 /**
  * The storage mapping we use for our SaaS. A reasonable default, but relies
  * on appropriate lifecycle rules being set up in the bucket.

app/server/lib/FlexServer.ts

@@ -1,7 +1,6 @@
 import {ApiError} from 'app/common/ApiError';
 import {ICustomWidget} from 'app/common/CustomWidget';
 import {delay} from 'app/common/delay';
-import {DocCreationInfo} from 'app/common/DocListAPI';
 import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
         GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
         sanitizePathTail} from 'app/common/gristUrls';
@@ -38,7 +37,6 @@ import {create} from 'app/server/lib/create';
 import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
 import {addDocApiRoutes} from 'app/server/lib/DocApi';
 import {DocManager} from 'app/server/lib/DocManager';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import {DocWorker} from 'app/server/lib/DocWorker';
 import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@@ -47,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
 import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
   RequestWithGrist} from 'app/server/lib/GristServer';
 import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
-import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
 import {IBilling} from 'app/server/lib/IBilling';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
 import {InstallAdmin} from 'app/server/lib/InstallAdmin';
 import log from 'app/server/lib/log';
-import {getLoginSystem} from 'app/server/lib/logins';
 import {IPermitStore} from 'app/server/lib/Permit';
 import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
@@ -185,7 +181,7 @@ export class FlexServer implements GristServer {
   private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
   private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
   private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
-  private _getLoginSystem?: () => Promise<GristLoginSystem>;
+  private _getLoginSystem: () => Promise<GristLoginSystem>;
   // Set once ready() is called
   private _isReady: boolean = false;
   private _updateManager: UpdateManager;
@@ -193,6 +189,7 @@ export class FlexServer implements GristServer {
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
+    this._getLoginSystem = create.getLoginSystem;
     this.settings = options.settings;
     this.app = express();
     this.app.set('port', port);
@@ -250,7 +247,6 @@ export class FlexServer implements GristServer {
       recentItems: [],
     };
     this.electronServerMethods = {
-      async importDoc() { throw new Error('not implemented'); },
       onDocOpen(cb) {
         // currently only a stub.
         cb('');
@@ -272,11 +268,6 @@ export class FlexServer implements GristServer {
     });
   }
 
-  // Allow overridding the login system.
-  public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
-    this._getLoginSystem = loginSystem;
-  }
-
   public getHost(): string {
     return `${this.host}:${this.getOwnPort()}`;
   }
@@ -405,6 +396,11 @@ export class FlexServer implements GristServer {
     return this._auditLogger;
   }
 
+  public getDocManager(): DocManager {
+    if (!this._docManager) { throw new Error('no document manager available'); }
+    return this._docManager;
+  }
+
   public getTelemetry(): ITelemetry {
     if (!this._telemetry) { throw new Error('no telemetry available'); }
     return this._telemetry;
@@ -1341,12 +1337,15 @@ export class FlexServer implements GristServer {
       const workers = this._docWorkerMap;
       const docWorkerId = await this._addSelfAsWorker(workers);
 
-      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
+      const storageManager = await this.create.createHostedDocStorageManager(
-                                                      this._dbManager, this.create);
+        this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage
+      );
       this._storageManager = storageManager;
     } else {
       const samples = getAppPathTo(this.appRoot, 'public_samples');
-      const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this);
+      const storageManager = await this.create.createLocalDocStorageManager(
+        this.docsRoot, samples, this._comm, this.create.Shell?.()
+      );
       this._storageManager = storageManager;
     }
 
@@ -2012,8 +2011,7 @@ export class FlexServer implements GristServer {
 
   public resolveLoginSystem() {
     return isTestLoginAllowed() ?
-      getTestLoginSystem() :
+      getTestLoginSystem() : this._getLoginSystem();
-      (this._getLoginSystem?.() || getLoginSystem());
   }
 
   public addUpdatesCheck() {
@@ -2609,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne
 
 // Methods that Electron app relies on.
 export interface ElectronServerMethods {
-  importDoc(filepath: string): Promise<DocCreationInfo>;
   onDocOpen(cb: (filePath: string) => void): void;
   getUserConfig(): Promise<any>;
   updateUserConfig(obj: any): Promise<void>;

app/server/lib/HostedStorageManager.ts

@@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
 import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
-import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
+import {
+  ChecksummedExternalStorage,
+  DELETED_TOKEN,
+  ExternalStorage,
+  ExternalStorageCreator, ExternalStorageSettings,
+  Unchanged
+} from 'app/server/lib/ExternalStorage';
 import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
-import {ICreate} from 'app/server/lib/ICreate';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {LogMethods} from "app/server/lib/LogMethods";
 import {fromCallback} from 'app/server/lib/serverUtils';
@@ -51,11 +56,6 @@ export interface HostedStorageOptions {
   secondsBeforePush: number;
   secondsBeforeFirstRetry: number;
   pushDocUpdateTimes: boolean;
-  // A function returning the core ExternalStorage implementation,
-  // which may then be wrapped in additional layer(s) of ExternalStorage.
-  // See ICreate.ExternalStorage.
-  // Uses S3 by default in hosted Grist.
-  externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
 }
 
 const defaultOptions: HostedStorageOptions = {
@@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager {
     private _disableS3: boolean,
     private _docWorkerMap: IDocWorkerMap,
     dbManager: HomeDBManager,
-    create: ICreate,
+    createExternalStorage: ExternalStorageCreator,
     options: HostedStorageOptions = defaultOptions
   ) {
-    const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
+    const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, ''));
     // We store documents either in a test store, or in an s3 store
     // at s3://<s3Bucket>/<s3Prefix><docId>.grist
     const externalStoreDoc = this._disableS3 ? undefined : creator('doc');

app/server/lib/ICreate.ts

@@ -1,10 +1,11 @@
 import {GristDeploymentType} from 'app/common/gristUrls';
+import {getCoreLoginSystem} from 'app/server/lib/coreLogins';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {IAuditLogger} from 'app/server/lib/AuditLogger';
-import {ExternalStorage} from 'app/server/lib/ExternalStorage';
+import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage';
-import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
+import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
 import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
 import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@@ -13,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell';
 import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
 import {ITelemetry} from 'app/server/lib/Telemetry';
+import {IDocStorageManager} from './IDocStorageManager';
+import { Comm } from "./Comm";
+import { IDocWorkerMap } from "./DocWorkerMap";
+import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager";
+import { DocStorageManager } from "./DocStorageManager";
 
 // In the past, the session secret was used as an additional
 // protection passed on to expressjs-session for security when
@@ -37,7 +43,30 @@ import {ITelemetry} from 'app/server/lib/Telemetry';
 export const DEFAULT_SESSION_SECRET =
   'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
 
+export type LocalDocStorageManagerCreator =
+  (docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise<IDocStorageManager>;
+export type HostedDocStorageManagerCreator = (
+    docsRoot: string,
+    docWorkerId: string,
+    disableS3: boolean,
+    docWorkerMap: IDocWorkerMap,
+    dbManager: HomeDBManager,
+    createExternalStorage: ExternalStorageCreator,
+    options?: HostedStorageOptions
+  ) => Promise<IDocStorageManager>;
+
 export interface ICreate {
+  // Create a space to store files externally, for storing either:
+  //  - documents. This store should be versioned, and can be eventually consistent.
+  //  - meta. This store need not be versioned, and can be eventually consistent.
+  // For test purposes an extra prefix may be supplied.  Stores with different prefixes
+  // should not interfere with each other.
+  ExternalStorage: ExternalStorageCreator;
+
+  // Creates a IDocStorageManager for storing documents on the local machine.
+  createLocalDocStorageManager: LocalDocStorageManagerCreator;
+  // Creates a IDocStorageManager for storing documents on an external storage (e.g S3)
+  createHostedDocStorageManager: HostedDocStorageManagerCreator;
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
@@ -45,13 +74,6 @@ export interface ICreate {
   Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
   Shell?(): IShell;  // relevant to electron version of Grist only.
 
-  // Create a space to store files externally, for storing either:
-  //  - documents. This store should be versioned, and can be eventually consistent.
-  //  - meta. This store need not be versioned, and can be eventually consistent.
-  // For test purposes an extra prefix may be supplied.  Stores with different prefixes
-  // should not interfere with each other.
-  ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
-
   NSandbox(options: ISandboxCreationOptions): ISandbox;
 
   // Create the logic to determine which users are authorized to manage this Grist installation.
@@ -69,6 +91,8 @@ export interface ICreate {
   getStorageOptions?(name: string): ICreateStorageOptions|undefined;
   getSqliteVariant?(): SqliteVariant;
   getSandboxVariants?(): Record<string, SpawnFn>;
+
+  getLoginSystem(): Promise<GristLoginSystem>;
 }
 
 export interface ICreateActiveDocOptions {
@@ -126,6 +150,9 @@ export function makeSimpleCreator(opts: {
   getSqliteVariant?: () => SqliteVariant,
   getSandboxVariants?: () => Record<string, SpawnFn>,
   createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
+  getLoginSystem?: () => Promise<GristLoginSystem>,
+  createHostedDocStorageManager?: HostedDocStorageManagerCreator,
+  createLocalDocStorageManager?: LocalDocStorageManagerCreator,
 }): ICreate {
   const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
   return {
@@ -199,5 +226,23 @@ export function makeSimpleCreator(opts: {
     getSqliteVariant: opts.getSqliteVariant,
     getSandboxVariants: opts.getSandboxVariants,
     createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
+    getLoginSystem: opts.getLoginSystem || getCoreLoginSystem,
+    createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager,
+    createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager,
   };
 }
+
+const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async (
+      docsRoot,
+      docWorkerId,
+      disableS3,
+      docWorkerMap,
+      dbManager,
+      createExternalStorage, options
+) =>
+  new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options);
+
+const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async (
+  docsRoot, samplesRoot, comm, shell
+) => new DocStorageManager(docsRoot, samplesRoot, comm, shell);
+

app/server/mergedServerMain.ts

@@ -1,230 +0,0 @@
-/**
- *
- * A version of hosted grist that recombines a home server,
- * a doc worker, and a static server on a single port.
- *
- */
-
-import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
-import {GristLoginSystem} from 'app/server/lib/GristServer';
-import log from 'app/server/lib/log';
-import {getGlobalConfig} from "app/server/lib/globalConfig";
-
-// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
-// environment variable.
-export type ServerType = "home" | "docs" | "static" | "app";
-const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
-
-// Parse a comma-separate list of server types into an array, with validation.
-export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
-  // Split and filter out empty strings (including the one we get when splitting "").
-  const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
-
-  // Check that parts is non-empty and only contains valid options.
-  if (!types.length) {
-    throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
-  }
-  for (const t of types) {
-    if (!allServerTypes.includes(t as ServerType)) {
-      throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
-    }
-  }
-  return types as ServerType[];
-}
-
-function checkUserContentPort(): number | null {
-  // Check whether a port is explicitly set for user content.
-  if (process.env.GRIST_UNTRUSTED_PORT) {
-    return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
-  }
-  // Checks whether to serve user content on same domain but on different port
-  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
-    const homeUrl = new URL(process.env.APP_HOME_URL);
-    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
-    // If the hostname of both home and plugin url are the same,
-    // but the ports are different
-    if (homeUrl.hostname === pluginUrl.hostname &&
-        homeUrl.port !== pluginUrl.port) {
-      const port = parseInt(pluginUrl.port || '80', 10);
-      return port;
-    }
-  }
-  return null;
-}
-
-interface ServerOptions extends FlexServerOptions {
-  logToConsole?: boolean;  // If set, messages logged to console (default: false)
-                           //   (but if options are not given at all in call to main,
-                           //    logToConsole is set to true)
-  externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
-                           // variables, which get set in various ways in dev/test entry points)
-  loginSystem?: () => Promise<GristLoginSystem>;
-}
-
-/**
- * Start a server on the given port, including the functionality specified in serverTypes.
- */
-export async function main(port: number, serverTypes: ServerType[],
-                           options: ServerOptions = {}) {
-  const includeHome = serverTypes.includes("home");
-  const includeDocs = serverTypes.includes("docs");
-  const includeStatic = serverTypes.includes("static");
-  const includeApp = serverTypes.includes("app");
-
-  options.settings ??= getGlobalConfig();
-
-  const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
-
-  // We need to know early on whether we will be serving plugins or not.
-  if (includeHome) {
-    const userPort = checkUserContentPort();
-    server.setServesPlugins(userPort !== undefined);
-  } else {
-    server.setServesPlugins(false);
-  }
-
-  if (options.loginSystem) {
-    server.setLoginSystem(options.loginSystem);
-  }
-
-  server.addCleanup();
-  server.setDirectory();
-
-  if (process.env.GRIST_TEST_ROUTER) {
-    // Add a mock api for adding/removing doc workers from load balancer.
-    server.testAddRouter();
-  }
-
-  if (options.logToConsole !== false) { server.addLogging(); }
-  if (options.externalStorage === false) { server.disableExternalStorage(); }
-  await server.addLoginMiddleware();
-
-  if (includeDocs) {
-    // It is important that /dw and /v prefixes are accepted (if present) by health check
-    // in this case, since they are included in the url registered for the doc worker.
-    server.stripDocWorkerIdPathPrefixIfPresent();
-    server.addTagChecker();
-  }
-
-  server.addHealthCheck();
-  if (includeHome || includeApp) {
-    server.addBootPage();
-  }
-  server.denyRequestsIfNotReady();
-
-  if (includeHome || includeStatic || includeApp) {
-    server.setDirectory();
-  }
-
-  if (includeHome || includeStatic) {
-    server.addStaticAndBowerDirectories();
-  }
-
-  await server.initHomeDBManager();
-  server.addHosts();
-
-  server.addDocWorkerMap();
-
-  if (includeHome || includeStatic) {
-    await server.addAssetsForPlugins();
-  }
-
-  if (includeHome) {
-    server.addEarlyWebhooks();
-  }
-
-  if (includeHome || includeDocs || includeApp) {
-    server.addSessions();
-  }
-
-  server.addAccessMiddleware();
-  server.addApiMiddleware();
-  await server.addBillingMiddleware();
-
-  try {
-    await server.start();
-
-    if (includeHome) {
-      server.addUsage();
-      if (!includeDocs) {
-        server.addDocApiForwarder();
-      }
-      server.addJsonSupport();
-      server.addUpdatesCheck();
-      await server.addLandingPages();
-      // todo: add support for home api to standalone app
-      server.addHomeApi();
-      server.addBillingApi();
-      server.addNotifier();
-      server.addAuditLogger();
-      await server.addTelemetry();
-      await server.addHousekeeper();
-      await server.addLoginRoutes();
-      server.addAccountPage();
-      server.addBillingPages();
-      server.addWelcomePaths();
-      server.addLogEndpoint();
-      server.addGoogleAuthEndpoint();
-      server.addInstallEndpoints();
-      server.addConfigEndpoints();
-    }
-
-    if (includeDocs) {
-      server.addJsonSupport();
-      server.addAuditLogger();
-      await server.addTelemetry();
-      await server.addDoc();
-    }
-
-    if (includeHome) {
-      server.addClientSecrets();
-    }
-
-    server.finalizeEndpoints();
-    await server.finalizePlugins(includeHome ? checkUserContentPort() : null);
-    server.checkOptionCombinations();
-    server.summary();
-    server.ready();
-
-    // Some tests have their timing perturbed by having this earlier
-    // TODO: update those tests.
-    if (includeDocs) {
-      await server.checkSandbox();
-    }
-    return server;
-  } catch(e) {
-    await server.close();
-    throw e;
-  }
-}
-
-
-export async function startMain() {
-  try {
-
-    const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
-
-    // No defaults for a port, since this server can serve very different purposes.
-    if (!process.env.GRIST_PORT) {
-      throw new Error("GRIST_PORT must be specified");
-    }
-
-    const port = parseInt(process.env.GRIST_PORT, 10);
-
-    const server = await main(port, serverTypes);
-
-    const opt = process.argv[2];
-    if (opt === '--testingHooks') {
-      await server.addTestingHooks();
-    }
-
-    return server;
-  } catch (e) {
-    log.error('mergedServer failed to start', e);
-    process.exit(1);
-  }
-}
-
-if (require.main === module) {
-  startMain().catch((e) => log.error('mergedServer failed to start', e));
-}

app/server/utils/pruneActionHistory.ts

@@ -1,9 +1,9 @@
 import * as gutil from 'app/common/gutil';
 import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
 import {DocStorage} from 'app/server/lib/DocStorage';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import log from 'app/server/lib/log';
+import {create} from "app/server/lib/create";
 
 /**
  * A utility script for cleaning up the action log.
@@ -18,7 +18,7 @@ export async function pruneActionHistory(docPath: string, keepN: number) {
     throw new Error('Invalid document: Document should be a valid .grist file');
   }
 
-  const storageManager = new DocStorageManager(".", ".");
+  const storageManager = await create.createLocalDocStorageManager(".", ".");
   const docStorage = new DocStorage(storageManager, docPath);
   const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist";
 

sandbox/watch.sh

@@ -2,6 +2,13 @@
 
 set -x
 
+NO_NODEMON=false
+for arg in $@; do
+  if [[ $arg == "--no-nodemon" ]]; then
+    NO_NODEMON=true
+  fi
+done
+
 PROJECT=""
 if [[ -e ext/app ]]; then
   PROJECT="tsconfig-ext.json"
@@ -19,6 +26,6 @@ tsc --build -w --preserveWatchOutput $PROJECT &
 css_files="app/client/**/*.css"
 chokidar "${css_files}" -c "bash -O globstar -c 'cat ${css_files} > static/bundle.css'" &
 webpack --config $WEBPACK_CONFIG --mode development --watch &
-NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js &
+! $NO_NODEMON && NODE_PATH=_build:_build/stubs:_build/ext nodemon ${NODE_INSPECT} --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js &
 
 wait

stubs/app/client/ui/HomeImports.ts

@@ -0,0 +1,2 @@
+import * as coreHomeImports from "app/client/ui/CoreHomeImports";
+export const homeImports = coreHomeImports;

stubs/app/client/ui/NewDocMethods.ts

@@ -0,0 +1,2 @@
+import * as coreNewDocMethods from "app/client/ui/CoreNewDocMethods";
+export const newDocMethods = coreNewDocMethods;

stubs/app/server/lib/logins.ts

@@ -1,6 +0,0 @@
-import { getCoreLoginSystem } from "app/server/lib/coreLogins";
-import { GristLoginSystem } from "app/server/lib/GristServer";
-
-export async function getLoginSystem(): Promise<GristLoginSystem> {
-  return getCoreLoginSystem();
-}

stubs/app/server/server.ts

@@ -34,7 +34,7 @@ setDefaultEnv('GRIST_UI_FEATURES',
   'helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,createSite,supportGrist');
 setDefaultEnv('GRIST_WIDGET_LIST_URL', commonUrls.gristLabsWidgetRepository);
 import {updateDb} from 'app/server/lib/dbUtils';
-import {main as mergedServerMain, parseServerTypes} from 'app/server/mergedServerMain';
+import {MergedServer, parseServerTypes} from 'app/server/MergedServer';
 import * as fse from 'fs-extra';
 import {runPrometheusExporter} from './prometheus-exporter';
 
@@ -124,20 +124,20 @@ export async function main() {
   }
 
   // Launch single-port, self-contained version of Grist.
-  const server = await mergedServerMain(G.port, serverTypes);
+  const mergedServer = await MergedServer.create(G.port, serverTypes);
+  await mergedServer.run();
   if (process.env.GRIST_TESTING_SOCKET) {
-    await server.addTestingHooks();
+    await mergedServer.flexServer.addTestingHooks();
   }
   if (process.env.GRIST_SERVE_PLUGINS_PORT) {
-    await server.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10));
+    await mergedServer.flexServer.startCopy('pluginServer', parseInt(process.env.GRIST_SERVE_PLUGINS_PORT, 10));
   }
-
   await fixSiteProducts({
-    deploymentType: server.getDeploymentType(),
+    deploymentType: mergedServer.flexServer.getDeploymentType(),
-    db: server.getHomeDBManager()
+    db: mergedServer.flexServer.getHomeDBManager()
   });
 
-  return server;
+  return mergedServer.flexServer;
 }
 
 if (require.main === module) {

test/gen-server/AuthCaching.ts

@@ -2,7 +2,7 @@ import {delay} from 'app/common/delay';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import log from 'app/server/lib/log';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from 'app/server/MergedServer';
 import axios from 'axios';
 import {assert} from 'chai';
 import * as fse from 'fs-extra';
@@ -50,12 +50,17 @@ describe('AuthCaching', function() {
     setUpDB();
     await createInitialDb();
     process.env.GRIST_DATA_DIR = testDocDir;
-    homeServer = await mergedServerMain(0, ['home'],
+
+    const homeMS = await MergedServer.create(0, ['home'],
       {logToConsole: false, externalStorage: false});
+    await homeMS.run();
+    homeServer = homeMS.flexServer;
     homeUrl = homeServer.getOwnUrl();
     process.env.APP_HOME_URL = homeUrl;
-    docsServer = await mergedServerMain(0, ['docs'],
+    const docsMS = await MergedServer.create(0, ['docs'],
       {logToConsole: false, externalStorage: false});
+    await docsMS.run();
+    docsServer = docsMS.flexServer;
 
     // Helpers for getting cookie-based logins.
     session = new TestSession(homeServer);

test/gen-server/apiUtils.ts

@@ -14,7 +14,7 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
-import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
+import {MergedServer, ServerType} from 'app/server/MergedServer';
 import axios from 'axios';
 import FormData from 'form-data';
 import fetch from 'node-fetch';
@@ -37,9 +37,10 @@ export class TestServer {
   public async start(servers: ServerType[] = ["home"],
                      options: FlexServerOptions = {}): Promise<string> {
     await createInitialDb();
-    this.server = await mergedServerMain(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
+    const mergedServer = await MergedServer.create(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
-                                                      externalStorage: false,
+                                                      externalStorage: false, ...options});
-                                                      ...options});
+    await mergedServer.run();
+    this.server = mergedServer.flexServer;
     this.serverUrl = this.server.getOwnUrl();
     this.dbManager = this.server.getHomeDBManager();
     this.defaultSession = new TestSession(this.server);
@@ -263,7 +264,7 @@ export class TestSession {
     if (clearCache) { this.home.getSessions().clearCacheIfNeeded(); }
     this.headers.Cookie = cookie;
     return {
-      validateStatus: (status: number) => true,
+      validateStatus: (_status: number) => true,
       headers: {
         'Cookie': cookie,
         'X-Requested-With': 'XMLHttpRequest',

test/gen-server/lib/DocWorkerMap.ts

@@ -2,7 +2,7 @@ import {DocWorkerMap, getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import {Permit} from 'app/server/lib/Permit';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from "app/server/MergedServer";
 import {delay, promisifyAll} from 'bluebird';
 import {assert, expect} from 'chai';
 import {countBy, values} from 'lodash';
@@ -387,24 +387,34 @@ describe('DocWorkerMap', function() {
       process.env.REDIS_URL = process.env.TEST_REDIS_URL;
 
       // Make home server.
-      const home = await mergedServerMain(0, ['home'], opts);
+      const homeMergedServer = await MergedServer.create(0, ['home'], opts);
+      const home = homeMergedServer.flexServer;
+      await homeMergedServer.run();
 
       // Make a worker, not associated with any group.
       process.env.GRIST_DOC_WORKER_ID = 'worker1';
-      const docs1 = await mergedServerMain(0, ['docs'], opts);
+      const docs1MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs1 = docs1MergedServer.flexServer;
+      await docs1MergedServer.run();
 
       // Make a worker in "special" group.
       process.env.GRIST_DOC_WORKER_ID = 'worker2';
       process.env.GRIST_WORKER_GROUP = 'special';
-      const docs2 = await mergedServerMain(0, ['docs'], opts);
+      const docs2MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs2 = docs2MergedServer.flexServer;
+      await docs2MergedServer.run();
 
       // Make two worker in "other" group.
       process.env.GRIST_DOC_WORKER_ID = 'worker3';
       process.env.GRIST_WORKER_GROUP = 'other';
-      const docs3 = await mergedServerMain(0, ['docs'], opts);
+      const docs3MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs3 = docs3MergedServer.flexServer;
+      await docs3MergedServer.run();
       process.env.GRIST_DOC_WORKER_ID = 'worker4';
       process.env.GRIST_WORKER_GROUP = 'other';
-      const docs4 = await mergedServerMain(0, ['docs'], opts);
+      const docs4MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs4 = docs4MergedServer.flexServer;
+      await docs4MergedServer.run();
 
       servers = {home, docs1, docs2, docs3, docs4};
       workers = getDocWorkerMap();

test/gen-server/lib/mergedOrgs.ts

@@ -1,7 +1,7 @@
 import {Workspace} from 'app/common/UserAPI';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from "app/server/MergedServer";
 import axios from 'axios';
 import {assert} from 'chai';
 import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
@@ -9,6 +9,7 @@ import {configForUser, createUser, setPlan} from 'test/gen-server/testUtils';
 import * as testUtils from 'test/server/testUtils';
 
 describe('mergedOrgs', function() {
+  let mergedServer: MergedServer;
   let home: FlexServer;
   let dbManager: HomeDBManager;
   let homeUrl: string;
@@ -20,8 +21,10 @@ describe('mergedOrgs', function() {
   before(async function() {
     setUpDB(this);
     await createInitialDb();
-    home = await mergedServerMain(0, ["home", "docs"],
+    mergedServer = await MergedServer.create(0, ["home", "docs"],
                                   {logToConsole: false, externalStorage: false});
+    home = mergedServer.flexServer;
+    await mergedServer.run();
     dbManager = home.getHomeDBManager();
     homeUrl = home.getOwnUrl();
   });

test/server/docTools.ts

@@ -4,7 +4,6 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DummyAuthorizer} from 'app/server/lib/Authorizer';
 import {DocManager} from 'app/server/lib/DocManager';
 import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {getAppRoot} from 'app/server/lib/places';
@@ -17,6 +16,7 @@ import * as fse from 'fs-extra';
 import {tmpdir} from 'os';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import {create} from "app/server/lib/create";
 
 tmp.setGracefulCleanup();
 
@@ -138,7 +138,7 @@ export async function createDocManager(
               server?: GristServer} = {}): Promise<DocManager> {
   // Set Grist home to a temporary directory, and wipe it out on exit.
   const tmpDir = options.tmpDir || await createTmpDir();
-  const docStorageManager = options.storageManager || new DocStorageManager(tmpDir);
+  const docStorageManager = options.storageManager || await create.createLocalDocStorageManager(tmpDir);
   const pluginManager = options.pluginManager || await getGlobalPluginManager();
   const store = getDocWorkerMap();
   const internalPermitStore = store.getPermitStore('1');

test/server/lib/HostedStorageManager.ts

@@ -7,7 +7,12 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {create} from 'app/server/lib/create';
 import {DocManager} from 'app/server/lib/DocManager';
 import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
-import {DELETED_TOKEN, ExternalStorage, wrapWithKeyMappedStorage} from 'app/server/lib/ExternalStorage';
+import {
+  DELETED_TOKEN,
+  ExternalStorage, ExternalStorageCreator,
+  ExternalStorageSettings,
+  wrapWithKeyMappedStorage
+} from 'app/server/lib/ExternalStorage';
 import {createDummyGristServer} from 'app/server/lib/GristServer';
 import {
   BackupEvent,
@@ -270,7 +275,7 @@ class TestStore {
     private _localDirectory: string,
     private _workerId: string,
     private _workers: DocWorkerMap,
-    private _externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined) {
+    private _externalStorageCreate: ExternalStorageCreator) {
   }
 
   public async run<T>(fn: () => Promise<T>): Promise<T> {
@@ -296,18 +301,20 @@ class TestStore {
       secondsBeforeFirstRetry: 3,   // rumors online suggest delays of 10-11 secs
                                     // are not super-unusual.
       pushDocUpdateTimes: false,
-      externalStorageCreator: (purpose) => {
+
+    };
+    const externalStorageCreator = (purpose: ExternalStorageSettings["purpose"]) => {
         const result = this._externalStorageCreate(purpose, this._extraPrefix);
         if (!result) { throw new Error('no storage'); }
         return result;
-      }
     };
+
     const storageManager = new HostedStorageManager(this._localDirectory,
                                                     this._workerId,
                                                     false,
                                                     this._workers,
                                                     dbManager,
-                                                    create,
+                                                    externalStorageCreator,
                                                     options);
     this.storageManager = storageManager;
     this.docManager = new DocManager(storageManager, await getGlobalPluginManager(),

test/server/lib/helpers/TestServer.ts

@@ -91,7 +91,7 @@ export class TestServer {
       ...this._defaultEnv,
       ...customEnv
     };
-    const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
+    const main = await testUtils.getBuildFile('app/server/MergedServer.js');
     this._server = spawn('node', [main, '--testingHooks'], {
       env,
       stdio: ['inherit', serverLog, serverLog]