Make changes required for Desktop FS updates (#1099)

Make a set of changes required for Desktop FS improvements, see
https://github.com/gristlabs/grist-desktop/pull/42

---------

Co-authored-by: Spoffy <contact@spoffy.net>
Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com>

app/server/MergedServer.ts

@@ -0,0 +1,237 @@
+/**
+ *
+ * A version of hosted grist that recombines a home server,
+ * a doc worker, and a static server on a single port.
+ *
+ */
+
+import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
+import log from 'app/server/lib/log';
+import {getGlobalConfig} from "app/server/lib/globalConfig";
+
+// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
+// environment variable.
+export type ServerType = "home" | "docs" | "static" | "app";
+const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
+
+// Parse a comma-separate list of server types into an array, with validation.
+export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
+  // Split and filter out empty strings (including the one we get when splitting "").
+  const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
+
+  // Check that parts is non-empty and only contains valid options.
+  if (!types.length) {
+    throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
+  }
+  for (const t of types) {
+    if (!allServerTypes.includes(t as ServerType)) {
+      throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
+    }
+  }
+  return types as ServerType[];
+}
+
+function checkUserContentPort(): number | null {
+  // Check whether a port is explicitly set for user content.
+  if (process.env.GRIST_UNTRUSTED_PORT) {
+    return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
+  }
+  // Checks whether to serve user content on same domain but on different port
+  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
+    const homeUrl = new URL(process.env.APP_HOME_URL);
+    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
+    // If the hostname of both home and plugin url are the same,
+    // but the ports are different
+    if (homeUrl.hostname === pluginUrl.hostname &&
+        homeUrl.port !== pluginUrl.port) {
+      const port = parseInt(pluginUrl.port || '80', 10);
+      return port;
+    }
+  }
+  return null;
+}
+
+interface ServerOptions extends FlexServerOptions {
+  // If set, messages logged to console (default: false)
+  // (but if options are not given at all in call to main, logToConsole is set to true)
+  logToConsole?: boolean;
+
+  // If set, documents saved to external storage such as s3 (default is to check environment variables,
+  // which get set in various ways in dev/test entry points)
+  externalStorage?: boolean;
+}
+
+export class MergedServer {
+
+  public static async create(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
+    options.settings ??= getGlobalConfig();
+    const ms = new MergedServer(port, serverTypes, options);
+    // We need to know early on whether we will be serving plugins or not.
+    if (ms.hasComponent("home")) {
+      const userPort = checkUserContentPort();
+      ms.flexServer.setServesPlugins(userPort !== undefined);
+    } else {
+      ms.flexServer.setServesPlugins(false);
+    }
+
+    ms.flexServer.addCleanup();
+    ms.flexServer.setDirectory();
+
+    if (process.env.GRIST_TEST_ROUTER) {
+      // Add a mock api for adding/removing doc workers from load balancer.
+      ms.flexServer.testAddRouter();
+    }
+
+    if (ms._options.logToConsole !== false) { ms.flexServer.addLogging(); }
+    if (ms._options.externalStorage === false) { ms.flexServer.disableExternalStorage(); }
+    await ms.flexServer.addLoginMiddleware();
+
+    if (ms.hasComponent("docs")) {
+      // It is important that /dw and /v prefixes are accepted (if present) by health check
+      // in ms case, since they are included in the url registered for the doc worker.
+      ms.flexServer.stripDocWorkerIdPathPrefixIfPresent();
+      ms.flexServer.addTagChecker();
+    }
+
+    ms.flexServer.addHealthCheck();
+    if (ms.hasComponent("home") || ms.hasComponent("app")) {
+      ms.flexServer.addBootPage();
+    }
+    ms.flexServer.denyRequestsIfNotReady();
+
+    if (ms.hasComponent("home") || ms.hasComponent("static") || ms.hasComponent("app")) {
+      ms.flexServer.setDirectory();
+    }
+
+    if (ms.hasComponent("home") || ms.hasComponent("static")) {
+      ms.flexServer.addStaticAndBowerDirectories();
+    }
+
+    await ms.flexServer.initHomeDBManager();
+    ms.flexServer.addHosts();
+
+    ms.flexServer.addDocWorkerMap();
+
+    if (ms.hasComponent("home") || ms.hasComponent("static")) {
+      await ms.flexServer.addAssetsForPlugins();
+    }
+
+    if (ms.hasComponent("home")) {
+      ms.flexServer.addEarlyWebhooks();
+    }
+
+    if (ms.hasComponent("home") || ms.hasComponent("docs") || ms.hasComponent("app")) {
+      ms.flexServer.addSessions();
+    }
+
+    ms.flexServer.addAccessMiddleware();
+    ms.flexServer.addApiMiddleware();
+    await ms.flexServer.addBillingMiddleware();
+
+    return ms;
+  }
+
+  public readonly flexServer: FlexServer;
+  private readonly _serverTypes: ServerType[];
+  private readonly _options: ServerOptions;
+
+  private constructor(port: number, serverTypes: ServerType[], options: ServerOptions = {}) {
+    this._serverTypes = serverTypes;
+    this._options = options;
+    this.flexServer = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
+  }
+
+  public hasComponent(serverType: ServerType) {
+    return this._serverTypes.includes(serverType);
+  }
+
+
+  public async run() {
+
+    try {
+      await this.flexServer.start();
+
+      if (this.hasComponent("home")) {
+        this.flexServer.addUsage();
+        if (!this.hasComponent("docs")) {
+          this.flexServer.addDocApiForwarder();
+        }
+        this.flexServer.addJsonSupport();
+        this.flexServer.addUpdatesCheck();
+        await this.flexServer.addLandingPages();
+        // todo: add support for home api to standalone app
+        this.flexServer.addHomeApi();
+        this.flexServer.addBillingApi();
+        this.flexServer.addNotifier();
+        this.flexServer.addAuditLogger();
+        await this.flexServer.addTelemetry();
+        await this.flexServer.addHousekeeper();
+        await this.flexServer.addLoginRoutes();
+        this.flexServer.addAccountPage();
+        this.flexServer.addBillingPages();
+        this.flexServer.addWelcomePaths();
+        this.flexServer.addLogEndpoint();
+        this.flexServer.addGoogleAuthEndpoint();
+        this.flexServer.addInstallEndpoints();
+        this.flexServer.addConfigEndpoints();
+      }
+
+      if (this.hasComponent("docs")) {
+        this.flexServer.addJsonSupport();
+        this.flexServer.addAuditLogger();
+        await this.flexServer.addTelemetry();
+        await this.flexServer.addDoc();
+      }
+
+      if (this.hasComponent("home")) {
+        this.flexServer.addClientSecrets();
+      }
+
+      this.flexServer.finalizeEndpoints();
+      await this.flexServer.finalizePlugins(this.hasComponent("home") ? checkUserContentPort() : null);
+      this.flexServer.checkOptionCombinations();
+      this.flexServer.summary();
+      this.flexServer.ready();
+
+      // Some tests have their timing perturbed by having this earlier
+      // TODO: update those tests.
+      if (this.hasComponent("docs")) {
+        await this.flexServer.checkSandbox();
+      }
+    } catch(e) {
+      await this.flexServer.close();
+      throw e;
+    }
+  }
+}
+
+export async function startMain() {
+  try {
+
+    const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
+
+    // No defaults for a port, since this server can serve very different purposes.
+    if (!process.env.GRIST_PORT) {
+      throw new Error("GRIST_PORT must be specified");
+    }
+
+    const port = parseInt(process.env.GRIST_PORT, 10);
+
+    const server = await MergedServer.create(port, serverTypes);
+    await server.run();
+
+    const opt = process.argv[2];
+    if (opt === '--testingHooks') {
+      await server.flexServer.addTestingHooks();
+    }
+
+    return server.flexServer;
+  } catch (e) {
+    log.error('mergedServer failed to start', e);
+    process.exit(1);
+  }
+}
+
+if (require.main === module) {
+  startMain().catch((e) => log.error('mergedServer failed to start', e));
+}

app/server/devServerMain.ts

@@ -21,7 +21,7 @@
 import {updateDb} from 'app/server/lib/dbUtils';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import log from 'app/server/lib/log';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from 'app/server/MergedServer';
 import {promisifyAll} from 'bluebird';
 import * as fse from 'fs-extra';
 import * as path from 'path';
@@ -96,8 +96,9 @@ export async function main() {
     if (!process.env.APP_HOME_URL) {
       process.env.APP_HOME_URL = `http://localhost:${port}`;
     }
-    const server = await mergedServerMain(port, ["home", "docs", "static"]);
-    await server.addTestingHooks();
+    const mergedServer = await MergedServer.create(port, ["home", "docs", "static"]);
+    await mergedServer.flexServer.addTestingHooks();
+    await mergedServer.run();
     return;
   }
 
@@ -118,17 +119,18 @@ export async function main() {
   log.info("== staticServer");
   const staticPort = getPort("STATIC_PORT", 9001);
   process.env.APP_STATIC_URL = `http://localhost:${staticPort}`;
-  await mergedServerMain(staticPort, ["static"]);
+  await MergedServer.create(staticPort, ["static"]).then((s) => s.run());
 
   // Bring up a home server
   log.info("==========================================================================");
   log.info("== homeServer");
-  const home = await mergedServerMain(homeServerPort, ["home"]);
+  const homeServer = await MergedServer.create(homeServerPort, ["home"]);
+  await homeServer.run();
 
   // If a distinct webServerPort is specified, we listen also on that port, though serving
   // exactly the same content.  This is handy for testing CORS issues.
   if (webServerPort !== 0 && webServerPort !== homeServerPort) {
-    await home.startCopy('webServer', webServerPort);
+    await homeServer.flexServer.startCopy('webServer', webServerPort);
   }
 
   // Bring up the docWorker(s)
@@ -147,10 +149,10 @@ export async function main() {
   }
   const workers = new Array<FlexServer>();
   for (const port of ports) {
-    workers.push(await mergedServerMain(port, ["docs"]));
+    workers.push((await MergedServer.create(port, ["docs"])).flexServer);
   }
 
-  await home.addTestingHooks(workers);
+  await homeServer.flexServer.addTestingHooks(workers);
 }
 
 

app/server/lib/ActiveDocImport.ts

@@ -382,7 +382,7 @@ export class ActiveDocImport {
    * @param {String} tmpPath: The path from of the original file.
    * @param {FileImportOptions} importOptions: File import options.
    * @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
-   * or guessed by the plugin, and `tables`, which is which is a list of objects with information about
+   * or guessed by the plugin, and `tables`, which is a list of objects with information about
    * tables, such as `hiddenTableId`, `uploadFileIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
    */
   private async _importFileAsNewTable(docSession: OptDocSession, tmpPath: string,

app/server/lib/DocStorageManager.ts

@@ -10,7 +10,6 @@ import {DocumentUsage} from 'app/common/DocUsage';
 import * as gutil from 'app/common/gutil';
 import {Comm} from 'app/server/lib/Comm';
 import * as docUtils from 'app/server/lib/docUtils';
-import {GristServer} from 'app/server/lib/GristServer';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {IShell} from 'app/server/lib/IShell';
 import log from 'app/server/lib/log';
@@ -39,10 +38,10 @@ export class DocStorageManager implements IDocStorageManager {
    * The file watcher is created if the optComm argument is given.
    */
   constructor(private _docsRoot: string, private _samplesRoot?: string,
-              private _comm?: Comm, gristServer?: GristServer) {
+              private _comm?: Comm, shell?: IShell) {
     // If we have a way to communicate with clients, watch the docsRoot for changes.
     this._watcher = null;
-    this._shell = gristServer?.create.Shell?.() || {
+    this._shell = shell ?? {
       trashItem() { throw new Error('Unable to move document to trash'); },
       showItemInFolder() { throw new Error('Unable to show item in folder'); }
     };

app/server/lib/ExternalStorage.ts

@@ -377,6 +377,15 @@ export interface ExternalStorageSettings {
   extraPrefix?: string;
 }
 
+/**
+ * Function returning the core ExternalStorage implementation,
+ * which may then be wrapped in additional layer(s) of ExternalStorage.
+ * See ICreate.ExternalStorage.
+ * Uses S3 by default in hosted Grist.
+*/
+export type ExternalStorageCreator =
+  (purpose: ExternalStorageSettings["purpose"], extraPrefix: string) => ExternalStorage | undefined;
+
 /**
  * The storage mapping we use for our SaaS. A reasonable default, but relies
  * on appropriate lifecycle rules being set up in the bucket.

app/server/lib/FlexServer.ts

@@ -1,7 +1,6 @@
 import {ApiError} from 'app/common/ApiError';
 import {ICustomWidget} from 'app/common/CustomWidget';
 import {delay} from 'app/common/delay';
-import {DocCreationInfo} from 'app/common/DocListAPI';
 import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
         GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
         sanitizePathTail} from 'app/common/gristUrls';
@@ -38,7 +37,6 @@ import {create} from 'app/server/lib/create';
 import {addDiscourseConnectEndpoints} from 'app/server/lib/DiscourseConnect';
 import {addDocApiRoutes} from 'app/server/lib/DocApi';
 import {DocManager} from 'app/server/lib/DocManager';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import {DocWorker} from 'app/server/lib/DocWorker';
 import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
@@ -47,13 +45,11 @@ import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
 import {DocTemplate, GristLoginMiddleware, GristLoginSystem, GristServer,
   RequestWithGrist} from 'app/server/lib/GristServer';
 import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
-import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
 import {IBilling} from 'app/server/lib/IBilling';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
 import {InstallAdmin} from 'app/server/lib/InstallAdmin';
 import log from 'app/server/lib/log';
-import {getLoginSystem} from 'app/server/lib/logins';
 import {IPermitStore} from 'app/server/lib/Permit';
 import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
@@ -185,7 +181,7 @@ export class FlexServer implements GristServer {
   private _getSignUpRedirectUrl: (req: express.Request, target: URL) => Promise<string>;
   private _getLogoutRedirectUrl: (req: express.Request, nextUrl: URL) => Promise<string>;
   private _sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
-  private _getLoginSystem?: () => Promise<GristLoginSystem>;
+  private _getLoginSystem: () => Promise<GristLoginSystem>;
   // Set once ready() is called
   private _isReady: boolean = false;
   private _updateManager: UpdateManager;
@@ -193,6 +189,7 @@ export class FlexServer implements GristServer {
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
+    this._getLoginSystem = create.getLoginSystem;
     this.settings = options.settings;
     this.app = express();
     this.app.set('port', port);
@@ -250,7 +247,6 @@ export class FlexServer implements GristServer {
       recentItems: [],
     };
     this.electronServerMethods = {
-      async importDoc() { throw new Error('not implemented'); },
       onDocOpen(cb) {
         // currently only a stub.
         cb('');
@@ -272,11 +268,6 @@ export class FlexServer implements GristServer {
     });
   }
 
-  // Allow overridding the login system.
-  public setLoginSystem(loginSystem: () => Promise<GristLoginSystem>) {
-    this._getLoginSystem = loginSystem;
-  }
-
   public getHost(): string {
     return `${this.host}:${this.getOwnPort()}`;
   }
@@ -405,6 +396,11 @@ export class FlexServer implements GristServer {
     return this._auditLogger;
   }
 
+  public getDocManager(): DocManager {
+    if (!this._docManager) { throw new Error('no document manager available'); }
+    return this._docManager;
+  }
+
   public getTelemetry(): ITelemetry {
     if (!this._telemetry) { throw new Error('no telemetry available'); }
     return this._telemetry;
@@ -1341,12 +1337,15 @@ export class FlexServer implements GristServer {
       const workers = this._docWorkerMap;
       const docWorkerId = await this._addSelfAsWorker(workers);
 
-      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
-                                                      this._dbManager, this.create);
+      const storageManager = await this.create.createHostedDocStorageManager(
+        this.docsRoot, docWorkerId, this._disableExternalStorage, workers, this._dbManager, this.create.ExternalStorage
+      );
       this._storageManager = storageManager;
     } else {
       const samples = getAppPathTo(this.appRoot, 'public_samples');
-      const storageManager = new DocStorageManager(this.docsRoot, samples, this._comm, this);
+      const storageManager = await this.create.createLocalDocStorageManager(
+        this.docsRoot, samples, this._comm, this.create.Shell?.()
+      );
       this._storageManager = storageManager;
     }
 
@@ -2012,8 +2011,7 @@ export class FlexServer implements GristServer {
 
   public resolveLoginSystem() {
     return isTestLoginAllowed() ?
-      getTestLoginSystem() :
-      (this._getLoginSystem?.() || getLoginSystem());
+      getTestLoginSystem() : this._getLoginSystem();
   }
 
   public addUpdatesCheck() {
@@ -2609,7 +2607,6 @@ function noCaching(req: express.Request, res: express.Response, next: express.Ne
 
 // Methods that Electron app relies on.
 export interface ElectronServerMethods {
-  importDoc(filepath: string): Promise<DocCreationInfo>;
   onDocOpen(cb: (filePath: string) => void): void;
   getUserConfig(): Promise<any>;
   updateUserConfig(obj: any): Promise<void>;

app/server/lib/HostedStorageManager.ts

@@ -12,9 +12,14 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
 import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
-import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
+import {
+  ChecksummedExternalStorage,
+  DELETED_TOKEN,
+  ExternalStorage,
+  ExternalStorageCreator, ExternalStorageSettings,
+  Unchanged
+} from 'app/server/lib/ExternalStorage';
 import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
-import {ICreate} from 'app/server/lib/ICreate';
 import {EmptySnapshotProgress, IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {LogMethods} from "app/server/lib/LogMethods";
 import {fromCallback} from 'app/server/lib/serverUtils';
@@ -51,11 +56,6 @@ export interface HostedStorageOptions {
   secondsBeforePush: number;
   secondsBeforeFirstRetry: number;
   pushDocUpdateTimes: boolean;
-  // A function returning the core ExternalStorage implementation,
-  // which may then be wrapped in additional layer(s) of ExternalStorage.
-  // See ICreate.ExternalStorage.
-  // Uses S3 by default in hosted Grist.
-  externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
 }
 
 const defaultOptions: HostedStorageOptions = {
@@ -134,10 +134,10 @@ export class HostedStorageManager implements IDocStorageManager {
     private _disableS3: boolean,
     private _docWorkerMap: IDocWorkerMap,
     dbManager: HomeDBManager,
-    create: ICreate,
+    createExternalStorage: ExternalStorageCreator,
     options: HostedStorageOptions = defaultOptions
   ) {
-    const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
+    const creator = ((purpose: ExternalStorageSettings['purpose']) => createExternalStorage(purpose, ''));
     // We store documents either in a test store, or in an s3 store
     // at s3://<s3Bucket>/<s3Prefix><docId>.grist
     const externalStoreDoc = this._disableS3 ? undefined : creator('doc');

app/server/lib/ICreate.ts

@@ -1,10 +1,11 @@
 import {GristDeploymentType} from 'app/common/gristUrls';
+import {getCoreLoginSystem} from 'app/server/lib/coreLogins';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {IAuditLogger} from 'app/server/lib/AuditLogger';
-import {ExternalStorage} from 'app/server/lib/ExternalStorage';
-import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
+import {ExternalStorage, ExternalStorageCreator} from 'app/server/lib/ExternalStorage';
+import {createDummyAuditLogger, createDummyTelemetry, GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
 import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
 import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@@ -13,6 +14,11 @@ import {IShell} from 'app/server/lib/IShell';
 import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
 import {SqliteVariant} from 'app/server/lib/SqliteCommon';
 import {ITelemetry} from 'app/server/lib/Telemetry';
+import {IDocStorageManager} from './IDocStorageManager';
+import { Comm } from "./Comm";
+import { IDocWorkerMap } from "./DocWorkerMap";
+import { HostedStorageManager, HostedStorageOptions } from "./HostedStorageManager";
+import { DocStorageManager } from "./DocStorageManager";
 
 // In the past, the session secret was used as an additional
 // protection passed on to expressjs-session for security when
@@ -37,7 +43,30 @@ import {ITelemetry} from 'app/server/lib/Telemetry';
 export const DEFAULT_SESSION_SECRET =
   'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
 
+export type LocalDocStorageManagerCreator =
+  (docsRoot: string, samplesRoot?: string, comm?: Comm, shell?: IShell) => Promise<IDocStorageManager>;
+export type HostedDocStorageManagerCreator = (
+    docsRoot: string,
+    docWorkerId: string,
+    disableS3: boolean,
+    docWorkerMap: IDocWorkerMap,
+    dbManager: HomeDBManager,
+    createExternalStorage: ExternalStorageCreator,
+    options?: HostedStorageOptions
+  ) => Promise<IDocStorageManager>;
+
 export interface ICreate {
+  // Create a space to store files externally, for storing either:
+  //  - documents. This store should be versioned, and can be eventually consistent.
+  //  - meta. This store need not be versioned, and can be eventually consistent.
+  // For test purposes an extra prefix may be supplied.  Stores with different prefixes
+  // should not interfere with each other.
+  ExternalStorage: ExternalStorageCreator;
+
+  // Creates a IDocStorageManager for storing documents on the local machine.
+  createLocalDocStorageManager: LocalDocStorageManagerCreator;
+  // Creates a IDocStorageManager for storing documents on an external storage (e.g S3)
+  createHostedDocStorageManager: HostedDocStorageManagerCreator;
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
@@ -45,13 +74,6 @@ export interface ICreate {
   Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
   Shell?(): IShell;  // relevant to electron version of Grist only.
 
-  // Create a space to store files externally, for storing either:
-  //  - documents. This store should be versioned, and can be eventually consistent.
-  //  - meta. This store need not be versioned, and can be eventually consistent.
-  // For test purposes an extra prefix may be supplied.  Stores with different prefixes
-  // should not interfere with each other.
-  ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
-
   NSandbox(options: ISandboxCreationOptions): ISandbox;
 
   // Create the logic to determine which users are authorized to manage this Grist installation.
@@ -69,6 +91,8 @@ export interface ICreate {
   getStorageOptions?(name: string): ICreateStorageOptions|undefined;
   getSqliteVariant?(): SqliteVariant;
   getSandboxVariants?(): Record<string, SpawnFn>;
+
+  getLoginSystem(): Promise<GristLoginSystem>;
 }
 
 export interface ICreateActiveDocOptions {
@@ -126,6 +150,9 @@ export function makeSimpleCreator(opts: {
   getSqliteVariant?: () => SqliteVariant,
   getSandboxVariants?: () => Record<string, SpawnFn>,
   createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
+  getLoginSystem?: () => Promise<GristLoginSystem>,
+  createHostedDocStorageManager?: HostedDocStorageManagerCreator,
+  createLocalDocStorageManager?: LocalDocStorageManagerCreator,
 }): ICreate {
   const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
   return {
@@ -199,5 +226,23 @@ export function makeSimpleCreator(opts: {
     getSqliteVariant: opts.getSqliteVariant,
     getSandboxVariants: opts.getSandboxVariants,
     createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
+    getLoginSystem: opts.getLoginSystem || getCoreLoginSystem,
+    createLocalDocStorageManager: opts.createLocalDocStorageManager ?? createDefaultLocalStorageManager,
+    createHostedDocStorageManager: opts.createHostedDocStorageManager ?? createDefaultHostedStorageManager,
   };
 }
+
+const createDefaultHostedStorageManager: HostedDocStorageManagerCreator = async (
+      docsRoot,
+      docWorkerId,
+      disableS3,
+      docWorkerMap,
+      dbManager,
+      createExternalStorage, options
+) =>
+  new HostedStorageManager(docsRoot, docWorkerId, disableS3, docWorkerMap, dbManager, createExternalStorage, options);
+
+const createDefaultLocalStorageManager: LocalDocStorageManagerCreator = async (
+  docsRoot, samplesRoot, comm, shell
+) => new DocStorageManager(docsRoot, samplesRoot, comm, shell);
+

app/server/mergedServerMain.ts

@@ -1,230 +0,0 @@
-/**
- *
- * A version of hosted grist that recombines a home server,
- * a doc worker, and a static server on a single port.
- *
- */
-
-import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
-import {GristLoginSystem} from 'app/server/lib/GristServer';
-import log from 'app/server/lib/log';
-import {getGlobalConfig} from "app/server/lib/globalConfig";
-
-// Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
-// environment variable.
-export type ServerType = "home" | "docs" | "static" | "app";
-const allServerTypes: ServerType[] = ["home", "docs", "static", "app"];
-
-// Parse a comma-separate list of server types into an array, with validation.
-export function parseServerTypes(serverTypes: string|undefined): ServerType[] {
-  // Split and filter out empty strings (including the one we get when splitting "").
-  const types = (serverTypes || "").trim().split(',').filter(part => Boolean(part));
-
-  // Check that parts is non-empty and only contains valid options.
-  if (!types.length) {
-    throw new Error(`No server types; should be a comma-separated list of ${allServerTypes.join(", ")}`);
-  }
-  for (const t of types) {
-    if (!allServerTypes.includes(t as ServerType)) {
-      throw new Error(`Invalid server type '${t}'; should be in ${allServerTypes.join(", ")}`);
-    }
-  }
-  return types as ServerType[];
-}
-
-function checkUserContentPort(): number | null {
-  // Check whether a port is explicitly set for user content.
-  if (process.env.GRIST_UNTRUSTED_PORT) {
-    return parseInt(process.env.GRIST_UNTRUSTED_PORT, 10);
-  }
-  // Checks whether to serve user content on same domain but on different port
-  if (process.env.APP_UNTRUSTED_URL && process.env.APP_HOME_URL) {
-    const homeUrl = new URL(process.env.APP_HOME_URL);
-    const pluginUrl = new URL(process.env.APP_UNTRUSTED_URL);
-    // If the hostname of both home and plugin url are the same,
-    // but the ports are different
-    if (homeUrl.hostname === pluginUrl.hostname &&
-        homeUrl.port !== pluginUrl.port) {
-      const port = parseInt(pluginUrl.port || '80', 10);
-      return port;
-    }
-  }
-  return null;
-}
-
-interface ServerOptions extends FlexServerOptions {
-  logToConsole?: boolean;  // If set, messages logged to console (default: false)
-                           //   (but if options are not given at all in call to main,
-                           //    logToConsole is set to true)
-  externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
-                           // variables, which get set in various ways in dev/test entry points)
-  loginSystem?: () => Promise<GristLoginSystem>;
-}
-
-/**
- * Start a server on the given port, including the functionality specified in serverTypes.
- */
-export async function main(port: number, serverTypes: ServerType[],
-                           options: ServerOptions = {}) {
-  const includeHome = serverTypes.includes("home");
-  const includeDocs = serverTypes.includes("docs");
-  const includeStatic = serverTypes.includes("static");
-  const includeApp = serverTypes.includes("app");
-
-  options.settings ??= getGlobalConfig();
-
-  const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
-
-  // We need to know early on whether we will be serving plugins or not.
-  if (includeHome) {
-    const userPort = checkUserContentPort();
-    server.setServesPlugins(userPort !== undefined);
-  } else {
-    server.setServesPlugins(false);
-  }
-
-  if (options.loginSystem) {
-    server.setLoginSystem(options.loginSystem);
-  }
-
-  server.addCleanup();
-  server.setDirectory();
-
-  if (process.env.GRIST_TEST_ROUTER) {
-    // Add a mock api for adding/removing doc workers from load balancer.
-    server.testAddRouter();
-  }
-
-  if (options.logToConsole !== false) { server.addLogging(); }
-  if (options.externalStorage === false) { server.disableExternalStorage(); }
-  await server.addLoginMiddleware();
-
-  if (includeDocs) {
-    // It is important that /dw and /v prefixes are accepted (if present) by health check
-    // in this case, since they are included in the url registered for the doc worker.
-    server.stripDocWorkerIdPathPrefixIfPresent();
-    server.addTagChecker();
-  }
-
-  server.addHealthCheck();
-  if (includeHome || includeApp) {
-    server.addBootPage();
-  }
-  server.denyRequestsIfNotReady();
-
-  if (includeHome || includeStatic || includeApp) {
-    server.setDirectory();
-  }
-
-  if (includeHome || includeStatic) {
-    server.addStaticAndBowerDirectories();
-  }
-
-  await server.initHomeDBManager();
-  server.addHosts();
-
-  server.addDocWorkerMap();
-
-  if (includeHome || includeStatic) {
-    await server.addAssetsForPlugins();
-  }
-
-  if (includeHome) {
-    server.addEarlyWebhooks();
-  }
-
-  if (includeHome || includeDocs || includeApp) {
-    server.addSessions();
-  }
-
-  server.addAccessMiddleware();
-  server.addApiMiddleware();
-  await server.addBillingMiddleware();
-
-  try {
-    await server.start();
-
-    if (includeHome) {
-      server.addUsage();
-      if (!includeDocs) {
-        server.addDocApiForwarder();
-      }
-      server.addJsonSupport();
-      server.addUpdatesCheck();
-      await server.addLandingPages();
-      // todo: add support for home api to standalone app
-      server.addHomeApi();
-      server.addBillingApi();
-      server.addNotifier();
-      server.addAuditLogger();
-      await server.addTelemetry();
-      await server.addHousekeeper();
-      await server.addLoginRoutes();
-      server.addAccountPage();
-      server.addBillingPages();
-      server.addWelcomePaths();
-      server.addLogEndpoint();
-      server.addGoogleAuthEndpoint();
-      server.addInstallEndpoints();
-      server.addConfigEndpoints();
-    }
-
-    if (includeDocs) {
-      server.addJsonSupport();
-      server.addAuditLogger();
-      await server.addTelemetry();
-      await server.addDoc();
-    }
-
-    if (includeHome) {
-      server.addClientSecrets();
-    }
-
-    server.finalizeEndpoints();
-    await server.finalizePlugins(includeHome ? checkUserContentPort() : null);
-    server.checkOptionCombinations();
-    server.summary();
-    server.ready();
-
-    // Some tests have their timing perturbed by having this earlier
-    // TODO: update those tests.
-    if (includeDocs) {
-      await server.checkSandbox();
-    }
-    return server;
-  } catch(e) {
-    await server.close();
-    throw e;
-  }
-}
-
-
-export async function startMain() {
-  try {
-
-    const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
-
-    // No defaults for a port, since this server can serve very different purposes.
-    if (!process.env.GRIST_PORT) {
-      throw new Error("GRIST_PORT must be specified");
-    }
-
-    const port = parseInt(process.env.GRIST_PORT, 10);
-
-    const server = await main(port, serverTypes);
-
-    const opt = process.argv[2];
-    if (opt === '--testingHooks') {
-      await server.addTestingHooks();
-    }
-
-    return server;
-  } catch (e) {
-    log.error('mergedServer failed to start', e);
-    process.exit(1);
-  }
-}
-
-if (require.main === module) {
-  startMain().catch((e) => log.error('mergedServer failed to start', e));
-}

app/server/utils/pruneActionHistory.ts

@@ -1,9 +1,9 @@
 import * as gutil from 'app/common/gutil';
 import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
 import {DocStorage} from 'app/server/lib/DocStorage';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import log from 'app/server/lib/log';
+import {create} from "app/server/lib/create";
 
 /**
  * A utility script for cleaning up the action log.
@@ -18,7 +18,7 @@ export async function pruneActionHistory(docPath: string, keepN: number) {
     throw new Error('Invalid document: Document should be a valid .grist file');
   }
 
-  const storageManager = new DocStorageManager(".", ".");
+  const storageManager = await create.createLocalDocStorageManager(".", ".");
   const docStorage = new DocStorage(storageManager, docPath);
   const backupPath = gutil.removeSuffix(docPath, '.grist') + "-backup.grist";