Make changes required for Desktop FS updates (#1099)

Make a set of changes required for Desktop FS improvements, see
https://github.com/gristlabs/grist-desktop/pull/42

---------

Co-authored-by: Spoffy <contact@spoffy.net>
Co-authored-by: Spoffy <4805393+Spoffy@users.noreply.github.com>

test/gen-server/AuthCaching.ts

@@ -2,7 +2,7 @@ import {delay} from 'app/common/delay';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import log from 'app/server/lib/log';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from 'app/server/MergedServer';
 import axios from 'axios';
 import {assert} from 'chai';
 import * as fse from 'fs-extra';
@@ -50,12 +50,17 @@ describe('AuthCaching', function() {
     setUpDB();
     await createInitialDb();
     process.env.GRIST_DATA_DIR = testDocDir;
-    homeServer = await mergedServerMain(0, ['home'],
+
+    const homeMS = await MergedServer.create(0, ['home'],
       {logToConsole: false, externalStorage: false});
+    await homeMS.run();
+    homeServer = homeMS.flexServer;
     homeUrl = homeServer.getOwnUrl();
     process.env.APP_HOME_URL = homeUrl;
-    docsServer = await mergedServerMain(0, ['docs'],
+    const docsMS = await MergedServer.create(0, ['docs'],
       {logToConsole: false, externalStorage: false});
+    await docsMS.run();
+    docsServer = docsMS.flexServer;
 
     // Helpers for getting cookie-based logins.
     session = new TestSession(homeServer);

test/gen-server/apiUtils.ts

@@ -14,7 +14,7 @@ import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
-import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
+import {MergedServer, ServerType} from 'app/server/MergedServer';
 import axios from 'axios';
 import FormData from 'form-data';
 import fetch from 'node-fetch';
@@ -37,9 +37,10 @@ export class TestServer {
   public async start(servers: ServerType[] = ["home"],
                      options: FlexServerOptions = {}): Promise<string> {
     await createInitialDb();
-    this.server = await mergedServerMain(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
-                                                      externalStorage: false,
-                                                      ...options});
+    const mergedServer = await MergedServer.create(0, servers, {logToConsole: isAffirmative(process.env.DEBUG),
+                                                      externalStorage: false, ...options});
+    await mergedServer.run();
+    this.server = mergedServer.flexServer;
     this.serverUrl = this.server.getOwnUrl();
     this.dbManager = this.server.getHomeDBManager();
     this.defaultSession = new TestSession(this.server);
@@ -263,7 +264,7 @@ export class TestSession {
     if (clearCache) { this.home.getSessions().clearCacheIfNeeded(); }
     this.headers.Cookie = cookie;
     return {
-      validateStatus: (status: number) => true,
+      validateStatus: (_status: number) => true,
       headers: {
         'Cookie': cookie,
         'X-Requested-With': 'XMLHttpRequest',

test/gen-server/lib/DocWorkerMap.ts

@@ -2,7 +2,7 @@ import {DocWorkerMap, getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import {Permit} from 'app/server/lib/Permit';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from "app/server/MergedServer";
 import {delay, promisifyAll} from 'bluebird';
 import {assert, expect} from 'chai';
 import {countBy, values} from 'lodash';
@@ -387,24 +387,34 @@ describe('DocWorkerMap', function() {
       process.env.REDIS_URL = process.env.TEST_REDIS_URL;
 
       // Make home server.
-      const home = await mergedServerMain(0, ['home'], opts);
+      const homeMergedServer = await MergedServer.create(0, ['home'], opts);
+      const home = homeMergedServer.flexServer;
+      await homeMergedServer.run();
 
       // Make a worker, not associated with any group.
       process.env.GRIST_DOC_WORKER_ID = 'worker1';
-      const docs1 = await mergedServerMain(0, ['docs'], opts);
+      const docs1MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs1 = docs1MergedServer.flexServer;
+      await docs1MergedServer.run();
 
       // Make a worker in "special" group.
       process.env.GRIST_DOC_WORKER_ID = 'worker2';
       process.env.GRIST_WORKER_GROUP = 'special';
-      const docs2 = await mergedServerMain(0, ['docs'], opts);
+      const docs2MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs2 = docs2MergedServer.flexServer;
+      await docs2MergedServer.run();
 
       // Make two worker in "other" group.
       process.env.GRIST_DOC_WORKER_ID = 'worker3';
       process.env.GRIST_WORKER_GROUP = 'other';
-      const docs3 = await mergedServerMain(0, ['docs'], opts);
+      const docs3MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs3 = docs3MergedServer.flexServer;
+      await docs3MergedServer.run();
       process.env.GRIST_DOC_WORKER_ID = 'worker4';
       process.env.GRIST_WORKER_GROUP = 'other';
-      const docs4 = await mergedServerMain(0, ['docs'], opts);
+      const docs4MergedServer = await MergedServer.create(0, ['docs'], opts);
+      const docs4 = docs4MergedServer.flexServer;
+      await docs4MergedServer.run();
 
       servers = {home, docs1, docs2, docs3, docs4};
       workers = getDocWorkerMap();

test/gen-server/lib/mergedOrgs.ts

@@ -1,7 +1,7 @@
 import {Workspace} from 'app/common/UserAPI';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
-import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import {MergedServer} from "app/server/MergedServer";
 import axios from 'axios';
 import {assert} from 'chai';
 import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
@@ -9,6 +9,7 @@ import {configForUser, createUser, setPlan} from 'test/gen-server/testUtils';
 import * as testUtils from 'test/server/testUtils';
 
 describe('mergedOrgs', function() {
+  let mergedServer: MergedServer;
   let home: FlexServer;
   let dbManager: HomeDBManager;
   let homeUrl: string;
@@ -20,8 +21,10 @@ describe('mergedOrgs', function() {
   before(async function() {
     setUpDB(this);
     await createInitialDb();
-    home = await mergedServerMain(0, ["home", "docs"],
+    mergedServer = await MergedServer.create(0, ["home", "docs"],
                                   {logToConsole: false, externalStorage: false});
+    home = mergedServer.flexServer;
+    await mergedServer.run();
     dbManager = home.getHomeDBManager();
     homeUrl = home.getOwnUrl();
   });

test/server/docTools.ts

@@ -4,7 +4,6 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {DummyAuthorizer} from 'app/server/lib/Authorizer';
 import {DocManager} from 'app/server/lib/DocManager';
 import {DocSession, makeExceptionalDocSession} from 'app/server/lib/DocSession';
-import {DocStorageManager} from 'app/server/lib/DocStorageManager';
 import {createDummyGristServer, GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {getAppRoot} from 'app/server/lib/places';
@@ -17,6 +16,7 @@ import * as fse from 'fs-extra';
 import {tmpdir} from 'os';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import {create} from "app/server/lib/create";
 
 tmp.setGracefulCleanup();
 
@@ -138,7 +138,7 @@ export async function createDocManager(
               server?: GristServer} = {}): Promise<DocManager> {
   // Set Grist home to a temporary directory, and wipe it out on exit.
   const tmpDir = options.tmpDir || await createTmpDir();
-  const docStorageManager = options.storageManager || new DocStorageManager(tmpDir);
+  const docStorageManager = options.storageManager || await create.createLocalDocStorageManager(tmpDir);
   const pluginManager = options.pluginManager || await getGlobalPluginManager();
   const store = getDocWorkerMap();
   const internalPermitStore = store.getPermitStore('1');

test/server/lib/HostedStorageManager.ts

@@ -7,7 +7,12 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {create} from 'app/server/lib/create';
 import {DocManager} from 'app/server/lib/DocManager';
 import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
-import {DELETED_TOKEN, ExternalStorage, wrapWithKeyMappedStorage} from 'app/server/lib/ExternalStorage';
+import {
+  DELETED_TOKEN,
+  ExternalStorage, ExternalStorageCreator,
+  ExternalStorageSettings,
+  wrapWithKeyMappedStorage
+} from 'app/server/lib/ExternalStorage';
 import {createDummyGristServer} from 'app/server/lib/GristServer';
 import {
   BackupEvent,
@@ -270,7 +275,7 @@ class TestStore {
     private _localDirectory: string,
     private _workerId: string,
     private _workers: DocWorkerMap,
-    private _externalStorageCreate: (purpose: 'doc'|'meta', extraPrefix: string) => ExternalStorage|undefined) {
+    private _externalStorageCreate: ExternalStorageCreator) {
   }
 
   public async run<T>(fn: () => Promise<T>): Promise<T> {
@@ -296,18 +301,20 @@ class TestStore {
       secondsBeforeFirstRetry: 3,   // rumors online suggest delays of 10-11 secs
                                     // are not super-unusual.
       pushDocUpdateTimes: false,
-      externalStorageCreator: (purpose) => {
+
+    };
+    const externalStorageCreator = (purpose: ExternalStorageSettings["purpose"]) => {
         const result = this._externalStorageCreate(purpose, this._extraPrefix);
         if (!result) { throw new Error('no storage'); }
         return result;
-      }
     };
+
     const storageManager = new HostedStorageManager(this._localDirectory,
                                                     this._workerId,
                                                     false,
                                                     this._workers,
                                                     dbManager,
-                                                    create,
+                                                    externalStorageCreator,
                                                     options);
     this.storageManager = storageManager;
     this.docManager = new DocManager(storageManager, await getGlobalPluginManager(),

test/server/lib/helpers/TestServer.ts

@@ -91,7 +91,7 @@ export class TestServer {
       ...this._defaultEnv,
       ...customEnv
     };
-    const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
+    const main = await testUtils.getBuildFile('app/server/MergedServer.js');
     this._server = spawn('node', [main, '--testingHooks'], {
       env,
       stdio: ['inherit', serverLog, serverLog]