Extracts config.json into its own module (#1061)

This adds a config file that's loaded very early on during startup. 

It enables us to save/load settings from within Grist's admin panel, that affect the startup of the FlexServer.

The config file loading:
- Is type-safe, 
- Validates the config file on startup
- Provides a path to upgrade to future versions.

It should be extensible from other versions of Grist (such as desktop), by overriding `getGlobalConfig` in stubs.

----

Some minor refactors needed to occur to make this possible. This includes:
- Extracting config loading into its own module (out of FlexServer).
- Cleaning up the `loadConfig` function in FlexServer into `loadLoginSystem` (which is what its main purpose was before).

test/gen-server/seed.ts

@@ -605,7 +605,7 @@ export async function createServer(port: number, initDb = createInitialDb): Prom
   await flexServer.start();
   await flexServer.initHomeDBManager();
   flexServer.addDocWorkerMap();
-  await flexServer.loadConfig();
+  await flexServer.addLoginMiddleware();
   flexServer.addHosts();
   flexServer.addAccessMiddleware();
   flexServer.addApiMiddleware();

test/server/lib/Authorizer.ts

@@ -17,13 +17,13 @@ let server: FlexServer;
 let dbManager: HomeDBManager;
 
 async function activateServer(home: FlexServer, docManager: DocManager) {
-  await home.loadConfig();
+  await home.addLoginMiddleware();
   await home.initHomeDBManager();
   home.addHosts();
   home.addDocWorkerMap();
   home.addAccessMiddleware();
   dbManager = home.getHomeDBManager();
-  await home.loadConfig();
+  await home.addLoginMiddleware();
   home.addSessions();
   home.addHealthCheck();
   docManager.testSetHomeDbManager(dbManager);

test/server/lib/config.ts

@@ -0,0 +1,107 @@
+import { assert } from 'chai';
+import * as sinon from 'sinon';
+import { ConfigAccessors, createConfigValue, Deps, FileConfig } from "app/server/lib/config";
+
+interface TestFileContents {
+  myNum?: number
+  myStr?: string
+}
+
+const testFileContentsExample: TestFileContents = {
+  myNum: 1,
+  myStr: "myStr",
+};
+
+const testFileContentsJSON = JSON.stringify(testFileContentsExample);
+
+describe('FileConfig', () => {
+  const useFakeConfigFile = (contents: string) => {
+    const fakeFile = { contents };
+    sinon.replace(Deps, 'pathExists', sinon.fake.resolves(true));
+    sinon.replace(Deps, 'readFile', sinon.fake((path, encoding: string) => Promise.resolve(fakeFile.contents)) as any);
+    sinon.replace(Deps, 'writeFile', sinon.fake((path, newContents) => {
+      fakeFile.contents = newContents;
+      return Promise.resolve();
+    }));
+
+    return fakeFile;
+  };
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('throws an error from create if the validator does not return a value', async () => {
+    useFakeConfigFile(testFileContentsJSON);
+    const validator = () => null;
+    await assert.isRejected(FileConfig.create<TestFileContents>("anypath.json", validator));
+  });
+
+  it('persists changes when values are assigned', async () => {
+    const fakeFile = useFakeConfigFile(testFileContentsJSON);
+    // Don't validate - this is guaranteed to be valid above.
+    const validator = (input: any) => input as TestFileContents;
+    const fileConfig = await FileConfig.create("anypath.json", validator);
+    await fileConfig.set("myNum", 999);
+
+    assert.equal(fileConfig.get("myNum"), 999);
+    assert.equal(JSON.parse(fakeFile.contents).myNum, 999);
+  });
+
+  // Avoid removing extra properties from the file, in case another edition of grist is doing something.
+  it('does not remove extra values from the file', async () => {
+    const configWithExtraProperties = {
+      ...testFileContentsExample,
+      someProperty: "isPresent",
+    };
+
+    const fakeFile = useFakeConfigFile(JSON.stringify(configWithExtraProperties));
+    // It's entirely possible the validator can damage the extra properties, but that's not in scope for this test.
+    const validator = (input: any) => input as TestFileContents;
+    const fileConfig = await FileConfig.create("anypath.json", validator);
+    // Triggering a write to the file
+    await fileConfig.set("myNum", 999);
+    await fileConfig.set("myStr", "Something");
+
+    const newContents = JSON.parse(fakeFile.contents);
+    assert.equal(newContents.myNum, 999);
+    assert.equal(newContents.myStr, "Something");
+    assert.equal(newContents.someProperty, "isPresent");
+  });
+});
+
+describe('createConfigValue', () => {
+  const makeInMemoryAccessors = <T>(initialValue: T): ConfigAccessors<T> => {
+    let value: T = initialValue;
+    return {
+      get: () => value,
+      set: async (newValue: T) => { value = newValue; },
+    };
+  };
+
+  it('works without persistence', async () => {
+    const configValue = createConfigValue(1);
+    assert.equal(configValue.get(), 1);
+    await configValue.set(2);
+    assert.equal(configValue.get(), 2);
+  });
+
+  it('writes to persistence when saved', async () => {
+    const accessors = makeInMemoryAccessors(1);
+    const configValue = createConfigValue(1, accessors);
+    assert.equal(accessors.get(), 1);
+    await configValue.set(2);
+    assert.equal(accessors.get(), 2);
+  });
+
+  it('initialises with the persistent value if available', async () => {
+    const accessors = makeInMemoryAccessors(22);
+    const configValue = createConfigValue(1, accessors);
+    assert.equal(configValue.get(), 22);
+
+    const accessorsWithUndefinedValue = makeInMemoryAccessors<number | undefined>(undefined);
+    const configValueWithDefault = createConfigValue(333, accessorsWithUndefinedValue);
+    assert.equal(configValueWithDefault.get(), 333);
+  });
+});
+

test/server/lib/configCore.ts

@@ -0,0 +1,48 @@
+import * as sinon from 'sinon';
+import { assert } from 'chai';
+import { IGristCoreConfig, loadGristCoreConfig, loadGristCoreConfigFile } from "app/server/lib/configCore";
+import { createConfigValue, Deps, IWritableConfigValue } from "app/server/lib/config";
+
+describe('loadGristCoreConfig', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('can be used with an in-memory store if no file config is provided', async () => {
+    const config = loadGristCoreConfig();
+    await config.edition.set("enterprise");
+    assert.equal(config.edition.get(), "enterprise");
+  });
+
+  it('will function correctly when no config file is present', async () => {
+    sinon.replace(Deps, 'pathExists', sinon.fake.resolves(false));
+    sinon.replace(Deps, 'readFile', sinon.fake.resolves(""));
+    const writeFileFake = sinon.fake.resolves(undefined);
+    sinon.replace(Deps, 'writeFile', writeFileFake);
+
+    const config = await loadGristCoreConfigFile("doesntmatter.json");
+    assert.exists(config.edition.get());
+
+    await config.edition.set("enterprise");
+    // Make sure that the change was written back to the file.
+    assert.isTrue(writeFileFake.calledOnce);
+  });
+
+  it('can be extended', async () => {
+    // Extend the core config
+    type NewConfig = IGristCoreConfig & {
+      newThing: IWritableConfigValue<number>
+    };
+
+    const coreConfig = loadGristCoreConfig();
+
+    const newConfig: NewConfig = {
+      ...coreConfig,
+      newThing: createConfigValue(3)
+    };
+
+    // Ensure that it's backwards compatible.
+    const gristConfig: IGristCoreConfig = newConfig;
+    return gristConfig;
+  });
+});

test/server/lib/configCoreFileFormats.ts

@@ -0,0 +1,29 @@
+import { assert } from 'chai';
+import { convertToCoreFileContents, IGristCoreConfigFileLatest } from "app/server/lib/configCoreFileFormats";
+
+describe('convertToCoreFileContents', () => {
+  it('fails with a malformed config', async () => {
+    const badConfig = {
+      version: "This is a random version number that will never exist",
+    };
+
+    assert.throws(() => convertToCoreFileContents(badConfig));
+  });
+
+  // This is necessary to handle users who don't have a config file yet.
+  it('will upgrade an empty object to a valid config', () => {
+    const validConfig = convertToCoreFileContents({});
+    assert.exists(validConfig?.version);
+  });
+
+  it('will validate the latest config file format', () => {
+    const validRawObject: IGristCoreConfigFileLatest = {
+      version: "1",
+      edition: "enterprise",
+    };
+
+    const validConfig = convertToCoreFileContents(validRawObject);
+    assert.exists(validConfig?.version);
+    assert.exists(validConfig?.edition);
+  });
+});