(core) updates from grist-core

.github/workflows/docker_latest.yml

@@ -11,11 +11,36 @@ on:
     - cron:  '41 5 * * *'
   workflow_dispatch:
     inputs:
-      latest_branch:
+      branch:
         description: "Branch from which to create the latest Docker image (default: latest_candidate)"
         type: string
         required: true
-        default_value: latest_candidate
+        default: latest_candidate
+      disable_tests:
+        description: "Should the tests be skipped?"
+        type: boolean
+        required: True
+        default: False
+      platforms:
+        description: "Platforms to build"
+        type: choice
+        required: True
+        options:
+          - linux/amd64
+          - linux/arm64/v8
+          - linux/amd64,linux/arm64/v8
+        default: linux/amd64,linux/arm64/v8
+      tag:
+        description: "Tag for the resulting images"
+        type: string
+        required: True
+        default: 'experimental'
+
+env:
+  BRANCH: ${{ inputs.branch || 'latest_candidate' }}
+  PLATFORMS: ${{ inputs.platforms || 'linux/amd64,linux/arm64/v8' }}
+  TAG: ${{ inputs.tag || 'experimental' }}
+  DOCKER_HUB_OWNER: ${{ vars.DOCKER_HUB_OWNER || github.repository_owner }}
 
 jobs:
   push_to_registry:
@@ -32,21 +57,23 @@ jobs:
             repo: "grist-core"
           - name: "grist"
             repo: "grist-ee"
-          # For now, we build it twice, with `grist-ee` being a
-          # backwards-compatible synonym for `grist`.
-          - name: "grist-ee"
-            repo: "grist-ee"
     steps:
+      - name: Build settings
+        run: |
+          echo "Branch: $BRANCH"
+          echo "Platforms: $PLATFORMS"
+          echo "Docker Hub Owner: $DOCKER_HUB_OWNER"
+          echo "Tag: $TAG"
+
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.latest_branch }}
+          ref: ${{ env.BRANCH }}
 
       - name: Check out the ext/ directory
         if: matrix.image.name != 'grist-oss'
         run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }}
 
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
 
@@ -58,38 +85,44 @@ jobs:
         with:
           context: .
           load: true
-          tags: ${{ github.repository_owner }}/${{ matrix.image.name }}:experimental
+          tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}
           cache-from: type=gha
-          build-contexts: ${{ matrix.image.name != 'grist-oss' && 'ext=ext' || '' }}
+          build-contexts: ext=ext
 
       - name: Use Node.js ${{ matrix.node-version }} for testing
+        if: ${{ !inputs.disable_tests }}
         uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node-version }}
 
       - name: Set up Python ${{ matrix.python-version }} for testing - maybe not needed
+        if: ${{ !inputs.disable_tests }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Python packages
+        if: ${{ !inputs.disable_tests }}
         run: |
           pip install virtualenv
           yarn run install:python
 
       - name: Install Node.js packages
+        if: ${{ !inputs.disable_tests }}
         run: yarn install
 
       - name: Build Node.js code
+        if: ${{ !inputs.disable_tests }}
         run: |
           rm -rf ext
           yarn run build:prod
 
       - name: Run tests
-        run: TEST_IMAGE=${{ github.repository_owner }}/${{ matrix.image.name }}:experimental VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker
+        if: ${{ !inputs.disable_tests }}
+        run: TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker
 
       - name: Restore the ext/ directory
-        if: matrix.image.name != 'grist-oss'
+        if: ${{ matrix.image.name != 'grist-oss' && !inputs.disable_tests }}
         run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }}
 
       - name: Log in to Docker Hub
@@ -102,12 +135,27 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          platforms: linux/amd64,linux/arm64/v8
+          platforms: ${{ env.PLATFORMS }}
           push: true
-          tags: ${{ github.repository_owner }}/${{ matrix.image.name }}:experimental
+          tags: ${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          build-contexts: ext=ext
+
+      - name: Push Enterprise to Docker Hub
+        if: ${{ matrix.image.name == 'grist' }}
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          build-args: |
+            BASE_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name}}
+            BASE_VERSION=${{ env.TAG }}
+          file: ext/Dockerfile
+          platforms: ${{ env.PLATFORMS }}
+          push: true
+          tags: ${{ env.DOCKER_HUB_OWNER }}/grist-ee:${{ env.TAG }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          build-contexts: ${{ matrix.image.name != 'grist-oss' && 'ext=ext' || '' }}
 
   update_latest_branch:
     name: Update latest branch

.gitignore

@@ -80,3 +80,6 @@ xunit.xml
 .clipboard.lock
 
 **/_build
+
+# ext directory can be overwritten
+ext/**

app/server/lib/FlexServer.ts

@@ -54,7 +54,7 @@ import {InstallAdmin} from 'app/server/lib/InstallAdmin';
 import log from 'app/server/lib/log';
 import {getLoginSystem} from 'app/server/lib/logins';
 import {IPermitStore} from 'app/server/lib/Permit';
-import {getAppPathTo, getAppRoot, getUnpackedAppRoot} from 'app/server/lib/places';
+import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
 import {PluginManager} from 'app/server/lib/PluginManager';
 import * as ProcessMonitor from 'app/server/lib/ProcessMonitor';
@@ -87,6 +87,7 @@ import {AddressInfo} from 'net';
 import fetch from 'node-fetch';
 import * as path from 'path';
 import * as serveStatic from "serve-static";
+import {IGristCoreConfig} from "./configCore";
 
 // Health checks are a little noisy in the logs, so we don't show them all.
 // We show the first N health checks:
@@ -105,6 +106,9 @@ export interface FlexServerOptions {
   baseDomain?: string;
   // Base URL for plugins, if permitted. Defaults to APP_UNTRUSTED_URL.
   pluginUrl?: string;
+
+  // Global grist config options
+  settings?: IGristCoreConfig;
 }
 
 const noop: express.RequestHandler = (req, res, next) => next();
@@ -122,7 +126,7 @@ export class FlexServer implements GristServer {
   public housekeeper: Housekeeper;
   public server: http.Server;
   public httpsServer?: https.Server;
-  public settings?: Readonly<Record<string, unknown>>;
+  public settings?: IGristCoreConfig;
   public worker: DocWorkerInfo;
   public electronServerMethods: ElectronServerMethods;
   public readonly docsRoot: string;
@@ -186,6 +190,7 @@ export class FlexServer implements GristServer {
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
+    this.settings = options.settings;
     this.app = express();
     this.app.set('port', port);
 
@@ -662,7 +667,7 @@ export class FlexServer implements GristServer {
 
   public get instanceRoot() {
     if (!this._instanceRoot) {
-      this._instanceRoot = path.resolve(process.env.GRIST_INST_DIR || this.appRoot);
+      this._instanceRoot = getInstanceRoot();
       this.info.push(['instanceRoot', this._instanceRoot]);
     }
     return this._instanceRoot;
@@ -774,7 +779,7 @@ export class FlexServer implements GristServer {
   // Set up the main express middleware used.  For a single user setup, without logins,
   // all this middleware is currently a no-op.
   public addAccessMiddleware() {
-    if (this._check('middleware', 'map', 'config', isSingleUserMode() ? null : 'hosts')) { return; }
+    if (this._check('middleware', 'map', 'loginMiddleware', isSingleUserMode() ? null : 'hosts')) { return; }
 
     if (!isSingleUserMode()) {
       const skipSession = appSettings.section('login').flag('skipSession').readBool({
@@ -938,7 +943,7 @@ export class FlexServer implements GristServer {
   }
 
   public addSessions() {
-    if (this._check('sessions', 'config')) { return; }
+    if (this._check('sessions', 'loginMiddleware')) { return; }
     this.addTagChecker();
     this.addOrg();
 
@@ -1135,25 +1140,8 @@ export class FlexServer implements GristServer {
     });
   }
 
-  /**
-   * Load user config file from standard location (if present).
-   *
-   * Note that the user config file doesn't do anything today, but may be useful in
-   * the future for configuring things that don't fit well into environment variables.
-   *
-   * TODO: Revisit this, and update `GristServer.settings` type to match the expected shape
-   * of config.json. (ts-interface-checker could be useful here for runtime validation.)
-   */
-  public async loadConfig() {
-    if (this._check('config')) { return; }
-    const settingsPath = path.join(this.instanceRoot, 'config.json');
-    if (await fse.pathExists(settingsPath)) {
-      log.info(`Loading config from ${settingsPath}`);
-      this.settings = JSON.parse(await fse.readFile(settingsPath, 'utf8'));
-    } else {
-      log.info(`Loading empty config because ${settingsPath} missing`);
-      this.settings = {};
-    }
+  public async addLoginMiddleware() {
+    if (this._check('loginMiddleware')) { return; }
 
     // TODO: We could include a third mock provider of login/logout URLs for better tests. Or we
     // could create a mock SAML identity provider for testing this using the SAML flow.
@@ -1169,9 +1157,9 @@ export class FlexServer implements GristServer {
   }
 
   public addComm() {
-    if (this._check('comm', 'start', 'homedb', 'config')) { return; }
+    if (this._check('comm', 'start', 'homedb', 'loginMiddleware')) { return; }
     this._comm = new Comm(this.server, {
-      settings: this.settings,
+      settings: {},
       sessions: this._sessions,
       hosts: this._hosts,
       loginMiddleware: this._loginMiddleware,
@@ -1311,7 +1299,7 @@ export class FlexServer implements GristServer {
       null : 'homedb', 'api-mw', 'map', 'telemetry');
     // add handlers for cleanup, if we are in charge of the doc manager.
     if (!this._docManager) { this.addCleanup(); }
-    await this.loadConfig();
+    await this.addLoginMiddleware();
     this.addComm();
 
     await this.create.configure?.();

app/server/lib/GristServer.ts

@@ -25,6 +25,7 @@ import { Sessions } from 'app/server/lib/Sessions';
 import { ITelemetry } from 'app/server/lib/Telemetry';
 import * as express from 'express';
 import { IncomingMessage } from 'http';
+import { IGristCoreConfig, loadGristCoreConfig } from "./configCore";
 
 /**
  * Basic information about a Grist server.  Accessible in many
@@ -32,7 +33,7 @@ import { IncomingMessage } from 'http';
  */
 export interface GristServer {
   readonly create: ICreate;
-  settings?: Readonly<Record<string, unknown>>;
+  settings?: IGristCoreConfig;
   getHost(): string;
   getHomeUrl(req: express.Request, relPath?: string): string;
   getHomeInternalUrl(relPath?: string): string;
@@ -126,7 +127,7 @@ export interface DocTemplate {
 export function createDummyGristServer(): GristServer {
   return {
     create,
-    settings: {},
+    settings: loadGristCoreConfig(),
     getHost() { return 'localhost:4242'; },
     getHomeUrl() { return 'http://localhost:4242'; },
     getHomeInternalUrl() { return 'http://localhost:4242'; },

app/server/lib/config.ts

@@ -0,0 +1,143 @@
+import * as fse from "fs-extra";
+
+// Export dependencies for stubbing in tests.
+export const Deps = {
+  readFile: fse.readFile,
+  writeFile: fse.writeFile,
+  pathExists: fse.pathExists,
+};
+
+/**
+ * Readonly config value - no write access.
+ */
+export interface IReadableConfigValue<T> {
+  get(): T;
+}
+
+/**
+ * Writeable config value. Write behaviour is asynchronous and defined by the implementation.
+ */
+export interface IWritableConfigValue<T> extends IReadableConfigValue<T> {
+  set(value: T): Promise<void>;
+}
+
+type FileContentsValidator<T> = (value: any) => T | null;
+
+export class MissingConfigFileError extends Error {
+  public name: string = "MissingConfigFileError";
+
+  constructor(message: string) {
+    super(message);
+  }
+}
+
+export class ConfigValidationError extends Error {
+  public name: string = "ConfigValidationError";
+
+  constructor(message: string) {
+    super(message);
+  }
+}
+
+export interface ConfigAccessors<ValueType> {
+  get: () => ValueType,
+  set?: (value: ValueType) => Promise<void>
+}
+
+/**
+ * Provides type safe access to an underlying JSON file.
+ *
+ * Multiple FileConfigs for the same file shouldn't be used, as they risk going out of sync.
+ */
+export class FileConfig<FileContents> {
+  /**
+   * Creates a new type-safe FileConfig, by loading and checking the contents of the file with `validator`.
+   * @param configPath - Path to load.
+   * @param validator - Validates the contents are in the correct format, and converts to the correct type.
+   *  Should throw an error or return null if not valid.
+   */
+  public static async create<CreateConfigFileContents>(
+    configPath: string,
+    validator: FileContentsValidator<CreateConfigFileContents>
+  ): Promise<FileConfig<CreateConfigFileContents>> {
+    // Start with empty object, as it can be upgraded to a full config.
+    let rawFileContents: any = {};
+
+    if (await Deps.pathExists(configPath)) {
+      rawFileContents = JSON.parse(await Deps.readFile(configPath, 'utf8'));
+    }
+
+    let fileContents = null;
+
+    try {
+      fileContents = validator(rawFileContents);
+    } catch (error) {
+      const configError =
+        new ConfigValidationError(`Config at ${configPath} failed validation: ${error.message}`);
+      configError.cause = error;
+      throw configError;
+    }
+
+    if (!fileContents) {
+      throw new ConfigValidationError(`Config at ${configPath} failed validation - check the format?`);
+    }
+
+    return new FileConfig<CreateConfigFileContents>(configPath, fileContents);
+  }
+
+  constructor(private _filePath: string, private _rawConfig: FileContents) {
+  }
+
+  public get<Key extends keyof FileContents>(key: Key): FileContents[Key] {
+    return this._rawConfig[key];
+  }
+
+  public async set<Key extends keyof FileContents>(key: Key, value: FileContents[Key]) {
+    this._rawConfig[key] = value;
+    await this.persistToDisk();
+  }
+
+  public async persistToDisk(): Promise<void> {
+    await Deps.writeFile(this._filePath, JSON.stringify(this._rawConfig, null, 2));
+  }
+}
+
+/**
+ * Creates a function for creating accessors for a given key.
+ * Propagates undefined values, so if no file config is available, accessors are undefined.
+ * @param fileConfig - Config to load/save values to.
+ */
+export function fileConfigAccessorFactory<FileContents>(
+  fileConfig?: FileConfig<FileContents>
+): <Key extends keyof FileContents>(key: Key) => ConfigAccessors<FileContents[Key]> | undefined
+{
+  if (!fileConfig) { return (key) => undefined; }
+  return (key) => ({
+    get: () => fileConfig.get(key),
+    set: (value) => fileConfig.set(key, value)
+  });
+}
+
+/**
+ * Creates a config value optionally backed by persistent storage.
+ * Can be used as an in-memory value without persistent storage.
+ * @param defaultValue - Value to use if no persistent value is available.
+ * @param persistence - Accessors for saving/loading persistent value.
+ */
+export function createConfigValue<ValueType>(
+  defaultValue: ValueType,
+  persistence?: ConfigAccessors<ValueType> | ConfigAccessors<ValueType | undefined>,
+): IWritableConfigValue<ValueType> {
+  let inMemoryValue = (persistence && persistence.get());
+  return {
+    get(): ValueType {
+      return inMemoryValue ?? defaultValue;
+    },
+    async set(value: ValueType) {
+      if (persistence && persistence.set) {
+        await persistence.set(value);
+      }
+      inMemoryValue = value;
+    }
+  };
+}

app/server/lib/configCore.ts

@@ -0,0 +1,28 @@
+import {
+  createConfigValue,
+  FileConfig,
+  fileConfigAccessorFactory,
+  IWritableConfigValue
+} from "./config";
+import { convertToCoreFileContents, IGristCoreConfigFileLatest } from "./configCoreFileFormats";
+
+export type Edition = "core" | "enterprise";
+
+/**
+ * Config options for Grist Core.
+ */
+export interface IGristCoreConfig {
+  edition: IWritableConfigValue<Edition>;
+}
+
+export async function loadGristCoreConfigFile(configPath?: string): Promise<IGristCoreConfig> {
+  const fileConfig = configPath ? await FileConfig.create(configPath, convertToCoreFileContents) : undefined;
+  return loadGristCoreConfig(fileConfig);
+}
+
+export function loadGristCoreConfig(fileConfig?: FileConfig<IGristCoreConfigFileLatest>): IGristCoreConfig {
+  const fileConfigValue = fileConfigAccessorFactory(fileConfig);
+  return {
+    edition: createConfigValue<Edition>("core", fileConfigValue("edition"))
+  };
+}

app/server/lib/configCoreFileFormats-ti.ts

@@ -0,0 +1,23 @@
+/**
+ * This module was automatically generated by `ts-interface-builder`
+ */
+import * as t from "ts-interface-checker";
+// tslint:disable:object-literal-key-quotes
+
+export const IGristCoreConfigFileLatest = t.name("IGristCoreConfigFileV1");
+
+export const IGristCoreConfigFileV1 = t.iface([], {
+  "version": t.lit("1"),
+  "edition": t.opt(t.union(t.lit("core"), t.lit("enterprise"))),
+});
+
+export const IGristCoreConfigFileV0 = t.iface([], {
+  "version": "undefined",
+});
+
+const exportedTypeSuite: t.ITypeSuite = {
+  IGristCoreConfigFileLatest,
+  IGristCoreConfigFileV1,
+  IGristCoreConfigFileV0,
+};
+export default exportedTypeSuite;

app/server/lib/configCoreFileFormats.ts

@@ -0,0 +1,53 @@
+import configCoreTI from './configCoreFileFormats-ti';
+import { CheckerT, createCheckers } from "ts-interface-checker";
+
+/**
+ * Latest core config file format
+ */
+export type IGristCoreConfigFileLatest = IGristCoreConfigFileV1;
+
+/**
+ * Format of config files on disk - V1
+ */
+export interface IGristCoreConfigFileV1 {
+  version: "1"
+  edition?: "core" | "enterprise"
+}
+
+/**
+ * Format of config files on disk - V0
+ */
+export interface IGristCoreConfigFileV0 {
+  version: undefined;
+}
+
+export const checkers = createCheckers(configCoreTI) as
+  {
+    IGristCoreConfigFileV0: CheckerT<IGristCoreConfigFileV0>,
+    IGristCoreConfigFileV1: CheckerT<IGristCoreConfigFileV1>,
+    IGristCoreConfigFileLatest: CheckerT<IGristCoreConfigFileLatest>,
+  };
+
+function upgradeV0toV1(config: IGristCoreConfigFileV0): IGristCoreConfigFileV1 {
+  return {
+    ...config,
+    version: "1",
+  };
+}
+
+export function convertToCoreFileContents(input: any): IGristCoreConfigFileLatest | null {
+  if (!(input instanceof Object)) {
+    return null;
+  }
+
+  let configObject = { ...input };
+
+  if (checkers.IGristCoreConfigFileV0.test(configObject)) {
+    configObject = upgradeV0toV1(configObject);
+  }
+
+  // This will throw an exception if the config object is still not in the correct format.
+  checkers.IGristCoreConfigFileLatest.check(configObject);
+
+  return configObject;
+}

app/server/lib/places.ts

@@ -63,3 +63,10 @@ export function getAppRootFor(appRoot: string, subdirectory: string): string {
 export function getAppPathTo(appRoot: string, subdirectory: string): string {
   return path.resolve(getAppRootFor(appRoot, subdirectory), subdirectory);
 }
+
+/**
+ * Returns the instance root. Defaults to appRoot, unless overridden by GRIST_INST_DIR.
+ */
+export function getInstanceRoot() {
+  return path.resolve(process.env.GRIST_INST_DIR || getAppRoot());
+}

app/server/mergedServerMain.ts

@@ -8,6 +8,7 @@
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
 import {GristLoginSystem} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
+import {getGlobalConfig} from "app/server/lib/globalConfig";
 
 // Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
 // environment variable.
@@ -70,6 +71,8 @@ export async function main(port: number, serverTypes: ServerType[],
   const includeStatic = serverTypes.includes("static");
   const includeApp = serverTypes.includes("app");
 
+  options.settings ??= await getGlobalConfig();
+
   const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
 
   // We need to know early on whether we will be serving plugins or not.
@@ -94,7 +97,7 @@ export async function main(port: number, serverTypes: ServerType[],
 
   if (options.logToConsole !== false) { server.addLogging(); }
   if (options.externalStorage === false) { server.disableExternalStorage(); }
-  await server.loadConfig();
+  await server.addLoginMiddleware();
 
   if (includeDocs) {
     // It is important that /dw and /v prefixes are accepted (if present) by health check
@@ -195,12 +198,14 @@ export async function main(port: number, serverTypes: ServerType[],
 
 export async function startMain() {
   try {
+
     const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
 
     // No defaults for a port, since this server can serve very different purposes.
     if (!process.env.GRIST_PORT) {
       throw new Error("GRIST_PORT must be specified");
     }
+
     const port = parseInt(process.env.GRIST_PORT, 10);
 
     const server = await main(port, serverTypes);

buildtools/.grist-ee-version

@@ -1 +1 @@
-0.9.2
+0.9.4

buildtools/checkout-ext-directory.sh

@@ -14,5 +14,6 @@ pushd $repo
 git sparse-checkout set ext
 git checkout
 popd
+rm -rf ./ext
 mv $repo/ext .
 rm -rf $repo

ext/README.md

@@ -0,0 +1,5 @@
+`ext` is a directory that allows derivatives of Grist core to be created, without modifying any of the base files.
+
+Files placed in here should be new files, or replacing files in the `stubs` directory.
+
+When compiling, Typescript resolves files in `ext` before files in `stubs`, using the `ext` file instead (if it exists).

stubs/app/server/lib/globalConfig.ts

@@ -0,0 +1,19 @@
+import path from "path";
+import { getInstanceRoot } from "app/server/lib/places";
+import { IGristCoreConfig, loadGristCoreConfigFile } from "app/server/lib/configCore";
+import log from "app/server/lib/log";
+
+const globalConfigPath: string = path.join(getInstanceRoot(), 'config.json');
+let cachedGlobalConfig: IGristCoreConfig | undefined = undefined;
+
+/**
+ * Retrieves the cached grist config, or loads it from the default global path.
+ */
+export async function getGlobalConfig(): Promise<IGristCoreConfig> {
+  if (!cachedGlobalConfig) {
+    log.info(`Loading config file from ${globalConfigPath}`);
+    cachedGlobalConfig = await loadGristCoreConfigFile(globalConfigPath);
+  }
+
+  return cachedGlobalConfig;
+}

test/gen-server/seed.ts

@@ -605,7 +605,7 @@ export async function createServer(port: number, initDb = createInitialDb): Prom
   await flexServer.start();
   await flexServer.initHomeDBManager();
   flexServer.addDocWorkerMap();
-  await flexServer.loadConfig();
+  await flexServer.addLoginMiddleware();
   flexServer.addHosts();
   flexServer.addAccessMiddleware();
   flexServer.addApiMiddleware();

test/server/lib/Authorizer.ts

@@ -17,13 +17,13 @@ let server: FlexServer;
 let dbManager: HomeDBManager;
 
 async function activateServer(home: FlexServer, docManager: DocManager) {
-  await home.loadConfig();
+  await home.addLoginMiddleware();
   await home.initHomeDBManager();
   home.addHosts();
   home.addDocWorkerMap();
   home.addAccessMiddleware();
   dbManager = home.getHomeDBManager();
-  await home.loadConfig();
+  await home.addLoginMiddleware();
   home.addSessions();
   home.addHealthCheck();
   docManager.testSetHomeDbManager(dbManager);

test/server/lib/config.ts

@@ -0,0 +1,107 @@
+import { assert } from 'chai';
+import * as sinon from 'sinon';
+import { ConfigAccessors, createConfigValue, Deps, FileConfig } from "app/server/lib/config";
+
+interface TestFileContents {
+  myNum?: number
+  myStr?: string
+}
+
+const testFileContentsExample: TestFileContents = {
+  myNum: 1,
+  myStr: "myStr",
+};
+
+const testFileContentsJSON = JSON.stringify(testFileContentsExample);
+
+describe('FileConfig', () => {
+  const useFakeConfigFile = (contents: string) => {
+    const fakeFile = { contents };
+    sinon.replace(Deps, 'pathExists', sinon.fake.resolves(true));
+    sinon.replace(Deps, 'readFile', sinon.fake((path, encoding: string) => Promise.resolve(fakeFile.contents)) as any);
+    sinon.replace(Deps, 'writeFile', sinon.fake((path, newContents) => {
+      fakeFile.contents = newContents;
+      return Promise.resolve();
+    }));
+
+    return fakeFile;
+  };
+
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('throws an error from create if the validator does not return a value', async () => {
+    useFakeConfigFile(testFileContentsJSON);
+    const validator = () => null;
+    await assert.isRejected(FileConfig.create<TestFileContents>("anypath.json", validator));
+  });
+
+  it('persists changes when values are assigned', async () => {
+    const fakeFile = useFakeConfigFile(testFileContentsJSON);
+    // Don't validate - this is guaranteed to be valid above.
+    const validator = (input: any) => input as TestFileContents;
+    const fileConfig = await FileConfig.create("anypath.json", validator);
+    await fileConfig.set("myNum", 999);
+
+    assert.equal(fileConfig.get("myNum"), 999);
+    assert.equal(JSON.parse(fakeFile.contents).myNum, 999);
+  });
+
+  // Avoid removing extra properties from the file, in case another edition of grist is doing something.
+  it('does not remove extra values from the file', async () => {
+    const configWithExtraProperties = {
+      ...testFileContentsExample,
+      someProperty: "isPresent",
+    };
+
+    const fakeFile = useFakeConfigFile(JSON.stringify(configWithExtraProperties));
+    // It's entirely possible the validator can damage the extra properties, but that's not in scope for this test.
+    const validator = (input: any) => input as TestFileContents;
+    const fileConfig = await FileConfig.create("anypath.json", validator);
+    // Triggering a write to the file
+    await fileConfig.set("myNum", 999);
+    await fileConfig.set("myStr", "Something");
+
+    const newContents = JSON.parse(fakeFile.contents);
+    assert.equal(newContents.myNum, 999);
+    assert.equal(newContents.myStr, "Something");
+    assert.equal(newContents.someProperty, "isPresent");
+  });
+});
+
+describe('createConfigValue', () => {
+  const makeInMemoryAccessors = <T>(initialValue: T): ConfigAccessors<T> => {
+    let value: T = initialValue;
+    return {
+      get: () => value,
+      set: async (newValue: T) => { value = newValue; },
+    };
+  };
+
+  it('works without persistence', async () => {
+    const configValue = createConfigValue(1);
+    assert.equal(configValue.get(), 1);
+    await configValue.set(2);
+    assert.equal(configValue.get(), 2);
+  });
+
+  it('writes to persistence when saved', async () => {
+    const accessors = makeInMemoryAccessors(1);
+    const configValue = createConfigValue(1, accessors);
+    assert.equal(accessors.get(), 1);
+    await configValue.set(2);
+    assert.equal(accessors.get(), 2);
+  });
+
+  it('initialises with the persistent value if available', async () => {
+    const accessors = makeInMemoryAccessors(22);
+    const configValue = createConfigValue(1, accessors);
+    assert.equal(configValue.get(), 22);
+
+    const accessorsWithUndefinedValue = makeInMemoryAccessors<number | undefined>(undefined);
+    const configValueWithDefault = createConfigValue(333, accessorsWithUndefinedValue);
+    assert.equal(configValueWithDefault.get(), 333);
+  });
+});
+

test/server/lib/configCore.ts

@@ -0,0 +1,48 @@
+import * as sinon from 'sinon';
+import { assert } from 'chai';
+import { IGristCoreConfig, loadGristCoreConfig, loadGristCoreConfigFile } from "app/server/lib/configCore";
+import { createConfigValue, Deps, IWritableConfigValue } from "app/server/lib/config";
+
+describe('loadGristCoreConfig', () => {
+  afterEach(() => {
+    sinon.restore();
+  });
+
+  it('can be used with an in-memory store if no file config is provided', async () => {
+    const config = loadGristCoreConfig();
+    await config.edition.set("enterprise");
+    assert.equal(config.edition.get(), "enterprise");
+  });
+
+  it('will function correctly when no config file is present', async () => {
+    sinon.replace(Deps, 'pathExists', sinon.fake.resolves(false));
+    sinon.replace(Deps, 'readFile', sinon.fake.resolves(""));
+    const writeFileFake = sinon.fake.resolves(undefined);
+    sinon.replace(Deps, 'writeFile', writeFileFake);
+
+    const config = await loadGristCoreConfigFile("doesntmatter.json");
+    assert.exists(config.edition.get());
+
+    await config.edition.set("enterprise");
+    // Make sure that the change was written back to the file.
+    assert.isTrue(writeFileFake.calledOnce);
+  });
+
+  it('can be extended', async () => {
+    // Extend the core config
+    type NewConfig = IGristCoreConfig & {
+      newThing: IWritableConfigValue<number>
+    };
+
+    const coreConfig = loadGristCoreConfig();
+
+    const newConfig: NewConfig = {
+      ...coreConfig,
+      newThing: createConfigValue(3)
+    };
+
+    // Ensure that it's backwards compatible.
+    const gristConfig: IGristCoreConfig = newConfig;
+    return gristConfig;
+  });
+});

test/server/lib/configCoreFileFormats.ts

@@ -0,0 +1,29 @@
+import { assert } from 'chai';
+import { convertToCoreFileContents, IGristCoreConfigFileLatest } from "app/server/lib/configCoreFileFormats";
+
+describe('convertToCoreFileContents', () => {
+  it('fails with a malformed config', async () => {
+    const badConfig = {
+      version: "This is a random version number that will never exist",
+    };
+
+    assert.throws(() => convertToCoreFileContents(badConfig));
+  });
+
+  // This is necessary to handle users who don't have a config file yet.
+  it('will upgrade an empty object to a valid config', () => {
+    const validConfig = convertToCoreFileContents({});
+    assert.exists(validConfig?.version);
+  });
+
+  it('will validate the latest config file format', () => {
+    const validRawObject: IGristCoreConfigFileLatest = {
+      version: "1",
+      edition: "enterprise",
+    };
+
+    const validConfig = convertToCoreFileContents(validRawObject);
+    assert.exists(validConfig?.version);
+    assert.exists(validConfig?.edition);
+  });
+});