(core) updates from grist-core

app/server/lib/FlexServer.ts

@@ -54,7 +54,7 @@ import {InstallAdmin} from 'app/server/lib/InstallAdmin';
 import log from 'app/server/lib/log';
 import {getLoginSystem} from 'app/server/lib/logins';
 import {IPermitStore} from 'app/server/lib/Permit';
-import {getAppPathTo, getAppRoot, getUnpackedAppRoot} from 'app/server/lib/places';
+import {getAppPathTo, getAppRoot, getInstanceRoot, getUnpackedAppRoot} from 'app/server/lib/places';
 import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint';
 import {PluginManager} from 'app/server/lib/PluginManager';
 import * as ProcessMonitor from 'app/server/lib/ProcessMonitor';
@@ -87,6 +87,7 @@ import {AddressInfo} from 'net';
 import fetch from 'node-fetch';
 import * as path from 'path';
 import * as serveStatic from "serve-static";
+import {IGristCoreConfig} from "./configCore";
 
 // Health checks are a little noisy in the logs, so we don't show them all.
 // We show the first N health checks:
@@ -105,6 +106,9 @@ export interface FlexServerOptions {
   baseDomain?: string;
   // Base URL for plugins, if permitted. Defaults to APP_UNTRUSTED_URL.
   pluginUrl?: string;
+
+  // Global grist config options
+  settings?: IGristCoreConfig;
 }
 
 const noop: express.RequestHandler = (req, res, next) => next();
@@ -122,7 +126,7 @@ export class FlexServer implements GristServer {
   public housekeeper: Housekeeper;
   public server: http.Server;
   public httpsServer?: https.Server;
-  public settings?: Readonly<Record<string, unknown>>;
+  public settings?: IGristCoreConfig;
   public worker: DocWorkerInfo;
   public electronServerMethods: ElectronServerMethods;
   public readonly docsRoot: string;
@@ -186,6 +190,7 @@ export class FlexServer implements GristServer {
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
+    this.settings = options.settings;
     this.app = express();
     this.app.set('port', port);
 
@@ -662,7 +667,7 @@ export class FlexServer implements GristServer {
 
   public get instanceRoot() {
     if (!this._instanceRoot) {
-      this._instanceRoot = path.resolve(process.env.GRIST_INST_DIR || this.appRoot);
+      this._instanceRoot = getInstanceRoot();
       this.info.push(['instanceRoot', this._instanceRoot]);
     }
     return this._instanceRoot;
@@ -774,7 +779,7 @@ export class FlexServer implements GristServer {
   // Set up the main express middleware used.  For a single user setup, without logins,
   // all this middleware is currently a no-op.
   public addAccessMiddleware() {
-    if (this._check('middleware', 'map', 'config', isSingleUserMode() ? null : 'hosts')) { return; }
+    if (this._check('middleware', 'map', 'loginMiddleware', isSingleUserMode() ? null : 'hosts')) { return; }
 
     if (!isSingleUserMode()) {
       const skipSession = appSettings.section('login').flag('skipSession').readBool({
@@ -938,7 +943,7 @@ export class FlexServer implements GristServer {
   }
 
   public addSessions() {
-    if (this._check('sessions', 'config')) { return; }
+    if (this._check('sessions', 'loginMiddleware')) { return; }
     this.addTagChecker();
     this.addOrg();
 
@@ -1135,25 +1140,8 @@ export class FlexServer implements GristServer {
     });
   }
 
-  /**
-   * Load user config file from standard location (if present).
-   *
-   * Note that the user config file doesn't do anything today, but may be useful in
-   * the future for configuring things that don't fit well into environment variables.
-   *
-   * TODO: Revisit this, and update `GristServer.settings` type to match the expected shape
-   * of config.json. (ts-interface-checker could be useful here for runtime validation.)
-   */
-  public async loadConfig() {
-    if (this._check('config')) { return; }
-    const settingsPath = path.join(this.instanceRoot, 'config.json');
-    if (await fse.pathExists(settingsPath)) {
-      log.info(`Loading config from ${settingsPath}`);
-      this.settings = JSON.parse(await fse.readFile(settingsPath, 'utf8'));
-    } else {
-      log.info(`Loading empty config because ${settingsPath} missing`);
-      this.settings = {};
-    }
+  public async addLoginMiddleware() {
+    if (this._check('loginMiddleware')) { return; }
 
     // TODO: We could include a third mock provider of login/logout URLs for better tests. Or we
     // could create a mock SAML identity provider for testing this using the SAML flow.
@@ -1169,9 +1157,9 @@ export class FlexServer implements GristServer {
   }
 
   public addComm() {
-    if (this._check('comm', 'start', 'homedb', 'config')) { return; }
+    if (this._check('comm', 'start', 'homedb', 'loginMiddleware')) { return; }
     this._comm = new Comm(this.server, {
-      settings: this.settings,
+      settings: {},
       sessions: this._sessions,
       hosts: this._hosts,
       loginMiddleware: this._loginMiddleware,
@@ -1311,7 +1299,7 @@ export class FlexServer implements GristServer {
       null : 'homedb', 'api-mw', 'map', 'telemetry');
     // add handlers for cleanup, if we are in charge of the doc manager.
     if (!this._docManager) { this.addCleanup(); }
-    await this.loadConfig();
+    await this.addLoginMiddleware();
     this.addComm();
 
     await this.create.configure?.();

app/server/lib/GristServer.ts

@@ -25,6 +25,7 @@ import { Sessions } from 'app/server/lib/Sessions';
 import { ITelemetry } from 'app/server/lib/Telemetry';
 import * as express from 'express';
 import { IncomingMessage } from 'http';
+import { IGristCoreConfig, loadGristCoreConfig } from "./configCore";
 
 /**
  * Basic information about a Grist server.  Accessible in many
@@ -32,7 +33,7 @@ import { IncomingMessage } from 'http';
  */
 export interface GristServer {
   readonly create: ICreate;
-  settings?: Readonly<Record<string, unknown>>;
+  settings?: IGristCoreConfig;
   getHost(): string;
   getHomeUrl(req: express.Request, relPath?: string): string;
   getHomeInternalUrl(relPath?: string): string;
@@ -126,7 +127,7 @@ export interface DocTemplate {
 export function createDummyGristServer(): GristServer {
   return {
     create,
-    settings: {},
+    settings: loadGristCoreConfig(),
     getHost() { return 'localhost:4242'; },
     getHomeUrl() { return 'http://localhost:4242'; },
     getHomeInternalUrl() { return 'http://localhost:4242'; },

app/server/lib/config.ts

@@ -0,0 +1,143 @@
+import * as fse from "fs-extra";
+
+// Export dependencies for stubbing in tests.
+export const Deps = {
+  readFile: fse.readFile,
+  writeFile: fse.writeFile,
+  pathExists: fse.pathExists,
+};
+
+/**
+ * Readonly config value - no write access.
+ */
+export interface IReadableConfigValue<T> {
+  get(): T;
+}
+
+/**
+ * Writeable config value. Write behaviour is asynchronous and defined by the implementation.
+ */
+export interface IWritableConfigValue<T> extends IReadableConfigValue<T> {
+  set(value: T): Promise<void>;
+}
+
+type FileContentsValidator<T> = (value: any) => T | null;
+
+export class MissingConfigFileError extends Error {
+  public name: string = "MissingConfigFileError";
+
+  constructor(message: string) {
+    super(message);
+  }
+}
+
+export class ConfigValidationError extends Error {
+  public name: string = "ConfigValidationError";
+
+  constructor(message: string) {
+    super(message);
+  }
+}
+
+export interface ConfigAccessors<ValueType> {
+  get: () => ValueType,
+  set?: (value: ValueType) => Promise<void>
+}
+
+/**
+ * Provides type safe access to an underlying JSON file.
+ *
+ * Multiple FileConfigs for the same file shouldn't be used, as they risk going out of sync.
+ */
+export class FileConfig<FileContents> {
+  /**
+   * Creates a new type-safe FileConfig, by loading and checking the contents of the file with `validator`.
+   * @param configPath - Path to load.
+   * @param validator - Validates the contents are in the correct format, and converts to the correct type.
+   *  Should throw an error or return null if not valid.
+   */
+  public static async create<CreateConfigFileContents>(
+    configPath: string,
+    validator: FileContentsValidator<CreateConfigFileContents>
+  ): Promise<FileConfig<CreateConfigFileContents>> {
+    // Start with empty object, as it can be upgraded to a full config.
+    let rawFileContents: any = {};
+
+    if (await Deps.pathExists(configPath)) {
+      rawFileContents = JSON.parse(await Deps.readFile(configPath, 'utf8'));
+    }
+
+    let fileContents = null;
+
+    try {
+      fileContents = validator(rawFileContents);
+    } catch (error) {
+      const configError =
+        new ConfigValidationError(`Config at ${configPath} failed validation: ${error.message}`);
+      configError.cause = error;
+      throw configError;
+    }
+
+    if (!fileContents) {
+      throw new ConfigValidationError(`Config at ${configPath} failed validation - check the format?`);
+    }
+
+    return new FileConfig<CreateConfigFileContents>(configPath, fileContents);
+  }
+
+  constructor(private _filePath: string, private _rawConfig: FileContents) {
+  }
+
+  public get<Key extends keyof FileContents>(key: Key): FileContents[Key] {
+    return this._rawConfig[key];
+  }
+
+  public async set<Key extends keyof FileContents>(key: Key, value: FileContents[Key]) {
+    this._rawConfig[key] = value;
+    await this.persistToDisk();
+  }
+
+  public async persistToDisk(): Promise<void> {
+    await Deps.writeFile(this._filePath, JSON.stringify(this._rawConfig, null, 2));
+  }
+}
+
+/**
+ * Creates a function for creating accessors for a given key.
+ * Propagates undefined values, so if no file config is available, accessors are undefined.
+ * @param fileConfig - Config to load/save values to.
+ */
+export function fileConfigAccessorFactory<FileContents>(
+  fileConfig?: FileConfig<FileContents>
+): <Key extends keyof FileContents>(key: Key) => ConfigAccessors<FileContents[Key]> | undefined
+{
+  if (!fileConfig) { return (key) => undefined; }
+  return (key) => ({
+    get: () => fileConfig.get(key),
+    set: (value) => fileConfig.set(key, value)
+  });
+}
+
+/**
+ * Creates a config value optionally backed by persistent storage.
+ * Can be used as an in-memory value without persistent storage.
+ * @param defaultValue - Value to use if no persistent value is available.
+ * @param persistence - Accessors for saving/loading persistent value.
+ */
+export function createConfigValue<ValueType>(
+  defaultValue: ValueType,
+  persistence?: ConfigAccessors<ValueType> | ConfigAccessors<ValueType | undefined>,
+): IWritableConfigValue<ValueType> {
+  let inMemoryValue = (persistence && persistence.get());
+  return {
+    get(): ValueType {
+      return inMemoryValue ?? defaultValue;
+    },
+    async set(value: ValueType) {
+      if (persistence && persistence.set) {
+        await persistence.set(value);
+      }
+      inMemoryValue = value;
+    }
+  };
+}

app/server/lib/configCore.ts

@@ -0,0 +1,28 @@
+import {
+  createConfigValue,
+  FileConfig,
+  fileConfigAccessorFactory,
+  IWritableConfigValue
+} from "./config";
+import { convertToCoreFileContents, IGristCoreConfigFileLatest } from "./configCoreFileFormats";
+
+export type Edition = "core" | "enterprise";
+
+/**
+ * Config options for Grist Core.
+ */
+export interface IGristCoreConfig {
+  edition: IWritableConfigValue<Edition>;
+}
+
+export async function loadGristCoreConfigFile(configPath?: string): Promise<IGristCoreConfig> {
+  const fileConfig = configPath ? await FileConfig.create(configPath, convertToCoreFileContents) : undefined;
+  return loadGristCoreConfig(fileConfig);
+}
+
+export function loadGristCoreConfig(fileConfig?: FileConfig<IGristCoreConfigFileLatest>): IGristCoreConfig {
+  const fileConfigValue = fileConfigAccessorFactory(fileConfig);
+  return {
+    edition: createConfigValue<Edition>("core", fileConfigValue("edition"))
+  };
+}

app/server/lib/configCoreFileFormats-ti.ts

@@ -0,0 +1,23 @@
+/**
+ * This module was automatically generated by `ts-interface-builder`
+ */
+import * as t from "ts-interface-checker";
+// tslint:disable:object-literal-key-quotes
+
+export const IGristCoreConfigFileLatest = t.name("IGristCoreConfigFileV1");
+
+export const IGristCoreConfigFileV1 = t.iface([], {
+  "version": t.lit("1"),
+  "edition": t.opt(t.union(t.lit("core"), t.lit("enterprise"))),
+});
+
+export const IGristCoreConfigFileV0 = t.iface([], {
+  "version": "undefined",
+});
+
+const exportedTypeSuite: t.ITypeSuite = {
+  IGristCoreConfigFileLatest,
+  IGristCoreConfigFileV1,
+  IGristCoreConfigFileV0,
+};
+export default exportedTypeSuite;

app/server/lib/configCoreFileFormats.ts

@@ -0,0 +1,53 @@
+import configCoreTI from './configCoreFileFormats-ti';
+import { CheckerT, createCheckers } from "ts-interface-checker";
+
+/**
+ * Latest core config file format
+ */
+export type IGristCoreConfigFileLatest = IGristCoreConfigFileV1;
+
+/**
+ * Format of config files on disk - V1
+ */
+export interface IGristCoreConfigFileV1 {
+  version: "1"
+  edition?: "core" | "enterprise"
+}
+
+/**
+ * Format of config files on disk - V0
+ */
+export interface IGristCoreConfigFileV0 {
+  version: undefined;
+}
+
+export const checkers = createCheckers(configCoreTI) as
+  {
+    IGristCoreConfigFileV0: CheckerT<IGristCoreConfigFileV0>,
+    IGristCoreConfigFileV1: CheckerT<IGristCoreConfigFileV1>,
+    IGristCoreConfigFileLatest: CheckerT<IGristCoreConfigFileLatest>,
+  };
+
+function upgradeV0toV1(config: IGristCoreConfigFileV0): IGristCoreConfigFileV1 {
+  return {
+    ...config,
+    version: "1",
+  };
+}
+
+export function convertToCoreFileContents(input: any): IGristCoreConfigFileLatest | null {
+  if (!(input instanceof Object)) {
+    return null;
+  }
+
+  let configObject = { ...input };
+
+  if (checkers.IGristCoreConfigFileV0.test(configObject)) {
+    configObject = upgradeV0toV1(configObject);
+  }
+
+  // This will throw an exception if the config object is still not in the correct format.
+  checkers.IGristCoreConfigFileLatest.check(configObject);
+
+  return configObject;
+}

app/server/lib/places.ts

@@ -63,3 +63,10 @@ export function getAppRootFor(appRoot: string, subdirectory: string): string {
 export function getAppPathTo(appRoot: string, subdirectory: string): string {
   return path.resolve(getAppRootFor(appRoot, subdirectory), subdirectory);
 }
+
+/**
+ * Returns the instance root. Defaults to appRoot, unless overridden by GRIST_INST_DIR.
+ */
+export function getInstanceRoot() {
+  return path.resolve(process.env.GRIST_INST_DIR || getAppRoot());
+}

app/server/mergedServerMain.ts

@@ -8,6 +8,7 @@
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
 import {GristLoginSystem} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
+import {getGlobalConfig} from "app/server/lib/globalConfig";
 
 // Allowed server types. We'll start one or a combination based on the value of GRIST_SERVERS
 // environment variable.
@@ -70,6 +71,8 @@ export async function main(port: number, serverTypes: ServerType[],
   const includeStatic = serverTypes.includes("static");
   const includeApp = serverTypes.includes("app");
 
+  options.settings ??= await getGlobalConfig();
+
   const server = new FlexServer(port, `server(${serverTypes.join(",")})`, options);
 
   // We need to know early on whether we will be serving plugins or not.
@@ -94,7 +97,7 @@ export async function main(port: number, serverTypes: ServerType[],
 
   if (options.logToConsole !== false) { server.addLogging(); }
   if (options.externalStorage === false) { server.disableExternalStorage(); }
-  await server.loadConfig();
+  await server.addLoginMiddleware();
 
   if (includeDocs) {
     // It is important that /dw and /v prefixes are accepted (if present) by health check
@@ -195,12 +198,14 @@ export async function main(port: number, serverTypes: ServerType[],
 
 export async function startMain() {
   try {
+
     const serverTypes = parseServerTypes(process.env.GRIST_SERVERS);
 
     // No defaults for a port, since this server can serve very different purposes.
     if (!process.env.GRIST_PORT) {
       throw new Error("GRIST_PORT must be specified");
     }
+
     const port = parseInt(process.env.GRIST_PORT, 10);
 
     const server = await main(port, serverTypes);