(core) make it easier to enable Azure storage without setting GRIST_DOCS_S3_BUCKET

Summary:
Previously, absence of `GRIST_DOCS_S3_BUCKET` was equated with absence
of external storage, but that is no longer true now that Azure is
available. Azure could be used by setting `GRIST_DOCS_S3_BUCKET`
but the alternative `GRIST_AZURE_CONTAINER` flag is friendlier.

Test Plan:
confirmed manually that Azure can be configured and
used now without `GRIST_DOCS_S3_BUCKET`

Reviewers: alexmojaki

Reviewed By: alexmojaki

Subscribers: alexmojaki

Differential Revision: https://phab.getgrist.com/D3448

app/server/lib/AppSettings.ts

@@ -0,0 +1,204 @@
+import { isAffirmative } from 'app/common/gutil';
+
+/**
+ * A bundle of settings for the application. May contain
+ * a value directly, and/or via nested settings. Also
+ * may have some information about where we looked for
+ * the value, for reporting as a diagnostic.
+ */
+export class AppSettings {
+  private _value?: JSONValue;
+  private _children?: {[key: string]: AppSettings};
+  private _info?: AppSettingQueryResult;
+
+  public constructor(public readonly name: string) {}
+
+  /* access the setting - undefined if not set */
+  public get(): JSONValue|undefined {
+    return this._value;
+  }
+
+  /* access the setting as a boolean using isAffirmative - undefined if not set */
+  public getAsBool(): boolean|undefined {
+    return (this._value !== undefined) ? isAffirmative(this._value) : undefined;
+  }
+
+  /**
+   * Try to read the setting from the environment. Even if
+   * we fail, we record information about how we tried to
+   * find the setting, so we can report on that.
+   */
+  public read(query: AppSettingQuery) {
+    this._value = undefined;
+    this._info = undefined;
+    let value = undefined;
+    let found = false;
+    const envVars = getEnvVarsFromQuery(query);
+    if (!envVars.length) {
+      throw new Error('could not find an environment variable to read');
+    }
+    let envVar = envVars[0];
+    for (const synonym of envVars) {
+      value = process.env[synonym];
+      if (value !== undefined) {
+        envVar = synonym;
+        found = true;
+        break;
+      }
+    }
+    this._info = {
+      envVar: found ? envVar : undefined,
+      found,
+      query,
+    };
+    if (value !== undefined) {
+      this._value = value;
+    } else if (query.defaultValue !== undefined) {
+      this._value = query.defaultValue;
+    }
+    return this;
+  }
+
+  /**
+   * As for read() but type the result as a string.
+   */
+  public readString(query: AppSettingQuery): string|undefined {
+    this.read(query);
+    if (this._value === undefined) { return undefined; }
+    this._value = String(this._value);
+    return this._value;
+  }
+
+  /**
+   * As for readString() but fail if nothing was found.
+   */
+  public requireString(query: AppSettingQuery): string {
+    const result = this.readString(query);
+    if (result === undefined) {
+      throw new Error(`missing environment variable: ${query.envVar}`);
+    }
+    return result;
+  }
+
+  /* set this setting 'manually' */
+  public set(value: JSONValue): void {
+    this._value = value;
+    this._info = undefined;
+  }
+
+  /* access any nested settings */
+  public get nested(): {[key: string]: AppSettings} {
+    return this._children || {};
+  }
+
+  /**
+   * Add a named nested setting, returning an AppSettings
+   * object that can be used to access it. This method is
+   * named "section" to suggest that the nested setting
+   * will itself contain multiple settings, but doesn't
+   * require that.
+   */
+  public section(fname: string): AppSettings {
+    if (!this._children) { this._children = {}; }
+    let child = this._children[fname];
+    if (!child) {
+      this._children[fname] = child = new AppSettings(fname);
+    }
+    return child;
+  }
+
+  /**
+   * Add a named nested setting, returning an AppSettings
+   * object that can be used to access it. This method is
+   * named "flag" to suggest that tthe nested setting will
+   * not iself be nested, but doesn't require that - it is
+   * currently just an alias for the section() method.
+   */
+  public flag(fname: string): AppSettings {
+    return this.section(fname);
+  }
+
+  /**
+   * Produce a summary description of the setting and how it was
+   * derived.
+   */
+  public describe(): AppSettingDescription {
+    return {
+      name: this.name,
+      value: (this._info?.query.censor && this._value !== undefined) ? '*****' : this._value,
+      foundInEnvVar: this._info?.envVar,
+      wouldFindInEnvVar: this._info?.query.preferredEnvVar || getEnvVarsFromQuery(this._info?.query)[0],
+      usedDefault: this._value !== undefined && this._info !== undefined && !this._info?.found,
+    };
+  }
+
+  /**
+   * As for describe(), but include all nested settings also.
+   * Used dotted notation for setting names. Omit settings that
+   * are undefined and without useful information about how they
+   * might be defined.
+   */
+  public describeAll(): AppSettingDescription[] {
+    const inv: AppSettingDescription[] = [];
+    inv.push(this.describe());
+    if (this._children) {
+      for (const child of Object.values(this._children)) {
+        for (const item of child.describeAll()) {
+          inv.push({...item, name: this.name + '.' + item.name});
+        }
+      }
+    }
+    return inv.filter(item => item.value !== undefined ||
+      item.wouldFindInEnvVar !== undefined ||
+      item.usedDefault);
+  }
+}
+
+/**
+ * A global object for Grist application settings.
+ */
+export const appSettings = new AppSettings('grist');
+
+/**
+ * Hints for how to define a setting, including possible
+ * environment variables and default values.
+ */
+export interface AppSettingQuery {
+  envVar: string|string[];  // environment variable(s) to check.
+  preferredEnvVar?: string; // "Canonical" environment variable to suggest.
+                            // Should be in envVar (though this is not checked).
+  defaultValue?: JSONValue; // value to use if variable(s) unavailable.
+  censor?: boolean;   // should the value of the setting be obscured when printed.
+}
+
+/**
+ * Result of a query specifying whether the setting
+ * was found, and if so in what environment variable, and using
+ * what query.
+ */
+export interface AppSettingQueryResult {
+  envVar?: string;
+  found: boolean;
+  query: AppSettingQuery;
+}
+
+/**
+ * Output of AppSettings.describe().
+ */
+interface AppSettingDescription {
+  name: string;            // name of the setting.
+  value?: JSONValue;       // value of the setting, if available.
+  foundInEnvVar?: string;  // environment variable the setting was read from, if available.
+  wouldFindInEnvVar?: string;  // environment variable that would be checked for the setting.
+  usedDefault: boolean;    // whether a default value was used for the setting.
+}
+
+// Helper function to normalize the AppSettingQuery.envVar list.
+function getEnvVarsFromQuery(q?: AppSettingQuery): string[] {
+  if (!q) { return []; }
+  return Array.isArray(q.envVar) ? q.envVar : [q.envVar];
+}
+
+// Keep app settings JSON-like, in case later we decide to load them from
+// a JSON source.
+type JSONValue = string | number | boolean | null | { [member: string]: JSONValue } | JSONValue[];

app/server/lib/DocManager.ts

@@ -527,7 +527,7 @@ export class DocManager extends EventEmitter {
     const doc = await this._getDoc(docSession, docName);
     // Get URL for document for use with SELF_HYPERLINK().
     const docUrl = doc && await this._getDocUrl(doc);
-    return this.gristServer.create.ActiveDoc(this, docName, {docUrl, safeMode, doc});
+    return new ActiveDoc(this, docName, {docUrl, safeMode, doc});
   }
 
   /**

app/server/lib/DocStorageManager.ts

@@ -42,7 +42,7 @@ export class DocStorageManager implements IDocStorageManager {
               private _comm?: Comm, gristServer?: GristServer) {
     // If we have a way to communicate with clients, watch the docsRoot for changes.
     this._watcher = null;
-    this._shell = (gristServer && gristServer.create.Shell()) || {
+    this._shell = gristServer?.create.Shell?.() || {
       moveItemToTrash()  { throw new Error('Unable to move document to trash'); },
       showItemInFolder() { throw new Error('Unable to show item in folder'); }
     };

app/server/lib/FlexServer.ts

@@ -18,6 +18,7 @@ import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {Housekeeper} from 'app/gen-server/lib/Housekeeper';
 import {Usage} from 'app/gen-server/lib/Usage';
 import {attachAppEndpoint} from 'app/server/lib/AppEndpoint';
+import {appSettings} from 'app/server/lib/AppSettings';
 import {addRequestUser, getUser, getUserId, isSingleUserMode,
         redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
 import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
@@ -127,7 +128,7 @@ export class FlexServer implements GristServer {
   private _internalPermitStore: IPermitStore;  // store for permits that stay within our servers
   private _externalPermitStore: IPermitStore;  // store for permits that pass through outside servers
   private _disabled: boolean = false;
-  private _disableS3: boolean = false;
+  private _disableExternalStorage: boolean = false;
   private _healthy: boolean = true;  // becomes false if a serious error has occurred and
                                      // server cannot do its work.
   private _healthCheckCounter: number = 0;
@@ -1006,21 +1007,21 @@ export class FlexServer implements GristServer {
     await this.loadConfig();
     this.addComm();
 
+    await this.create.configure?.();
     if (!isSingleUserMode()) {
-      if (!process.env.GRIST_DOCS_S3_BUCKET || process.env.GRIST_DISABLE_S3 === 'true') {
+      const externalStorage = appSettings.section('externalStorage');
-        this._disableS3 = true;
+      const haveExternalStorage = Object.values(externalStorage.nested)
+        .some(storage => storage.flag('active').getAsBool());
+      const disabled = externalStorage.flag('disable')
+        .read({ envVar: 'GRIST_DISABLE_S3' }).getAsBool();
+      if (disabled || !haveExternalStorage) {
+        this._disableExternalStorage = true;
+        externalStorage.flag('active').set(false);
       }
-      for (const [key, val] of Object.entries(this.create.configurationOptions())) {
-        this.info.push([key, val]);
-      }
-      if (this._disableS3) {
-        this.info.push(['s3', 'disabled']);
-      }
-
       const workers = this._docWorkerMap;
       const docWorkerId = await this._addSelfAsWorker(workers);
 
-      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableS3, workers,
+      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableExternalStorage, workers,
                                                       this._dbManager, this.create);
       this._storageManager = storageManager;
     } else {
@@ -1065,11 +1066,11 @@ export class FlexServer implements GristServer {
     }
   }
 
-  public disableS3() {
+  public disableExternalStorage() {
     if (this.deps.has('doc')) {
-      throw new Error('disableS3 called too late');
+      throw new Error('disableExternalStorage called too late');
     }
-    this._disableS3 = true;
+    this._disableExternalStorage = true;
   }
 
   public addAccountPage() {
@@ -1203,6 +1204,14 @@ export class FlexServer implements GristServer {
     for (const [label, value] of this.info) {
       log.info("== %s: %s", label, value);
     }
+    for (const item of appSettings.describeAll()) {
+      const txt =
+        ((item.value !== undefined) ? String(item.value) : '-') +
+        (item.foundInEnvVar ? ` [${item.foundInEnvVar}]` : '') +
+        (item.usedDefault ? ' [default]' : '') +
+        ((item.wouldFindInEnvVar && !item.foundInEnvVar) ? ` [${item.wouldFindInEnvVar}]` : '');
+      log.info("== %s: %s", item.name, txt);
+    }
   }
 
   public async start() {

app/server/lib/ICreate.ts

@@ -1,7 +1,5 @@
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
-import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {DocManager} from 'app/server/lib/DocManager';
 import {ExternalStorage} from 'app/server/lib/ExternalStorage';
 import {GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
@@ -15,7 +13,7 @@ export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
-  Shell(): IShell|undefined;
+  Shell?(): IShell;  // relevant to electron version of Grist only.
 
   // Create a space to store files externally, for storing either:
   //  - documents. This store should be versioned, and can be eventually consistent.
@@ -24,12 +22,11 @@ export interface ICreate {
   // should not interfere with each other.
   ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
 
-  ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc;
   NSandbox(options: ISandboxCreationOptions): ISandbox;
 
   sessionSecret(): string;
-  // Get configuration information to show at start-up.
+  // Check configuration of the app early enough to show on startup.
-  configurationOptions(): {[key: string]: any};
+  configure?(): Promise<void>;
   // Return a string containing 1 or more HTML tags to insert into the head element of every
   // static page.
   getExtraHeadHtml?(): string;
@@ -42,7 +39,7 @@ export interface ICreateActiveDocOptions {
 }
 
 export interface ICreateStorageOptions {
-  check(): Record<string, string>|undefined;
+  check(): boolean;
   create(purpose: 'doc'|'meta', extraPrefix: string): ExternalStorage|undefined;
 }
 
@@ -75,20 +72,14 @@ export function makeSimpleCreator(opts: {
         deleteUser()      { throw new Error('deleteUser unavailable'); },
       };
     },
-    Shell() {
-      return {
-        moveItemToTrash()  { throw new Error('moveToTrash unavailable'); },
-        showItemInFolder() { throw new Error('showItemInFolder unavailable'); }
-      };
-    },
     ExternalStorage(purpose, extraPrefix) {
       for (const storage of opts.storage || []) {
-        const config = storage.check();
+        if (storage.check()) {
-        if (config) { return storage.create(purpose, extraPrefix); }
+          return storage.create(purpose, extraPrefix);
+        }
       }
       return undefined;
     },
-    ActiveDoc(docManager, docName, options) { return new ActiveDoc(docManager, docName, options); },
     NSandbox(options) {
       return createSandbox('unsandboxed', options);
     },
@@ -99,12 +90,10 @@ export function makeSimpleCreator(opts: {
       }
       return secret;
     },
-    configurationOptions() {
+    async configure() {
       for (const storage of opts.storage || []) {
-        const config = storage.check();
+        if (storage.check()) { break; }
-        if (config) { return config; }
       }
-      return {};
     },
     getExtraHeadHtml() {
       let customHeadHtmlSnippet = '';

app/server/mergedServerMain.ts

@@ -34,7 +34,7 @@ interface ServerOptions extends FlexServerOptions {
   logToConsole?: boolean;  // If set, messages logged to console (default: false)
                            //   (but if options are not given at all in call to main,
                            //    logToConsole is set to true)
-  s3?: boolean;            // If set, documents saved to s3 (default is to check environment
+  externalStorage?: boolean; // If set, documents saved to external storage such as s3 (default is to check environment
                            // variables, which get set in various ways in dev/test entry points)
 }
 
@@ -59,7 +59,7 @@ export async function main(port: number, serverTypes: ServerType[],
   }
 
   if (options.logToConsole) { server.addLogging(); }
-  if (options.s3 === false) { server.disableS3(); }
+  if (options.externalStorage === false) { server.disableExternalStorage(); }
   await server.loadConfig();
 
   if (includeDocs) {