make a /boot/GRIST_BOOT_KEY page for diagnosing configuration problems (#850)

This is a start at a page for diagnosing problems while setting up Grist. Starting to add some diagnostics based on feedback in github issues. We should make Grist installation easier! But when there is a problem it should be easier to diagnose than it is now, and this may help. The page is ugly and doesn't have many diagnostics yet, but we can iterate.

Visit `/boot` on a Grist server for tips on how to use this feature.

app/server/lib/BootProbes.ts

@@ -0,0 +1,185 @@
+import { ApiError } from 'app/common/ApiError';
+import { BootProbeIds, BootProbeResult } from 'app/common/BootProbe';
+import { removeTrailingSlash } from 'app/common/gutil';
+import { expressWrap, jsonErrorHandler } from 'app/server/lib/expressWrap';
+import { GristServer } from 'app/server/lib/GristServer';
+import * as express from 'express';
+import fetch from 'node-fetch';
+
+/**
+ * Self-diagnostics useful when installing Grist.
+ */
+export class BootProbes {
+  // List of probes.
+  public _probes = new Array<Probe>();
+
+  // Probes indexed by id.
+  public _probeById = new Map<string, Probe>();
+
+  public constructor(private _app: express.Application,
+                     private _server: GristServer,
+                     private _base: string) {
+    this._addProbes();
+  }
+
+  public addEndpoints() {
+    // Return a list of available probes.
+    this._app.use(`${this._base}/probe$`, expressWrap(async (_, res) => {
+      res.json({
+        'probes': this._probes.map(probe => {
+          return { id: probe.id, name: probe.name };
+        }),
+      });
+    }));
+
+    // Return result of running an individual probe.
+    this._app.use(`${this._base}/probe/:probeId`, expressWrap(async (req, res) => {
+      const probe = this._probeById.get(req.params.probeId);
+      if (!probe) {
+        throw new ApiError('unknown probe', 400);
+      }
+      const result = await probe.apply(this._server, req);
+      res.json(result);
+    }));
+
+    // Fall-back for errors.
+    this._app.use(`${this._base}/probe`, jsonErrorHandler);
+  }
+
+  private _addProbes() {
+    this._probes.push(_homeUrlReachableProbe);
+    this._probes.push(_statusCheckProbe);
+    this._probes.push(_userProbe);
+    this._probes.push(_bootProbe);
+    this._probes.push(_hostHeaderProbe);
+    this._probeById = new Map(this._probes.map(p => [p.id, p]));
+  }
+}
+
+/**
+ * An individual probe has an id, a name, an optional description,
+ * and a method that returns a probe result.
+ */
+export interface Probe {
+  id: BootProbeIds;
+  name: string;
+  description?: string;
+  apply: (server: GristServer, req: express.Request) => Promise<BootProbeResult>;
+}
+
+const _homeUrlReachableProbe: Probe = {
+  id: 'reachable',
+  name: 'Grist is reachable',
+  apply: async (server, req) => {
+    const url = server.getHomeUrl(req);
+    try {
+      const resp = await fetch(url);
+      if (resp.status !== 200) {
+        throw new ApiError(await resp.text(), resp.status);
+      }
+      return {
+        success: true,
+      };
+    } catch (e) {
+      return {
+        success: false,
+        details: {
+          error: String(e),
+        },
+        severity: 'fault',
+      };
+    }
+  }
+};
+
+const _statusCheckProbe: Probe = {
+  id: 'health-check',
+  name: 'Built-in Health check',
+  apply: async (server, req) => {
+    const baseUrl = server.getHomeUrl(req);
+    const url = new URL(baseUrl);
+    url.pathname = removeTrailingSlash(url.pathname) + '/status';
+    try {
+      const resp = await fetch(url);
+      if (resp.status !== 200) {
+        throw new Error(`Failed with status ${resp.status}`);
+      }
+      const txt = await resp.text();
+      if (!txt.includes('is alive')) {
+        throw new Error(`Failed, page has unexpected content`);
+      }
+      return {
+        success: true,
+      };
+    } catch (e) {
+      return {
+        success: false,
+        error: String(e),
+        severity: 'fault',
+      };
+    }
+  },
+};
+
+const _userProbe: Probe = {
+  id: 'system-user',
+  name: 'System user is sane',
+  apply: async () => {
+    if (process.getuid && process.getuid() === 0) {
+      return {
+        success: false,
+        verdict: 'User appears to be root (UID 0)',
+        severity: 'warning',
+      };
+    } else {
+      return {
+        success: true,
+      };
+    }
+  },
+};
+
+const _bootProbe: Probe = {
+  id: 'boot-page',
+  name: 'Boot page exposure',
+  apply: async (server) => {
+    if (!server.hasBoot) {
+      return { success: true };
+    }
+    const maybeSecureEnough = String(process.env.GRIST_BOOT_KEY).length > 10;
+    return {
+      success: maybeSecureEnough,
+      severity: 'hmm',
+    };
+  },
+};
+
+/**
+ * Based on:
+ * https://github.com/gristlabs/grist-core/issues/228#issuecomment-1803304438
+ *
+ * When GRIST_SERVE_SAME_ORIGIN is set, requests arriving to Grist need
+ * to have an accurate Host header.
+ */
+const _hostHeaderProbe: Probe = {
+  id: 'host-header',
+  name: 'Host header is sane',
+  apply: async (server, req) => {
+    const host = req.header('host');
+    const url = new URL(server.getHomeUrl(req));
+    if (url.hostname === 'localhost') {
+      return {
+        done: true,
+      };
+    }
+    if (String(url.hostname).toLowerCase() !== String(host).toLowerCase()) {
+      return {
+        success: false,
+        severity: 'hmm',
+      };
+    }
+    return {
+      done: true,
+    };
+  },
+};

app/server/lib/FlexServer.ts

@@ -28,6 +28,7 @@ import {appSettings} from 'app/server/lib/AppSettings';
 import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser,
         isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
 import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
+import {BootProbes} from 'app/server/lib/BootProbes';
 import {forceSessionChange} from 'app/server/lib/BrowserSession';
 import {Comm} from 'app/server/lib/Comm';
 import {create} from 'app/server/lib/create';
@@ -175,6 +176,7 @@ export class FlexServer implements GristServer {
   private _getLoginSystem?: () => Promise<GristLoginSystem>;
   // Set once ready() is called
   private _isReady: boolean = false;
+  private _probes: BootProbes;
 
   constructor(public port: number, public name: string = 'flexServer',
               public readonly options: FlexServerOptions = {}) {
@@ -481,6 +483,57 @@ export class FlexServer implements GristServer {
     });
   }
 
+  /**
+   *
+   * Adds a /boot/$GRIST_BOOT_KEY page that shows diagnostics.
+   * Accepts any /boot/... URL in order to let the front end
+   * give some guidance if the user is stumbling around trying
+   * to find the boot page, but won't actually provide diagnostics
+   * unless GRIST_BOOT_KEY is set in the environment, and is present
+   * in the URL.
+   *
+   * We take some steps to make the boot page available even when
+   * things are going wrong, and should take more in future.
+   *
+   * When rendering the page a hardcoded 'boot' tag is used, which
+   * is used to ensure that static assets are served locally and
+   * we aren't relying on APP_STATIC_URL being set correctly.
+   *
+   * We use a boot key so that it is more acceptable to have this
+   * boot page living outside of the authentication system, which
+   * could be broken.
+   *
+   * TODO: there are some configuration problems that currently
+   * result in Grist not running at all. ideally they would result in
+   * Grist running in a limited mode that is enough to bring up the boot
+   * page.
+   *
+   */
+  public addBootPage() {
+    if (this._check('boot')) { return; }
+    const bootKey = appSettings.section('boot').flag('key').readString({
+      envVar: 'GRIST_BOOT_KEY'
+    });
+    const base = `/boot/${bootKey}`;
+    this._probes = new BootProbes(this.app, this, base);
+    // Respond to /boot, /boot/, /boot/KEY, /boot/KEY/ to give
+    // a helpful message even if user gets KEY wrong or omits it.
+    this.app.get('/boot(/(:bootKey/?)?)?$', async (req, res) => {
+      const goodKey = bootKey && req.params.bootKey === bootKey;
+      return this._sendAppPage(req, res, {
+        path: 'boot.html', status: 200, config: goodKey ? {
+        } : {
+          errMessage: 'not-the-key',
+        }, tag: 'boot',
+      });
+    });
+    this._probes.addEndpoints();
+  }
+
+  public hasBoot(): boolean {
+    return Boolean(this._probes);
+  }
+
   public denyRequestsIfNotReady() {
     this.app.use((_req, res, next) => {
       if (!this._isReady) {

app/server/lib/GristServer.ts

@@ -60,6 +60,7 @@ export interface GristServer {
   getPlugins(): LocalPlugin[];
   servesPlugins(): boolean;
   getBundledWidgets(): ICustomWidget[];
+  hasBoot(): boolean;
 }
 
 export interface GristLoginSystem {
@@ -147,6 +148,7 @@ export function createDummyGristServer(): GristServer {
     servesPlugins() { return false; },
     getPlugins() { return []; },
     getBundledWidgets() { return []; },
+    hasBoot() { return false; },
   };
 }
 

app/server/lib/sendAppPage.ts

@@ -139,8 +139,11 @@ export function makeSendAppPage(opts: {
     const needTagManager = (options.googleTagManager === 'anon' && isAnonymousUser(req)) ||
       options.googleTagManager === true;
     const tagManagerSnippet = needTagManager ? getTagManagerSnippet(process.env.GOOGLE_TAG_MANAGER_ID) : '';
-    const staticOrigin = process.env.APP_STATIC_URL || "";
-    const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
+    const staticTag = options.tag || tag;
+    // If boot tag is used, serve assets locally, otherwise respect
+    // APP_STATIC_URL.
+    const staticOrigin = staticTag === 'boot' ? '' : (process.env.APP_STATIC_URL || '');
+    const staticBaseUrl = `${staticOrigin}/v/${staticTag}/`;
     const customHeadHtmlSnippet = server.create.getExtraHeadHtml?.() ?? "";
     const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
     // Preload all languages that will be used or are requested by client.

app/server/mergedServerMain.ts

@@ -104,6 +104,9 @@ export async function main(port: number, serverTypes: ServerType[],
   }
 
   server.addHealthCheck();
+  if (includeHome || includeApp) {
+    server.addBootPage();
+  }
   server.denyRequestsIfNotReady();
 
   if (includeHome || includeStatic || includeApp) {