(core) updates from grist-core

app/server/lib/Authorizer.ts

@@ -193,6 +193,24 @@ export async function addRequestUser(
     }
   }
 
+  // Check if we have a boot key. This is a fallback mechanism for an
+  // administrator to authenticate themselves by demonstrating access
+  // to the environment.
+  if (!authDone && mreq.headers && mreq.headers['x-boot-key']) {
+    const reqBootKey = String(mreq.headers['x-boot-key']);
+    const bootKey = options.gristServer.getBootKey();
+    if (!bootKey || bootKey !== reqBootKey) {
+      return res.status(401).send('Bad request: invalid Boot key');
+    }
+    const userId = dbManager.getSupportUserId();
+    const user = await dbManager.getUser(userId);
+    mreq.user = user;
+    mreq.userId = userId;
+    mreq.users = [dbManager.makeFullUser(user!)];
+    mreq.userIsAuthorized = true;
+    authDone = true;
+  }
+
   // Special permission header for internal housekeeping tasks
   if (!authDone && mreq.headers && mreq.headers.permit) {
     const permitKey = String(mreq.headers.permit);

app/server/lib/BootProbes.ts

@@ -4,6 +4,7 @@ import { removeTrailingSlash } from 'app/common/gutil';
 import { expressWrap, jsonErrorHandler } from 'app/server/lib/expressWrap';
 import { GristServer } from 'app/server/lib/GristServer';
 import * as express from 'express';
+import WS from 'ws';
 import fetch from 'node-fetch';
 
 /**
@@ -25,7 +26,7 @@ export class BootProbes {
 
   public addEndpoints() {
     // Return a list of available probes.
-    this._app.use(`${this._base}/probe$`,
+    this._app.use(`${this._base}/probes$`,
                   ...this._middleware,
                   expressWrap(async (_, res) => {
       res.json({
@@ -36,7 +37,7 @@ export class BootProbes {
     }));
 
     // Return result of running an individual probe.
-    this._app.use(`${this._base}/probe/:probeId`,
+    this._app.use(`${this._base}/probes/:probeId`,
                   ...this._middleware,
                   expressWrap(async (req, res) => {
       const probe = this._probeById.get(req.params.probeId);
@@ -48,7 +49,7 @@ export class BootProbes {
     }));
 
     // Fall-back for errors.
-    this._app.use(`${this._base}/probe`, jsonErrorHandler);
+    this._app.use(`${this._base}/probes`, jsonErrorHandler);
   }
 
   private _addProbes() {
@@ -59,6 +60,7 @@ export class BootProbes {
     this._probes.push(_hostHeaderProbe);
     this._probes.push(_sandboxingProbe);
     this._probes.push(_authenticationProbe);
+    this._probes.push(_webSocketsProbe);
     this._probeById = new Map(this._probes.map(p => [p.id, p]));
   }
 }
@@ -76,38 +78,78 @@ export interface Probe {
 
 const _homeUrlReachableProbe: Probe = {
   id: 'reachable',
-  name: 'Grist is reachable',
+  name: 'Is home page available at expected URL',
   apply: async (server, req) => {
     const url = server.getHomeInternalUrl();
+    const details: Record<string, any> = {
+      url,
+    };
     try {
       const resp = await fetch(url);
+      details.status = resp.status;
       if (resp.status !== 200) {
         throw new ApiError(await resp.text(), resp.status);
       }
       return {
-        success: true,
+        status: 'success',
+        details,
       };
     } catch (e) {
       return {
-        success: false,
         details: {
+          ...details,
           error: String(e),
         },
-        severity: 'fault',
+        status: 'fault',
       };
     }
   }
 };
 
+const _webSocketsProbe: Probe = {
+  id: 'websockets',
+  name: 'Can we open a websocket with the server',
+  apply: async (server, req) => {
+    return new Promise((resolve) => {
+      const url = new URL(server.getHomeUrl(req));
+      url.protocol = (url.protocol === 'https:') ? 'wss:' : 'ws:';
+      const ws = new WS.WebSocket(url.href);
+      const details: Record<string, any> = {
+        url,
+      };
+      ws.on('open', () => {
+        ws.send('Just nod if you can hear me.');
+        resolve({
+          status: 'success',
+          details,
+        });
+        ws.close();
+      });
+      ws.on('error', (ev) => {
+        details.error = ev.message;
+        resolve({
+          status: 'fault',
+          details,
+        });
+        ws.close();
+      });
+    });
+  }
+};
+
 const _statusCheckProbe: Probe = {
   id: 'health-check',
-  name: 'Built-in Health check',
+  name: 'Is an internal health check passing',
   apply: async (server, req) => {
     const baseUrl = server.getHomeInternalUrl();
     const url = new URL(baseUrl);
     url.pathname = removeTrailingSlash(url.pathname) + '/status';
+    const details: Record<string, any> = {
+      url: url.href,
+    };
     try {
       const resp = await fetch(url);
+      details.status = resp.status;
       if (resp.status !== 200) {
         throw new Error(`Failed with status ${resp.status}`);
       }
@@ -116,13 +158,16 @@ const _statusCheckProbe: Probe = {
         throw new Error(`Failed, page has unexpected content`);
       }
       return {
-        success: true,
+        status: 'success',
+        details,
       };
     } catch (e) {
       return {
-        success: false,
-        error: String(e),
-        severity: 'fault',
+        details: {
+          ...details,
+          error: String(e),
+        },
+        status: 'fault',
       };
     }
   },
@@ -130,17 +175,21 @@ const _statusCheckProbe: Probe = {
 
 const _userProbe: Probe = {
   id: 'system-user',
-  name: 'System user is sane',
+  name: 'Is the system user following best practice',
   apply: async () => {
+    const details = {
+      uid: process.getuid ? process.getuid() : 'unavailable',
+    };
     if (process.getuid && process.getuid() === 0) {
       return {
-        success: false,
+        details,
         verdict: 'User appears to be root (UID 0)',
-        severity: 'warning',
+        status: 'warning',
       };
     } else {
       return {
-        success: true,
+        status: 'success',
+        details,
       };
     }
   },
@@ -148,15 +197,28 @@ const _userProbe: Probe = {
 
 const _bootProbe: Probe = {
   id: 'boot-page',
-  name: 'Boot page exposure',
+  name: 'Is the boot page adequately protected',
   apply: async (server) => {
-    if (!server.hasBoot) {
-      return { success: true };
+    const bootKey = server.getBootKey() || '';
+    const hasBoot = Boolean(bootKey);
+    const details: Record<string, any> = {
+      bootKeySet: hasBoot,
+    };
+    if (!hasBoot) {
+      return { status: 'success', details };
+    }
+    details.bootKeyLength = bootKey.length;
+    if (bootKey.length < 10) {
+      return {
+        verdict: 'Boot key length is shorter than 10.',
+        details,
+        status: 'fault',
+      };
     }
-    const maybeSecureEnough = String(process.env.GRIST_BOOT_KEY).length > 10;
     return {
-      success: maybeSecureEnough,
-      severity: 'hmm',
+      verdict: 'Boot key ideally should be removed after installation.',
+      details,
+      status: 'warning',
     };
   },
 };
@@ -170,35 +232,40 @@ const _bootProbe: Probe = {
  */
 const _hostHeaderProbe: Probe = {
   id: 'host-header',
-  name: 'Host header is sane',
+  name: 'Does the host header look correct',
   apply: async (server, req) => {
     const host = req.header('host');
     const url = new URL(server.getHomeUrl(req));
+    const details = {
+      homeUrlHost: url.hostname,
+      headerHost: host,
+    };
     if (url.hostname === 'localhost') {
       return {
-        done: true,
+        status: 'none',
+        details,
       };
     }
     if (String(url.hostname).toLowerCase() !== String(host).toLowerCase()) {
       return {
-        success: false,
-        severity: 'hmm',
+        details,
+        status: 'hmm',
       };
     }
     return {
-      done: true,
+      status: 'none',
+      details,
     };
   },
 };
 
-
 const _sandboxingProbe: Probe = {
   id: 'sandboxing',
-  name: 'Sandboxing is working',
+  name: 'Is document sandboxing effective',
   apply: async (server, req) => {
     const details = server.getSandboxInfo();
     return {
-      success: details?.configured && details?.functional,
+      status: (details?.configured && details?.functional) ? 'success' : 'fault',
       details,
     };
   },
@@ -210,7 +277,7 @@ const _authenticationProbe: Probe = {
   apply: async(server, req) => {
     const loginSystemId = server.getInfo('loginMiddlewareComment');
     return {
-      success: loginSystemId != undefined,
+      status: (loginSystemId != undefined) ? 'success' : 'fault',
       details: {
         loginSystemId,
       }

app/server/lib/FlexServer.ts

@@ -181,7 +181,6 @@ export class FlexServer implements GristServer {
   private _getLoginSystem?: () => Promise<GristLoginSystem>;
   // Set once ready() is called
   private _isReady: boolean = false;
-  private _probes: BootProbes;
   private _updateManager: UpdateManager;
   private _sandboxInfo: SandboxInfo;
 
@@ -558,27 +557,17 @@ export class FlexServer implements GristServer {
    */
   public addBootPage() {
     if (this._check('boot')) { return; }
-    const bootKey = appSettings.section('boot').flag('key').readString({
-      envVar: 'GRIST_BOOT_KEY'
-    });
-    const base = `/boot/${bootKey}`;
-    this._probes = new BootProbes(this.app, this, base);
-    // Respond to /boot, /boot/, /boot/KEY, /boot/KEY/ to give
-    // a helpful message even if user gets KEY wrong or omits it.
     this.app.get('/boot(/(:bootKey/?)?)?$', async (req, res) => {
-      const goodKey = bootKey && req.params.bootKey === bootKey;
-      return this._sendAppPage(req, res, {
-        path: 'boot.html', status: 200, config: goodKey ? {
-        } : {
-          errMessage: 'not-the-key',
-        }, tag: 'boot',
-      });
+      // Doing a good redirect is actually pretty subtle and we might
+      // get it wrong, so just say /boot got moved.
+      res.send('The /boot/KEY page is now /admin?boot-key=KEY');
     });
-    this._probes.addEndpoints();
   }
 
-  public hasBoot(): boolean {
-    return Boolean(this._probes);
+  public getBootKey(): string|undefined {
+    return appSettings.section('boot').flag('key').readString({
+      envVar: 'GRIST_BOOT_KEY'
+    });
   }
 
   public denyRequestsIfNotReady() {
@@ -1879,22 +1868,21 @@ export class FlexServer implements GristServer {
 
     const requireInstallAdmin = this.getInstallAdmin().getMiddlewareRequireAdmin();
 
-    const adminPageMiddleware = [
-      this._redirectToHostMiddleware,
-      this._userIdMiddleware,
-      this._redirectToLoginWithoutExceptionsMiddleware,
-      // In principle, it may be safe to show the Admin Panel to non-admins but let's protect it
-      // since it's intended for admins, and it's easier not to have to worry how it should behave
-      // for others.
-      requireInstallAdmin,
-    ];
-    this.app.get('/admin', ...adminPageMiddleware, expressWrap(async (req, resp) => {
+    // Admin endpoint needs to have very little middleware since each
+    // piece of middleware creates a new way to fail and leave the admin
+    // panel inaccessible. Generally the admin panel should report problems
+    // rather than failing entirely.
+    this.app.get('/admin', this._userIdMiddleware, expressWrap(async (req, resp) => {
       return this.sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
     }));
-    const probes = new BootProbes(this.app, this, '/admin', adminPageMiddleware);
+    const adminMiddleware = [
+      this._userIdMiddleware,
+      requireInstallAdmin,
+    ];
+    const probes = new BootProbes(this.app, this, '/api', adminMiddleware);
     probes.addEndpoints();
 
-    // Restrict this endpoint to install admins too, for the same reason as the /admin page.
+    // Restrict this endpoint to install admins
     this.app.get('/api/install/prefs', requireInstallAdmin, expressWrap(async (_req, resp) => {
       const activation = await this._activations.current();
 
@@ -1922,7 +1910,7 @@ export class FlexServer implements GristServer {
 
     // GET api/checkUpdates
     // Retrieves the latest version of the client from Grist SAAS endpoint.
-    this.app.get('/api/install/updates', adminPageMiddleware, expressWrap(async (req, res) => {
+    this.app.get('/api/install/updates', adminMiddleware, expressWrap(async (req, res) => {
       // Prepare data for the telemetry that endpoint might expect.
       const installationId = (await this.getActivations().current()).id;
       const deploymentType = this.getDeploymentType();

app/server/lib/GristServer.ts

@@ -65,7 +65,7 @@ export interface GristServer {
   getPlugins(): LocalPlugin[];
   servesPlugins(): boolean;
   getBundledWidgets(): ICustomWidget[];
-  hasBoot(): boolean;
+  getBootKey(): string|undefined;
   getSandboxInfo(): SandboxInfo|undefined;
   getInfo(key: string): any;
 }
@@ -158,7 +158,7 @@ export function createDummyGristServer(): GristServer {
     servesPlugins() { return false; },
     getPlugins() { return []; },
     getBundledWidgets() { return []; },
-    hasBoot() { return false; },
+    getBootKey() { return undefined; },
     getSandboxInfo() { return undefined; },
     getInfo(key: string) { return undefined; }
   };

app/server/lib/ICreate.ts

@@ -158,6 +158,6 @@ export function makeSimpleCreator(opts: {
     },
     getSqliteVariant: opts.getSqliteVariant,
     getSandboxVariants: opts.getSandboxVariants,
-    createInstallAdmin: opts.createInstallAdmin || (async () => new SimpleInstallAdmin()),
+    createInstallAdmin: opts.createInstallAdmin || (async (dbManager) => new SimpleInstallAdmin(dbManager)),
   };
 }

app/server/lib/InstallAdmin.ts

@@ -1,4 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {appSettings} from 'app/server/lib/AppSettings';
 import {getUser, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {User} from 'app/gen-server/entity/User';
@@ -40,13 +41,19 @@ export abstract class InstallAdmin {
 }
 
 // Considers the user whose email matches GRIST_DEFAULT_EMAIL env var, if given, to be the
-// installation admin. If not given, then there is no admin.
+// installation admin. The support user is also accepted.
+// Otherwise, there is no admin.
 export class SimpleInstallAdmin extends InstallAdmin {
   private _installAdminEmail = appSettings.section('access').flag('installAdminEmail').readString({
     envVar: 'GRIST_DEFAULT_EMAIL',
   });
 
+  public constructor(private _dbManager: HomeDBManager) {
+    super();
+  }
+
   public override async isAdminUser(user: User): Promise<boolean> {
+    if (user.id === this._dbManager.getSupportUserId()) { return true; }
     return this._installAdminEmail ? (user.loginEmail === this._installAdminEmail) : false;
   }
 }

app/server/lib/coreCreator.ts

@@ -0,0 +1,22 @@
+import { checkMinIOBucket, checkMinIOExternalStorage,
+         configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage';
+import { makeSimpleCreator } from 'app/server/lib/ICreate';
+import { Telemetry } from 'app/server/lib/Telemetry';
+
+export const makeCoreCreator = () => makeSimpleCreator({
+  deploymentType: 'core',
+  // This can and should be overridden by GRIST_SESSION_SECRET
+  // (or generated randomly per install, like grist-omnibus does).
+  sessionSecret: 'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh',
+  storage: [
+    {
+      name: 'minio',
+      check: () => checkMinIOExternalStorage() !== undefined,
+      checkBackend: () => checkMinIOBucket(),
+      create: configureMinIOExternalStorage,
+    },
+  ],
+  telemetry: {
+    create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer),
+  }
+});

app/server/lib/coreLogins.ts

@@ -0,0 +1,12 @@
+import { getForwardAuthLoginSystem } from 'app/server/lib/ForwardAuthLogin';
+import { GristLoginSystem } from 'app/server/lib/GristServer';
+import { getMinimalLoginSystem } from 'app/server/lib/MinimalLogin';
+import { getOIDCLoginSystem } from 'app/server/lib/OIDCConfig';
+import { getSamlLoginSystem } from 'app/server/lib/SamlConfig';
+
+export async function getCoreLoginSystem(): Promise<GristLoginSystem> {
+  return await getSamlLoginSystem() ||
+    await getOIDCLoginSystem() ||
+    await getForwardAuthLoginSystem() ||
+    await getMinimalLoginSystem();
+}