(core) Add installation/site configuration endpoints

Summary:
A new set of endpoints for managing installation and site configuration have been added:
 - GET `/api/install/configs/:key` - get the value of the configuration item with the specified key
 - PUT `/api/install/configs/:key` - set the value of the configuration item with the specified key
     - body: the JSON value of the configuration item
 - DELETE `/api/install/configs/:key` - delete the configuration item with the specified key
 - GET `/api/orgs/:oid/configs/:key` - get the value of the configuration item with the specified key
 - PUT `/api/orgs/:oid/configs/:key` - set the value of the configuration item with the specified key
     - body: the JSON value of the configuration item
 - DELETE `/api/orgs/:oid/configs/:key` - delete the configuration item with the specified key

Configuration consists of key/value pairs, where keys are strings (e.g. `"audit_logs_streaming_destinations"`) and values are JSON, including literals like numbers and strings. Only installation admins and site owners are permitted to modify installation and site configuration, respectively.

The endpoints are planned to be used in an upcoming feature for enabling audit log streaming for an installation and/or site. Future functionality may use the endpoints as well, which may require extending the current capabilities (e.g. adding support for storing secrets, additional metadata fields, etc.).

Test Plan: Server tests

Reviewers: paulfitz, jarek

Reviewed By: paulfitz, jarek

Subscribers: jarek

Differential Revision: https://phab.getgrist.com/D4377

app/server/MergedServer.ts

@@ -156,9 +156,12 @@ export class MergedServer {
         if (!this.hasComponent("docs")) {
           this.flexServer.addDocApiForwarder();
         }
+        await this.flexServer.addLandingPages();
+        // Early endpoints use their own json handlers, so they come before
+        // `addJsonSupport`.
+        this.flexServer.addEarlyApi();
         this.flexServer.addJsonSupport();
         this.flexServer.addUpdatesCheck();
-        await this.flexServer.addLandingPages();
         // todo: add support for home api to standalone app
         this.flexServer.addHomeApi();
         this.flexServer.addBillingApi();
@@ -172,7 +175,6 @@ export class MergedServer {
         this.flexServer.addWelcomePaths();
         this.flexServer.addLogEndpoint();
         this.flexServer.addGoogleAuthEndpoint();
-        this.flexServer.addInstallEndpoints();
         this.flexServer.addConfigEndpoints();
       }
 

app/server/lib/FlexServer.ts

@@ -1,12 +1,11 @@
 import {ApiError} from 'app/common/ApiError';
 import {ICustomWidget} from 'app/common/CustomWidget';
 import {delay} from 'app/common/delay';
-import {commonUrls, encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
+import {encodeUrl, getSlugIfNeeded, GristDeploymentType, GristDeploymentTypes,
         GristLoadConfig, IGristUrlState, isOrgInPathOnly, parseSubdomain,
         sanitizePathTail} from 'app/common/gristUrls';
 import {getOrgUrlInfo} from 'app/common/gristUrls';
 import {isAffirmative, safeJsonParse} from 'app/common/gutil';
-import {InstallProperties} from 'app/common/InstallAPI';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {SandboxInfo} from 'app/common/SandboxInfo';
 import {tbind} from 'app/common/tbind';
@@ -26,11 +25,11 @@ import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens';
 import {createSandbox} from 'app/server/lib/ActiveDoc';
 import {attachAppEndpoint} from 'app/server/lib/AppEndpoint';
 import {appSettings} from 'app/server/lib/AppSettings';
+import {attachEarlyEndpoints} from 'app/server/lib/attachEarlyEndpoints';
 import {IAuditLogger} from 'app/server/lib/AuditLogger';
 import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser,
         isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
 import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
-import {BootProbes} from 'app/server/lib/BootProbes';
 import {forceSessionChange} from 'app/server/lib/BrowserSession';
 import {Comm} from 'app/server/lib/Comm';
 import {create} from 'app/server/lib/create';
@@ -57,14 +56,14 @@ import {addPluginEndpoints, limitToPlugins} from 'app/server/lib/PluginEndpoint'
 import {PluginManager} from 'app/server/lib/PluginManager';
 import * as ProcessMonitor from 'app/server/lib/ProcessMonitor';
 import {adaptServerUrl, getOrgUrl, getOriginUrl, getScope, integerParam, isParameterOn, optIntegerParam,
-        optStringParam, RequestWithGristInfo, sendOkReply, stringArrayParam, stringParam, TEST_HTTPS_OFFSET,
+        optStringParam, RequestWithGristInfo, stringArrayParam, stringParam, TEST_HTTPS_OFFSET,
         trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions, makeGristConfig, makeMessagePage, makeSendAppPage} from 'app/server/lib/sendAppPage';
 import {getDatabaseUrl, listenPromise, timeoutReached} from 'app/server/lib/serverUtils';
 import {Sessions} from 'app/server/lib/Sessions';
 import * as shutdown from 'app/server/lib/shutdown';
 import {TagChecker} from 'app/server/lib/TagChecker';
-import {getTelemetryPrefs, ITelemetry} from 'app/server/lib/Telemetry';
+import {ITelemetry} from 'app/server/lib/Telemetry';
 import {startTestingHooks} from 'app/server/lib/TestingHooks';
 import {getTestLoginSystem} from 'app/server/lib/TestLogin';
 import {UpdateManager} from 'app/server/lib/UpdateManager';
@@ -1548,10 +1547,7 @@ export class FlexServer implements GristServer {
    * we need to get these webhooks in before the bodyParser is added to parse json.
    */
   public addEarlyWebhooks() {
-    if (this._check('webhooks', 'homedb')) { return; }
-    if (this.deps.has('json')) {
-      throw new Error('addEarlyWebhooks called too late');
-    }
+    if (this._check('webhooks', 'homedb', '!json')) { return; }
     this._getBilling();
     this._billing.addWebhooks(this.app);
   }
@@ -1885,99 +1881,26 @@ export class FlexServer implements GristServer {
     addGoogleAuthEndpoint(this.app, messagePage);
   }
 
-  public addInstallEndpoints() {
-    if (this._check('install')) { return; }
+  /**
+   * Adds early API.
+   *
+   * These API endpoints are intentionally added before other middleware to
+   * minimize the impact of failures during startup. This includes, for
+   * example, endpoints used by the Admin Panel for status checks.
+   *
+   * It's also desirable for some endpoints to be loaded early so that they
+   * can set their own middleware, before any defaults are added.
+   * For example, `addJsonSupport` enforces strict parsing of JSON, but a
+   * handful of endpoints need relaxed parsing (e.g. /configs).
+   */
+  public addEarlyApi() {
+    if (this._check('early-api', 'api-mw', 'homedb', '!json')) { return; }
 
-    const requireInstallAdmin = this.getInstallAdmin().getMiddlewareRequireAdmin();
-
-    // Admin endpoint needs to have very little middleware since each
-    // piece of middleware creates a new way to fail and leave the admin
-    // panel inaccessible. Generally the admin panel should report problems
-    // rather than failing entirely.
-    this.app.get('/admin', this._userIdMiddleware, expressWrap(async (req, resp) => {
-      return this.sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
-    }));
-    const adminMiddleware = [
-      this._userIdMiddleware,
-      requireInstallAdmin,
-    ];
-    const probes = new BootProbes(this.app, this, '/api', adminMiddleware);
-    probes.addEndpoints();
-
-    this.app.post('/api/admin/restart', requireInstallAdmin, expressWrap(async (_, resp) => {
-      resp.on('finish', () => {
-        // If we have IPC with parent process (e.g. when running under
-        // Docker) tell the parent that we have a new environment so it
-        // can restart us.
-        if (process.send) {
-          process.send({ action: 'restart' });
-        }
-      });
-
-      if(!process.env.GRIST_RUNNING_UNDER_SUPERVISOR) {
-        // On the topic of http response codes, thus spake MDN:
-        // "409: This response is sent when a request conflicts with the current state of the server."
-        return resp.status(409).send({
-          error: "Cannot automatically restart the Grist server to enact changes. Please restart server manually."
-        });
-      }
-      return resp.status(200).send({ msg: 'ok' });
-    }));
-
-    // Restrict this endpoint to install admins
-    this.app.get('/api/install/prefs', requireInstallAdmin, expressWrap(async (_req, resp) => {
-      const activation = await this._activations.current();
-
-      return sendOkReply(null, resp, {
-        telemetry: await getTelemetryPrefs(this._dbManager, activation),
-      });
-    }));
-
-    this.app.patch('/api/install/prefs', requireInstallAdmin, expressWrap(async (req, resp) => {
-      const props = {prefs: req.body};
-      const activation = await this._activations.current();
-      activation.checkProperties(props);
-      activation.updateFromProperties(props);
-      await activation.save();
-
-      if ((props as Partial<InstallProperties>).prefs?.telemetry) {
-        // Make sure the Telemetry singleton picks up the changes to telemetry preferences.
-        // TODO: if there are multiple home server instances, notify them all of changes to
-        // preferences (via Redis Pub/Sub).
-        await this._telemetry.fetchTelemetryPrefs();
-      }
-
-      return resp.status(200).send();
-    }));
-
-    // GET api/checkUpdates
-    // Retrieves the latest version of the client from Grist SAAS endpoint.
-    this.app.get('/api/install/updates', adminMiddleware, expressWrap(async (req, res) => {
-      // Prepare data for the telemetry that endpoint might expect.
-      const installationId = (await this.getActivations().current()).id;
-      const deploymentType = this.getDeploymentType();
-      const currentVersion = version.version;
-      const response = await fetch(process.env.GRIST_TEST_VERSION_CHECK_URL || commonUrls.versionCheck, {
-        method: 'POST',
-        headers: {'Content-Type': 'application/json'},
-        body: JSON.stringify({
-          installationId,
-          deploymentType,
-          currentVersion,
-        }),
-      });
-      if (!response.ok) {
-        res.status(response.status);
-        if (response.headers.get('content-type')?.includes('application/json')) {
-          const data = await response.json();
-          res.json(data);
-        } else {
-          res.send(await response.text());
-        }
-      } else {
-        res.json(await response.json());
-      }
-    }));
+    attachEarlyEndpoints({
+      app: this.app,
+      gristServer: this,
+      userIdMiddleware: this._userIdMiddleware,
+    });
   }
 
   public addConfigEndpoints() {

app/server/lib/attachEarlyEndpoints.ts

@@ -0,0 +1,317 @@
+import { ApiError } from "app/common/ApiError";
+import {
+  ConfigKey,
+  ConfigKeyChecker,
+  ConfigValue,
+  ConfigValueCheckers,
+} from "app/common/Config";
+import { commonUrls } from "app/common/gristUrls";
+import { InstallProperties } from "app/common/InstallAPI";
+import * as version from "app/common/version";
+import { getOrgKey } from "app/gen-server/ApiServer";
+import { Config } from "app/gen-server/entity/Config";
+import {
+  PreviousAndCurrent,
+  QueryResult,
+} from "app/gen-server/lib/homedb/Interfaces";
+import { BootProbes } from "app/server/lib/BootProbes";
+import { expressWrap } from "app/server/lib/expressWrap";
+import { GristServer } from "app/server/lib/GristServer";
+import log from "app/server/lib/log";
+import {
+  getScope,
+  sendOkReply,
+  sendReply,
+  stringParam,
+} from "app/server/lib/requestUtils";
+import { getTelemetryPrefs } from "app/server/lib/Telemetry";
+import {
+  Application,
+  json,
+  NextFunction,
+  Request,
+  RequestHandler,
+  Response,
+} from "express";
+import pick from "lodash/pick";
+
+export interface AttachOptions {
+  app: Application;
+  gristServer: GristServer;
+  userIdMiddleware: RequestHandler;
+}
+
+export function attachEarlyEndpoints(options: AttachOptions) {
+  const { app, gristServer, userIdMiddleware } = options;
+
+  // Admin endpoint needs to have very little middleware since each
+  // piece of middleware creates a new way to fail and leave the admin
+  // panel inaccessible. Generally the admin panel should report problems
+  // rather than failing entirely.
+  app.get(
+    "/admin",
+    userIdMiddleware,
+    expressWrap(async (req, res) => {
+      return gristServer.sendAppPage(req, res, {
+        path: "app.html",
+        status: 200,
+        config: {},
+      });
+    })
+  );
+
+  const requireInstallAdmin = gristServer
+    .getInstallAdmin()
+    .getMiddlewareRequireAdmin();
+
+  const adminMiddleware = [requireInstallAdmin];
+  app.use("/api/admin", adminMiddleware);
+  app.use("/api/install", adminMiddleware);
+
+  const probes = new BootProbes(app, gristServer, "/api", adminMiddleware);
+  probes.addEndpoints();
+
+  app.post(
+    "/api/admin/restart",
+    expressWrap(async (_, res) => {
+      res.on("finish", () => {
+        // If we have IPC with parent process (e.g. when running under
+        // Docker) tell the parent that we have a new environment so it
+        // can restart us.
+        if (process.send) {
+          process.send({ action: "restart" });
+        }
+      });
+
+      if (!process.env.GRIST_RUNNING_UNDER_SUPERVISOR) {
+        // On the topic of http response codes, thus spake MDN:
+        // "409: This response is sent when a request conflicts with the current state of the server."
+        return res.status(409).send({
+          error:
+            "Cannot automatically restart the Grist server to enact changes. Please restart server manually.",
+        });
+      }
+      return res.status(200).send({ msg: "ok" });
+    })
+  );
+
+  // Restrict this endpoint to install admins.
+  app.get(
+    "/api/install/prefs",
+    expressWrap(async (_req, res) => {
+      const activation = await gristServer.getActivations().current();
+
+      return sendOkReply(null, res, {
+        telemetry: await getTelemetryPrefs(
+          gristServer.getHomeDBManager(),
+          activation
+        ),
+      });
+    })
+  );
+
+  app.patch(
+    "/api/install/prefs",
+    json({ limit: "1mb" }),
+    expressWrap(async (req, res) => {
+      const props = { prefs: req.body };
+      const activation = await gristServer.getActivations().current();
+      activation.checkProperties(props);
+      activation.updateFromProperties(props);
+      await activation.save();
+
+      if ((props as Partial<InstallProperties>).prefs?.telemetry) {
+        // Make sure the Telemetry singleton picks up the changes to telemetry preferences.
+        // TODO: if there are multiple home server instances, notify them all of changes to
+        // preferences (via Redis Pub/Sub).
+        await gristServer.getTelemetry().fetchTelemetryPrefs();
+      }
+
+      return res.status(200).send();
+    })
+  );
+
+  // Retrieves the latest version of the client from Grist SAAS endpoint.
+  app.get(
+    "/api/install/updates",
+    expressWrap(async (_req, res) => {
+      // Prepare data for the telemetry that endpoint might expect.
+      const installationId = (await gristServer.getActivations().current()).id;
+      const deploymentType = gristServer.getDeploymentType();
+      const currentVersion = version.version;
+      const response = await fetch(
+        process.env.GRIST_TEST_VERSION_CHECK_URL || commonUrls.versionCheck,
+        {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            installationId,
+            deploymentType,
+            currentVersion,
+          }),
+        }
+      );
+      if (!response.ok) {
+        res.status(response.status);
+        if (
+          response.headers.get("content-type")?.includes("application/json")
+        ) {
+          const data = await response.json();
+          res.json(data);
+        } else {
+          res.send(await response.text());
+        }
+      } else {
+        res.json(await response.json());
+      }
+    })
+  );
+
+  app.get(
+    "/api/install/configs/:key",
+    hasValidConfigKey,
+    expressWrap(async (req, res) => {
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const configResult = await gristServer
+        .getHomeDBManager()
+        .getInstallConfig(key);
+      const result = pruneConfigAPIResult(configResult);
+      return sendReply(req, res, result);
+    })
+  );
+
+  app.put(
+    "/api/install/configs/:key",
+    json({ limit: "1mb", strict: false }),
+    hasValidConfig,
+    expressWrap(async (req, res) => {
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const value = req.body as ConfigValue;
+      const configResult = await gristServer
+        .getHomeDBManager()
+        .updateInstallConfig(key, value);
+      const result = pruneConfigAPIResult(configResult);
+      return sendReply(req, res, result);
+    })
+  );
+
+  app.delete(
+    "/api/install/configs/:key",
+    hasValidConfigKey,
+    expressWrap(async (req, res) => {
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const { status } = await gristServer
+        .getHomeDBManager()
+        .deleteInstallConfig(key);
+      return sendReply(req, res, { status });
+    })
+  );
+
+  app.get(
+    "/api/orgs/:oid/configs/:key",
+    hasValidConfigKey,
+    expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const configResult = await gristServer
+        .getHomeDBManager()
+        .getOrgConfig(getScope(req), org, key);
+      const result = pruneConfigAPIResult(configResult);
+      return sendReply(req, res, result);
+    })
+  );
+
+  app.put(
+    "/api/orgs/:oid/configs/:key",
+    json({ limit: "1mb", strict: false }),
+    hasValidConfig,
+    expressWrap(async (req, res) => {
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const org = getOrgKey(req);
+      const value = req.body as ConfigValue;
+      const configResult = await gristServer
+        .getHomeDBManager()
+        .updateOrgConfig(getScope(req), org, key, value);
+      const result = pruneConfigAPIResult(configResult);
+      return sendReply(req, res, result);
+    })
+  );
+
+  app.delete(
+    "/api/orgs/:oid/configs/:key",
+    hasValidConfigKey,
+    expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const key = stringParam(req.params.key, "key") as ConfigKey;
+      const { status } = await gristServer
+        .getHomeDBManager()
+        .deleteOrgConfig(getScope(req), org, key);
+      return sendReply(req, res, { status });
+    })
+  );
+}
+
+function pruneConfigAPIResult(
+  result: QueryResult<Config | PreviousAndCurrent<Config>>
+) {
+  if (!result.data) {
+    return result as unknown as QueryResult<undefined>;
+  }
+
+  const config = "previous" in result.data ? result.data.current : result.data;
+  return {
+    ...result,
+    data: {
+      ...pick(config, "id", "key", "value", "createdAt", "updatedAt"),
+      ...(config.org
+        ? { org: pick(config.org, "id", "name", "domain") }
+        : undefined),
+    },
+  };
+}
+
+function hasValidConfig(req: Request, _res: Response, next: NextFunction) {
+  try {
+    assertValidConfig(req);
+    next();
+  } catch (e) {
+    next(e);
+  }
+}
+
+function hasValidConfigKey(req: Request, _res: Response, next: NextFunction) {
+  try {
+    assertValidConfigKey(req);
+    next();
+  } catch (e) {
+    next(e);
+  }
+}
+
+function assertValidConfig(req: Request) {
+  assertValidConfigKey(req);
+  const key = stringParam(req.params.key, "key") as ConfigKey;
+  try {
+    ConfigValueCheckers[key].check(req.body);
+  } catch (err) {
+    log.warn(
+      `Error during API call to ${req.path}: invalid config value (${String(
+        err
+      )})`
+    );
+    throw new ApiError("Invalid config value", 400, { userError: String(err) });
+  }
+}
+
+function assertValidConfigKey(req: Request) {
+  try {
+    ConfigKeyChecker.check(req.params.key);
+  } catch (err) {
+    log.warn(
+      `Error during API call to ${req.path}: invalid config key (${String(
+        err
+      )})`
+    );
+    throw new ApiError("Invalid config key", 400, { userError: String(err) });
+  }
+}

app/server/lib/requestUtils.ts

@@ -211,10 +211,11 @@ export async function sendReply<T>(
       result: data,
     });
   }
-  if (result.status === 200) {
+  res.status(result.status);
+  if (result.status >= 200 && result.status < 300) {
     return res.json(data ?? null); // can't handle undefined
   } else {
-    return res.status(result.status).json({error: result.errMessage});
+    return res.json({error: result.errMessage});
   }
 }