(core) Add audit logging machinery

Summary:
Adds machinery to support audit logging in the backend.

Logging is currently implemented by streaming events to external HTTP
endpoints. All flavors of Grist support a default "grist" payload format,
and Grist Enterprise additionally supports an HEC-compatible payload format.

Logging of all audit events will be added at a later date.

Test Plan: Server tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D4331

app/server/generateInitialDocSql.ts

@@ -3,7 +3,7 @@ import { create } from 'app/server/lib/create';
 import { DocManager } from 'app/server/lib/DocManager';
 import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
 import { DocStorageManager } from 'app/server/lib/DocStorageManager';
-import { createDummyTelemetry } from 'app/server/lib/GristServer';
+import { createDummyAuditLogger, createDummyTelemetry } from 'app/server/lib/GristServer';
 import { PluginManager } from 'app/server/lib/PluginManager';
 
 import * as childProcess from 'child_process';
@@ -34,6 +34,7 @@ export async function main(baseName: string) {
     }
     const docManager = new DocManager(storageManager, pluginManager, null as any, {
       create,
+      getAuditLogger() { return createDummyAuditLogger(); },
       getTelemetry() { return createDummyTelemetry(); },
     } as any);
     const activeDoc = new ActiveDoc(docManager, baseName);

app/server/lib/AuditLogger.ts

@@ -0,0 +1,40 @@
+import {AuditEventDetails, AuditEventName} from 'app/common/AuditEvent';
+import {RequestOrSession} from 'app/server/lib/requestUtils';
+
+export interface IAuditLogger {
+  /**
+   * Logs an audit event.
+   */
+  logEvent<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    props: AuditEventProperties<Name>
+  ): void;
+  /**
+   * Asynchronous variant of `logEvent`.
+   *
+   * Throws on failure to log an event.
+   */
+  logEventAsync<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    props: AuditEventProperties<Name>
+  ): Promise<void>;
+}
+
+export interface AuditEventProperties<Name extends AuditEventName> {
+  event: {
+    /**
+     * The event name.
+     */
+    name: Name;
+    /**
+     * Additional event details.
+     */
+    details?: AuditEventDetails[Name];
+  };
+  /**
+   * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured.
+   *
+   * Defaults to now.
+   */
+  timestamp?: string;
+}

app/server/lib/DocApi.ts

@@ -1730,6 +1730,12 @@ export class DocWorkerApi {
         docIdDigest: docId,
       },
     });
+    this._grist.getAuditLogger().logEvent(req as RequestWithLogin, {
+      event: {
+        name: 'createDocument',
+        details: {id: docId},
+      },
+    });
     return docId;
   }
 
@@ -1737,6 +1743,7 @@ export class DocWorkerApi {
     userId: number,
     browserSettings?: BrowserSettings,
   }): Promise<string> {
+    const mreq = req as RequestWithLogin;
     const {userId, browserSettings} = options;
     const isAnonymous = isAnonymousUser(req);
     const result = makeForkIds({
@@ -1747,10 +1754,7 @@ export class DocWorkerApi {
     });
     const docId = result.docId;
     await this._docManager.createNamedDoc(
-      makeExceptionalDocSession('nascent', {
-        req: req as RequestWithLogin,
-        browserSettings,
-      }),
+      makeExceptionalDocSession('nascent', {req: mreq, browserSettings}),
       docId
     );
     this._logDocumentCreatedTelemetryEvent(req, {
@@ -1767,6 +1771,12 @@ export class DocWorkerApi {
         docIdDigest: docId,
       },
     });
+    this._grist.getAuditLogger().logEvent(mreq, {
+      event: {
+        name: 'createDocument',
+        details: {id: docId},
+      },
+    });
     return docId;
   }
 

app/server/lib/DocManager.ts

@@ -254,6 +254,12 @@ export class DocManager extends EventEmitter {
         isSaved: workspaceId !== undefined,
       },
     }, telemetryMetadata));
+    this.gristServer.getAuditLogger().logEvent(mreq, {
+      event: {
+        name: 'createDocument',
+        details: {id: docCreationInfo.id},
+      },
+    });
 
     return docCreationInfo;
     // The imported document is associated with the worker that did the import.

app/server/lib/FlexServer.ts

@@ -27,6 +27,7 @@ import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens';
 import {createSandbox} from 'app/server/lib/ActiveDoc';
 import {attachAppEndpoint} from 'app/server/lib/AppEndpoint';
 import {appSettings} from 'app/server/lib/AppSettings';
+import {IAuditLogger} from 'app/server/lib/AuditLogger';
 import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser,
         isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer';
 import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer';
@@ -151,6 +152,7 @@ export class FlexServer implements GristServer {
   private _sessions: Sessions;
   private _sessionStore: SessionStore;
   private _storageManager: IDocStorageManager;
+  private _auditLogger: IAuditLogger;
   private _telemetry: ITelemetry;
   private _processMonitorStop?: () => void;    // Callback to stop the ProcessMonitor
   private _docWorkerMap: IDocWorkerMap;
@@ -398,6 +400,11 @@ export class FlexServer implements GristServer {
     return this._storageManager;
   }
 
+  public getAuditLogger(): IAuditLogger {
+    if (!this._auditLogger) { throw new Error('no audit logger available'); }
+    return this._auditLogger;
+  }
+
   public getTelemetry(): ITelemetry {
     if (!this._telemetry) { throw new Error('no telemetry available'); }
     return this._telemetry;
@@ -911,6 +918,12 @@ export class FlexServer implements GristServer {
     });
   }
 
+  public addAuditLogger() {
+    if (this._check('audit-logger')) { return; }
+
+    this._auditLogger = this.create.AuditLogger();
+  }
+
   public async addTelemetry() {
     if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; }
 

app/server/lib/GristAuditLogger.ts

@@ -0,0 +1,9 @@
+import {AuditEvent, AuditEventName} from 'app/common/AuditEvent';
+import {IAuditLogger} from 'app/server/lib/AuditLogger';
+import {HTTPAuditLogger} from 'app/server/lib/HTTPAuditLogger';
+
+export class GristAuditLogger extends HTTPAuditLogger implements IAuditLogger {
+  protected toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string {
+    return JSON.stringify(event);
+  }
+}

app/server/lib/GristServer.ts

@@ -9,6 +9,7 @@ import { User } from 'app/gen-server/entity/User';
 import { Workspace } from 'app/gen-server/entity/Workspace';
 import { Activations } from 'app/gen-server/lib/Activations';
 import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
+import { IAuditLogger } from 'app/server/lib/AuditLogger';
 import { IAccessTokens } from 'app/server/lib/AccessTokens';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
 import { Comm } from 'app/server/lib/Comm';
@@ -54,6 +55,7 @@ export interface GristServer {
   getInstallAdmin(): InstallAdmin;
   getHomeDBManager(): HomeDBManager;
   getStorageManager(): IDocStorageManager;
+  getAuditLogger(): IAuditLogger;
   getTelemetry(): ITelemetry;
   hasNotifier(): boolean;
   getNotifier(): INotifier;
@@ -147,6 +149,7 @@ export function createDummyGristServer(): GristServer {
     getInstallAdmin() { throw new Error('no install admin'); },
     getHomeDBManager() { throw new Error('no db'); },
     getStorageManager() { throw new Error('no storage manager'); },
+    getAuditLogger() { return createDummyAuditLogger(); },
     getTelemetry() { return createDummyTelemetry(); },
     getNotifier() { throw new Error('no notifier'); },
     hasNotifier() { return false; },
@@ -165,6 +168,13 @@ export function createDummyGristServer(): GristServer {
   };
 }
 
+export function createDummyAuditLogger(): IAuditLogger {
+  return {
+    logEvent() { /* do nothing */ },
+    logEventAsync() { return Promise.resolve(); },
+  };
+}
+
 export function createDummyTelemetry(): ITelemetry {
   return {
     addEndpoints() { /* do nothing */ },

app/server/lib/HTTPAuditLogger.ts

@@ -0,0 +1,135 @@
+import {AuditEvent, AuditEventName, AuditEventUser} from 'app/common/AuditEvent';
+import {AuditEventProperties, IAuditLogger} from 'app/server/lib/AuditLogger';
+import {getDocSessionUser} from 'app/server/lib/DocSession';
+import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods';
+import {RequestOrSession} from 'app/server/lib/requestUtils';
+import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
+import moment from 'moment-timezone';
+import fetch from 'node-fetch';
+
+interface HTTPAuditLoggerOptions {
+  /**
+   * The HTTP endpoint to send audit events to.
+   */
+  endpoint: string;
+  /**
+   * If set, the value to include in the `Authorization` header of each
+   * request to `endpoint`.
+   */
+  authorizationHeader?: string;
+}
+
+const MAX_PENDING_REQUESTS = 25;
+
+/**
+ * Base class for an audit event logger that logs events by sending them to an JSON-based HTTP
+ * endpoint.
+ *
+ * Subclasses are expected to provide a suitable `toJSON` implementation to handle serialization
+ * of audit events to JSON.
+ *
+ * See `GristAuditLogger` for an example.
+ */
+export abstract class HTTPAuditLogger implements IAuditLogger {
+  private _endpoint = this._options.endpoint;
+  private _authorizationHeader = this._options.authorizationHeader;
+  private _numPendingRequests = 0;
+  private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession: RequestOrSession | undefined) =>
+    getLogMeta(requestOrSession));
+
+  constructor(private _options: HTTPAuditLoggerOptions) {}
+
+  /**
+   * Logs an audit event.
+   */
+  public logEvent<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    event: AuditEventProperties<Name>
+  ): void {
+    this._logEventOrThrow(requestOrSession, event)
+      .catch((e) => this._logger.error(requestOrSession, `failed to log audit event`, event, e));
+  }
+
+  /**
+   * Asynchronous variant of `logEvent`.
+   *
+   * Throws on failure to log an event.
+   */
+  public async logEventAsync<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    event: AuditEventProperties<Name>
+  ): Promise<void> {
+    await this._logEventOrThrow(requestOrSession, event);
+  }
+
+  /**
+   * Serializes an audit event to JSON.
+   */
+  protected abstract toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string;
+
+  private async _logEventOrThrow<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    {event: {name, details}, timestamp}: AuditEventProperties<Name>
+  ) {
+    if (this._numPendingRequests === MAX_PENDING_REQUESTS) {
+      throw new Error(`exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`);
+    }
+
+    try {
+      this._numPendingRequests += 1;
+      const resp = await fetch(this._endpoint, {
+        method: 'POST',
+        headers: {
+          ...(this._authorizationHeader ? {'Authorization': this._authorizationHeader} : undefined),
+          'Content-Type': 'application/json',
+        },
+        body: this.toJSON({
+          event: {
+            name,
+            user: getAuditEventUser(requestOrSession),
+            details: details ?? null,
+          },
+          timestamp: timestamp ?? moment().toISOString(),
+        }),
+      });
+      if (!resp.ok) {
+        throw new Error(`received a non-200 response from ${resp.url}: ${resp.status} ${await resp.text()}`);
+      }
+    } finally {
+      this._numPendingRequests -= 1;
+    }
+  }
+}
+
+function getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser | null {
+  if (!requestOrSession) { return null; }
+
+  if ('get' in requestOrSession) {
+    return {
+      id: requestOrSession.userId ?? null,
+      email: requestOrSession.user?.loginEmail ?? null,
+      name: requestOrSession.user?.name ?? null,
+    };
+  } else {
+    const user = getDocSessionUser(requestOrSession);
+    if (!user) { return null; }
+
+    const {id, email, name} = user;
+    return {id, email, name};
+  }
+}
+
+function getLogMeta(requestOrSession?: RequestOrSession): ILogMeta {
+  if (!requestOrSession) { return {}; }
+
+  if ('get' in requestOrSession) {
+    return {
+      org: requestOrSession.org,
+      email: requestOrSession.user?.loginEmail,
+      userId: requestOrSession.userId,
+      altSessionId: requestOrSession.altSessionId,
+    };
+  } else {
+    return getLogMetaFromDocSession(requestOrSession);
+  }
+}

app/server/lib/ICreate.ts

@@ -2,8 +2,9 @@ import {GristDeploymentType} from 'app/common/gristUrls';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {IAuditLogger} from 'app/server/lib/AuditLogger';
 import {ExternalStorage} from 'app/server/lib/ExternalStorage';
-import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
+import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
 import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier';
 import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin';
@@ -40,6 +41,7 @@ export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
+  AuditLogger(): IAuditLogger;
   Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
   Shell?(): IShell;  // relevant to electron version of Grist only.
 
@@ -91,6 +93,12 @@ export interface ICreateBillingOptions {
   create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined;
 }
 
+export interface ICreateAuditLoggerOptions {
+  name: 'grist'|'hec';
+  check(): boolean;
+  create(): IAuditLogger|undefined;
+}
+
 export interface ICreateTelemetryOptions {
   create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined;
 }
@@ -110,6 +118,7 @@ export function makeSimpleCreator(opts: {
   storage?: ICreateStorageOptions[],
   billing?: ICreateBillingOptions,
   notifier?: ICreateNotifierOptions,
+  auditLogger?: ICreateAuditLoggerOptions[],
   telemetry?: ICreateTelemetryOptions,
   sandboxFlavor?: string,
   shell?: IShell,
@@ -118,7 +127,7 @@ export function makeSimpleCreator(opts: {
   getSandboxVariants?: () => Record<string, SpawnFn>,
   createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>,
 }): ICreate {
-  const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts;
+  const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts;
   return {
     deploymentType() { return deploymentType; },
     Billing(dbManager, gristConfig) {
@@ -141,6 +150,9 @@ export function makeSimpleCreator(opts: {
       }
       return undefined;
     },
+    AuditLogger() {
+      return auditLogger?.find(({check}) => check())?.create() ?? createDummyAuditLogger();
+    },
     Telemetry(dbManager, gristConfig) {
       return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();
     },

app/server/lib/Telemetry.ts

@@ -19,12 +19,12 @@ import {Activation} from 'app/gen-server/entity/Activation';
 import {Activations} from 'app/gen-server/lib/Activations';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
-import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession';
+import {getDocSessionUser} from 'app/server/lib/DocSession';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {GristServer} from 'app/server/lib/GristServer';
 import {hashId} from 'app/server/lib/hashingUtils';
 import {LogMethods} from 'app/server/lib/LogMethods';
-import {stringParam} from 'app/server/lib/requestUtils';
+import {RequestOrSession, stringParam} from 'app/server/lib/requestUtils';
 import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
 import * as cookie from 'cookie';
 import * as express from 'express';
@@ -32,8 +32,6 @@ import fetch from 'node-fetch';
 import merge = require('lodash/merge');
 import pickBy = require('lodash/pickBy');
 
-type RequestOrSession = RequestWithLogin | OptDocSession | null;
-
 interface RequestWithMatomoVisitorId extends RequestWithLogin {
   /**
    * Extracted from a cookie set by Matomo.

app/server/lib/configureGristAuditLogger.ts

@@ -0,0 +1,30 @@
+import {appSettings} from 'app/server/lib/AppSettings';
+import {GristAuditLogger} from 'app/server/lib/GristAuditLogger';
+
+export function configureGristAuditLogger() {
+  const options = checkGristAuditLogger();
+  if (!options) { return undefined; }
+
+  return new GristAuditLogger(options);
+}
+
+export function checkGristAuditLogger() {
+  const settings = appSettings.section('auditLogger').section('http');
+  const endpoint = settings.flag('endpoint').readString({
+    envVar: 'GRIST_AUDIT_HTTP_ENDPOINT',
+  });
+  if (!endpoint) { return undefined; }
+
+  const payloadFormat = settings.flag('payloadFormat').readString({
+    envVar: 'GRIST_AUDIT_HTTP_PAYLOAD_FORMAT',
+    defaultValue: 'grist',
+  });
+  if (payloadFormat !== 'grist') { return undefined; }
+
+  const authorizationHeader = settings.flag('authorizationHeader').readString({
+    envVar: 'GRIST_AUDIT_HTTP_AUTHORIZATION_HEADER',
+    censor: true,
+  });
+
+  return {endpoint, authorizationHeader};
+}

app/server/lib/coreCreator.ts

@@ -1,3 +1,4 @@
+import { checkGristAuditLogger, configureGristAuditLogger } from 'app/server/lib/configureGristAuditLogger';
 import { checkMinIOBucket, checkMinIOExternalStorage,
          configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage';
 import { makeSimpleCreator } from 'app/server/lib/ICreate';
@@ -13,6 +14,13 @@ export const makeCoreCreator = () => makeSimpleCreator({
       create: configureMinIOExternalStorage,
     },
   ],
+  auditLogger: [
+    {
+      name: 'grist',
+      check: () => checkGristAuditLogger() !== undefined,
+      create: configureGristAuditLogger,
+    },
+  ],
   telemetry: {
     create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer),
   }

app/server/lib/requestUtils.ts

@@ -1,16 +1,19 @@
 import {ApiError} from 'app/common/ApiError';
-import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
+import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
 import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {OptDocSession} from 'app/server/lib/DocSession';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {RequestWithGrist} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
 import {Permit} from 'app/server/lib/Permit';
 import {Request, Response} from 'express';
-import { IncomingMessage } from 'http';
+import {IncomingMessage} from 'http';
 import {Writable} from 'stream';
-import { TLSSocket } from 'tls';
+import {TLSSocket} from 'tls';
+
+export type RequestOrSession = RequestWithLogin | OptDocSession | null;
 
 // log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
 const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION);

app/server/mergedServerMain.ts

@@ -156,6 +156,7 @@ export async function main(port: number, serverTypes: ServerType[],
       server.addHomeApi();
       server.addBillingApi();
       server.addNotifier();
+      server.addAuditLogger();
       await server.addTelemetry();
       await server.addHousekeeper();
       await server.addLoginRoutes();
@@ -170,6 +171,7 @@ export async function main(port: number, serverTypes: ServerType[],
 
     if (includeDocs) {
       server.addJsonSupport();
+      server.addAuditLogger();
       await server.addTelemetry();
       await server.addDoc();
     }