mirror of
				https://github.com/gristlabs/grist-core.git
				synced 2025-06-13 20:53:59 +00:00 
			
		
		
		
	(core) Add audit logging machinery
Summary: Adds machinery to support audit logging in the backend. Logging is currently implemented by streaming events to external HTTP endpoints. All flavors of Grist support a default "grist" payload format, and Grist Enterprise additionally supports an HEC-compatible payload format. Logging of all audit events will be added at a later date. Test Plan: Server tests. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4331
This commit is contained in:
		
							parent
							
								
									14718120bd
								
							
						
					
					
						commit
						3e22b89fa2
					
				
							
								
								
									
										31
									
								
								app/common/AuditEvent.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										31
									
								
								app/common/AuditEvent.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,31 @@ | ||||
| export interface AuditEvent<Name extends AuditEventName> { | ||||
|   event: { | ||||
|     /** The event name. */ | ||||
|     name: Name; | ||||
|     /** The user that triggered the event. */ | ||||
|     user: AuditEventUser | null; | ||||
|     /** Additional event details. */ | ||||
|     details: AuditEventDetails[Name] | null; | ||||
|   }; | ||||
|   /** ISO 8601 timestamp of when the event was logged. */ | ||||
|   timestamp: string; | ||||
| } | ||||
| 
 | ||||
| export type AuditEventName = | ||||
|   | 'createDocument'; | ||||
| 
 | ||||
| export interface AuditEventUser { | ||||
|   /** The user's id. */ | ||||
|   id: number | null; | ||||
|   /** The user's email address. */ | ||||
|   email: string | null; | ||||
|   /** The user's name. */ | ||||
|   name: string | null; | ||||
| } | ||||
| 
 | ||||
| export interface AuditEventDetails { | ||||
|   createDocument: { | ||||
|     /** The ID of the document. */ | ||||
|     id: string; | ||||
|   }; | ||||
| } | ||||
| @ -283,6 +283,12 @@ export class ApiServer { | ||||
|           altSessionId: mreq.altSessionId, | ||||
|         }, | ||||
|       }); | ||||
|       this._gristServer.getAuditLogger().logEvent(mreq, { | ||||
|         event: { | ||||
|           name: 'createDocument', | ||||
|           details: {id: docId}, | ||||
|         }, | ||||
|       }); | ||||
|       return sendReply(req, res, query); | ||||
|     })); | ||||
| 
 | ||||
|  | ||||
| @ -3,7 +3,7 @@ import { create } from 'app/server/lib/create'; | ||||
| import { DocManager } from 'app/server/lib/DocManager'; | ||||
| import { makeExceptionalDocSession } from 'app/server/lib/DocSession'; | ||||
| import { DocStorageManager } from 'app/server/lib/DocStorageManager'; | ||||
| import { createDummyTelemetry } from 'app/server/lib/GristServer'; | ||||
| import { createDummyAuditLogger, createDummyTelemetry } from 'app/server/lib/GristServer'; | ||||
| import { PluginManager } from 'app/server/lib/PluginManager'; | ||||
| 
 | ||||
| import * as childProcess from 'child_process'; | ||||
| @ -34,6 +34,7 @@ export async function main(baseName: string) { | ||||
|     } | ||||
|     const docManager = new DocManager(storageManager, pluginManager, null as any, { | ||||
|       create, | ||||
|       getAuditLogger() { return createDummyAuditLogger(); }, | ||||
|       getTelemetry() { return createDummyTelemetry(); }, | ||||
|     } as any); | ||||
|     const activeDoc = new ActiveDoc(docManager, baseName); | ||||
|  | ||||
							
								
								
									
										40
									
								
								app/server/lib/AuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										40
									
								
								app/server/lib/AuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,40 @@ | ||||
| import {AuditEventDetails, AuditEventName} from 'app/common/AuditEvent'; | ||||
| import {RequestOrSession} from 'app/server/lib/requestUtils'; | ||||
| 
 | ||||
| export interface IAuditLogger { | ||||
|   /** | ||||
|    * Logs an audit event. | ||||
|    */ | ||||
|   logEvent<Name extends AuditEventName>( | ||||
|     requestOrSession: RequestOrSession, | ||||
|     props: AuditEventProperties<Name> | ||||
|   ): void; | ||||
|   /** | ||||
|    * Asynchronous variant of `logEvent`. | ||||
|    * | ||||
|    * Throws on failure to log an event. | ||||
|    */ | ||||
|   logEventAsync<Name extends AuditEventName>( | ||||
|     requestOrSession: RequestOrSession, | ||||
|     props: AuditEventProperties<Name> | ||||
|   ): Promise<void>; | ||||
| } | ||||
| 
 | ||||
| export interface AuditEventProperties<Name extends AuditEventName> { | ||||
|   event: { | ||||
|     /** | ||||
|      * The event name. | ||||
|      */ | ||||
|     name: Name; | ||||
|     /** | ||||
|      * Additional event details. | ||||
|      */ | ||||
|     details?: AuditEventDetails[Name]; | ||||
|   }; | ||||
|   /** | ||||
|    * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured. | ||||
|    * | ||||
|    * Defaults to now. | ||||
|    */ | ||||
|   timestamp?: string; | ||||
| } | ||||
| @ -1730,6 +1730,12 @@ export class DocWorkerApi { | ||||
|         docIdDigest: docId, | ||||
|       }, | ||||
|     }); | ||||
|     this._grist.getAuditLogger().logEvent(req as RequestWithLogin, { | ||||
|       event: { | ||||
|         name: 'createDocument', | ||||
|         details: {id: docId}, | ||||
|       }, | ||||
|     }); | ||||
|     return docId; | ||||
|   } | ||||
| 
 | ||||
| @ -1737,6 +1743,7 @@ export class DocWorkerApi { | ||||
|     userId: number, | ||||
|     browserSettings?: BrowserSettings, | ||||
|   }): Promise<string> { | ||||
|     const mreq = req as RequestWithLogin; | ||||
|     const {userId, browserSettings} = options; | ||||
|     const isAnonymous = isAnonymousUser(req); | ||||
|     const result = makeForkIds({ | ||||
| @ -1747,10 +1754,7 @@ export class DocWorkerApi { | ||||
|     }); | ||||
|     const docId = result.docId; | ||||
|     await this._docManager.createNamedDoc( | ||||
|       makeExceptionalDocSession('nascent', { | ||||
|         req: req as RequestWithLogin, | ||||
|         browserSettings, | ||||
|       }), | ||||
|       makeExceptionalDocSession('nascent', {req: mreq, browserSettings}), | ||||
|       docId | ||||
|     ); | ||||
|     this._logDocumentCreatedTelemetryEvent(req, { | ||||
| @ -1767,6 +1771,12 @@ export class DocWorkerApi { | ||||
|         docIdDigest: docId, | ||||
|       }, | ||||
|     }); | ||||
|     this._grist.getAuditLogger().logEvent(mreq, { | ||||
|       event: { | ||||
|         name: 'createDocument', | ||||
|         details: {id: docId}, | ||||
|       }, | ||||
|     }); | ||||
|     return docId; | ||||
|   } | ||||
| 
 | ||||
|  | ||||
| @ -254,6 +254,12 @@ export class DocManager extends EventEmitter { | ||||
|         isSaved: workspaceId !== undefined, | ||||
|       }, | ||||
|     }, telemetryMetadata)); | ||||
|     this.gristServer.getAuditLogger().logEvent(mreq, { | ||||
|       event: { | ||||
|         name: 'createDocument', | ||||
|         details: {id: docCreationInfo.id}, | ||||
|       }, | ||||
|     }); | ||||
| 
 | ||||
|     return docCreationInfo; | ||||
|     // The imported document is associated with the worker that did the import.
 | ||||
|  | ||||
| @ -27,6 +27,7 @@ import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens'; | ||||
| import {createSandbox} from 'app/server/lib/ActiveDoc'; | ||||
| import {attachAppEndpoint} from 'app/server/lib/AppEndpoint'; | ||||
| import {appSettings} from 'app/server/lib/AppSettings'; | ||||
| import {IAuditLogger} from 'app/server/lib/AuditLogger'; | ||||
| import {addRequestUser, getTransitiveHeaders, getUser, getUserId, isAnonymousUser, | ||||
|         isSingleUserMode, redirectToLoginUnconditionally} from 'app/server/lib/Authorizer'; | ||||
| import {redirectToLogin, RequestWithLogin, signInStatusMiddleware} from 'app/server/lib/Authorizer'; | ||||
| @ -151,6 +152,7 @@ export class FlexServer implements GristServer { | ||||
|   private _sessions: Sessions; | ||||
|   private _sessionStore: SessionStore; | ||||
|   private _storageManager: IDocStorageManager; | ||||
|   private _auditLogger: IAuditLogger; | ||||
|   private _telemetry: ITelemetry; | ||||
|   private _processMonitorStop?: () => void;    // Callback to stop the ProcessMonitor
 | ||||
|   private _docWorkerMap: IDocWorkerMap; | ||||
| @ -398,6 +400,11 @@ export class FlexServer implements GristServer { | ||||
|     return this._storageManager; | ||||
|   } | ||||
| 
 | ||||
|   public getAuditLogger(): IAuditLogger { | ||||
|     if (!this._auditLogger) { throw new Error('no audit logger available'); } | ||||
|     return this._auditLogger; | ||||
|   } | ||||
| 
 | ||||
|   public getTelemetry(): ITelemetry { | ||||
|     if (!this._telemetry) { throw new Error('no telemetry available'); } | ||||
|     return this._telemetry; | ||||
| @ -911,6 +918,12 @@ export class FlexServer implements GristServer { | ||||
|     }); | ||||
|   } | ||||
| 
 | ||||
|   public addAuditLogger() { | ||||
|     if (this._check('audit-logger')) { return; } | ||||
| 
 | ||||
|     this._auditLogger = this.create.AuditLogger(); | ||||
|   } | ||||
| 
 | ||||
|   public async addTelemetry() { | ||||
|     if (this._check('telemetry', 'homedb', 'json', 'api-mw')) { return; } | ||||
| 
 | ||||
|  | ||||
							
								
								
									
										9
									
								
								app/server/lib/GristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										9
									
								
								app/server/lib/GristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,9 @@ | ||||
| import {AuditEvent, AuditEventName} from 'app/common/AuditEvent'; | ||||
| import {IAuditLogger} from 'app/server/lib/AuditLogger'; | ||||
| import {HTTPAuditLogger} from 'app/server/lib/HTTPAuditLogger'; | ||||
| 
 | ||||
| export class GristAuditLogger extends HTTPAuditLogger implements IAuditLogger { | ||||
|   protected toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string { | ||||
|     return JSON.stringify(event); | ||||
|   } | ||||
| } | ||||
| @ -9,6 +9,7 @@ import { User } from 'app/gen-server/entity/User'; | ||||
| import { Workspace } from 'app/gen-server/entity/Workspace'; | ||||
| import { Activations } from 'app/gen-server/lib/Activations'; | ||||
| import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager'; | ||||
| import { IAuditLogger } from 'app/server/lib/AuditLogger'; | ||||
| import { IAccessTokens } from 'app/server/lib/AccessTokens'; | ||||
| import { RequestWithLogin } from 'app/server/lib/Authorizer'; | ||||
| import { Comm } from 'app/server/lib/Comm'; | ||||
| @ -54,6 +55,7 @@ export interface GristServer { | ||||
|   getInstallAdmin(): InstallAdmin; | ||||
|   getHomeDBManager(): HomeDBManager; | ||||
|   getStorageManager(): IDocStorageManager; | ||||
|   getAuditLogger(): IAuditLogger; | ||||
|   getTelemetry(): ITelemetry; | ||||
|   hasNotifier(): boolean; | ||||
|   getNotifier(): INotifier; | ||||
| @ -147,6 +149,7 @@ export function createDummyGristServer(): GristServer { | ||||
|     getInstallAdmin() { throw new Error('no install admin'); }, | ||||
|     getHomeDBManager() { throw new Error('no db'); }, | ||||
|     getStorageManager() { throw new Error('no storage manager'); }, | ||||
|     getAuditLogger() { return createDummyAuditLogger(); }, | ||||
|     getTelemetry() { return createDummyTelemetry(); }, | ||||
|     getNotifier() { throw new Error('no notifier'); }, | ||||
|     hasNotifier() { return false; }, | ||||
| @ -165,6 +168,13 @@ export function createDummyGristServer(): GristServer { | ||||
|   }; | ||||
| } | ||||
| 
 | ||||
| export function createDummyAuditLogger(): IAuditLogger { | ||||
|   return { | ||||
|     logEvent() { /* do nothing */ }, | ||||
|     logEventAsync() { return Promise.resolve(); }, | ||||
|   }; | ||||
| } | ||||
| 
 | ||||
| export function createDummyTelemetry(): ITelemetry { | ||||
|   return { | ||||
|     addEndpoints() { /* do nothing */ }, | ||||
|  | ||||
							
								
								
									
										135
									
								
								app/server/lib/HTTPAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										135
									
								
								app/server/lib/HTTPAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,135 @@ | ||||
| import {AuditEvent, AuditEventName, AuditEventUser} from 'app/common/AuditEvent'; | ||||
| import {AuditEventProperties, IAuditLogger} from 'app/server/lib/AuditLogger'; | ||||
| import {getDocSessionUser} from 'app/server/lib/DocSession'; | ||||
| import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods'; | ||||
| import {RequestOrSession} from 'app/server/lib/requestUtils'; | ||||
| import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils'; | ||||
| import moment from 'moment-timezone'; | ||||
| import fetch from 'node-fetch'; | ||||
| 
 | ||||
| interface HTTPAuditLoggerOptions { | ||||
|   /** | ||||
|    * The HTTP endpoint to send audit events to. | ||||
|    */ | ||||
|   endpoint: string; | ||||
|   /** | ||||
|    * If set, the value to include in the `Authorization` header of each | ||||
|    * request to `endpoint`. | ||||
|    */ | ||||
|   authorizationHeader?: string; | ||||
| } | ||||
| 
 | ||||
| const MAX_PENDING_REQUESTS = 25; | ||||
| 
 | ||||
| /** | ||||
|  * Base class for an audit event logger that logs events by sending them to an JSON-based HTTP | ||||
|  * endpoint. | ||||
|  * | ||||
|  * Subclasses are expected to provide a suitable `toJSON` implementation to handle serialization | ||||
|  * of audit events to JSON. | ||||
|  * | ||||
|  * See `GristAuditLogger` for an example. | ||||
|  */ | ||||
| export abstract class HTTPAuditLogger implements IAuditLogger { | ||||
|   private _endpoint = this._options.endpoint; | ||||
|   private _authorizationHeader = this._options.authorizationHeader; | ||||
|   private _numPendingRequests = 0; | ||||
|   private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession: RequestOrSession | undefined) => | ||||
|     getLogMeta(requestOrSession)); | ||||
| 
 | ||||
|   constructor(private _options: HTTPAuditLoggerOptions) {} | ||||
| 
 | ||||
|   /** | ||||
|    * Logs an audit event. | ||||
|    */ | ||||
|   public logEvent<Name extends AuditEventName>( | ||||
|     requestOrSession: RequestOrSession, | ||||
|     event: AuditEventProperties<Name> | ||||
|   ): void { | ||||
|     this._logEventOrThrow(requestOrSession, event) | ||||
|       .catch((e) => this._logger.error(requestOrSession, `failed to log audit event`, event, e)); | ||||
|   } | ||||
| 
 | ||||
|   /** | ||||
|    * Asynchronous variant of `logEvent`. | ||||
|    * | ||||
|    * Throws on failure to log an event. | ||||
|    */ | ||||
|   public async logEventAsync<Name extends AuditEventName>( | ||||
|     requestOrSession: RequestOrSession, | ||||
|     event: AuditEventProperties<Name> | ||||
|   ): Promise<void> { | ||||
|     await this._logEventOrThrow(requestOrSession, event); | ||||
|   } | ||||
| 
 | ||||
|   /** | ||||
|    * Serializes an audit event to JSON. | ||||
|    */ | ||||
|   protected abstract toJSON<Name extends AuditEventName>(event: AuditEvent<Name>): string; | ||||
| 
 | ||||
|   private async _logEventOrThrow<Name extends AuditEventName>( | ||||
|     requestOrSession: RequestOrSession, | ||||
|     {event: {name, details}, timestamp}: AuditEventProperties<Name> | ||||
|   ) { | ||||
|     if (this._numPendingRequests === MAX_PENDING_REQUESTS) { | ||||
|       throw new Error(`exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`); | ||||
|     } | ||||
| 
 | ||||
|     try { | ||||
|       this._numPendingRequests += 1; | ||||
|       const resp = await fetch(this._endpoint, { | ||||
|         method: 'POST', | ||||
|         headers: { | ||||
|           ...(this._authorizationHeader ? {'Authorization': this._authorizationHeader} : undefined), | ||||
|           'Content-Type': 'application/json', | ||||
|         }, | ||||
|         body: this.toJSON({ | ||||
|           event: { | ||||
|             name, | ||||
|             user: getAuditEventUser(requestOrSession), | ||||
|             details: details ?? null, | ||||
|           }, | ||||
|           timestamp: timestamp ?? moment().toISOString(), | ||||
|         }), | ||||
|       }); | ||||
|       if (!resp.ok) { | ||||
|         throw new Error(`received a non-200 response from ${resp.url}: ${resp.status} ${await resp.text()}`); | ||||
|       } | ||||
|     } finally { | ||||
|       this._numPendingRequests -= 1; | ||||
|     } | ||||
|   } | ||||
| } | ||||
| 
 | ||||
| function getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser | null { | ||||
|   if (!requestOrSession) { return null; } | ||||
| 
 | ||||
|   if ('get' in requestOrSession) { | ||||
|     return { | ||||
|       id: requestOrSession.userId ?? null, | ||||
|       email: requestOrSession.user?.loginEmail ?? null, | ||||
|       name: requestOrSession.user?.name ?? null, | ||||
|     }; | ||||
|   } else { | ||||
|     const user = getDocSessionUser(requestOrSession); | ||||
|     if (!user) { return null; } | ||||
| 
 | ||||
|     const {id, email, name} = user; | ||||
|     return {id, email, name}; | ||||
|   } | ||||
| } | ||||
| 
 | ||||
| function getLogMeta(requestOrSession?: RequestOrSession): ILogMeta { | ||||
|   if (!requestOrSession) { return {}; } | ||||
| 
 | ||||
|   if ('get' in requestOrSession) { | ||||
|     return { | ||||
|       org: requestOrSession.org, | ||||
|       email: requestOrSession.user?.loginEmail, | ||||
|       userId: requestOrSession.userId, | ||||
|       altSessionId: requestOrSession.altSessionId, | ||||
|     }; | ||||
|   } else { | ||||
|     return getLogMetaFromDocSession(requestOrSession); | ||||
|   } | ||||
| } | ||||
| @ -2,8 +2,9 @@ import {GristDeploymentType} from 'app/common/gristUrls'; | ||||
| import {getThemeBackgroundSnippet} from 'app/common/Themes'; | ||||
| import {Document} from 'app/gen-server/entity/Document'; | ||||
| import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; | ||||
| import {IAuditLogger} from 'app/server/lib/AuditLogger'; | ||||
| import {ExternalStorage} from 'app/server/lib/ExternalStorage'; | ||||
| import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer'; | ||||
| import {createDummyAuditLogger, createDummyTelemetry, GristServer} from 'app/server/lib/GristServer'; | ||||
| import {IBilling} from 'app/server/lib/IBilling'; | ||||
| import {EmptyNotifier, INotifier} from 'app/server/lib/INotifier'; | ||||
| import {InstallAdmin, SimpleInstallAdmin} from 'app/server/lib/InstallAdmin'; | ||||
| @ -40,6 +41,7 @@ export interface ICreate { | ||||
| 
 | ||||
|   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling; | ||||
|   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier; | ||||
|   AuditLogger(): IAuditLogger; | ||||
|   Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry; | ||||
|   Shell?(): IShell;  // relevant to electron version of Grist only.
 | ||||
| 
 | ||||
| @ -91,6 +93,12 @@ export interface ICreateBillingOptions { | ||||
|   create(dbManager: HomeDBManager, gristConfig: GristServer): IBilling|undefined; | ||||
| } | ||||
| 
 | ||||
| export interface ICreateAuditLoggerOptions { | ||||
|   name: 'grist'|'hec'; | ||||
|   check(): boolean; | ||||
|   create(): IAuditLogger|undefined; | ||||
| } | ||||
| 
 | ||||
| export interface ICreateTelemetryOptions { | ||||
|   create(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry|undefined; | ||||
| } | ||||
| @ -110,6 +118,7 @@ export function makeSimpleCreator(opts: { | ||||
|   storage?: ICreateStorageOptions[], | ||||
|   billing?: ICreateBillingOptions, | ||||
|   notifier?: ICreateNotifierOptions, | ||||
|   auditLogger?: ICreateAuditLoggerOptions[], | ||||
|   telemetry?: ICreateTelemetryOptions, | ||||
|   sandboxFlavor?: string, | ||||
|   shell?: IShell, | ||||
| @ -118,7 +127,7 @@ export function makeSimpleCreator(opts: { | ||||
|   getSandboxVariants?: () => Record<string, SpawnFn>, | ||||
|   createInstallAdmin?: (dbManager: HomeDBManager) => Promise<InstallAdmin>, | ||||
| }): ICreate { | ||||
|   const {deploymentType, sessionSecret, storage, notifier, billing, telemetry} = opts; | ||||
|   const {deploymentType, sessionSecret, storage, notifier, billing, auditLogger, telemetry} = opts; | ||||
|   return { | ||||
|     deploymentType() { return deploymentType; }, | ||||
|     Billing(dbManager, gristConfig) { | ||||
| @ -141,6 +150,9 @@ export function makeSimpleCreator(opts: { | ||||
|       } | ||||
|       return undefined; | ||||
|     }, | ||||
|     AuditLogger() { | ||||
|       return auditLogger?.find(({check}) => check())?.create() ?? createDummyAuditLogger(); | ||||
|     }, | ||||
|     Telemetry(dbManager, gristConfig) { | ||||
|       return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry(); | ||||
|     }, | ||||
|  | ||||
| @ -19,12 +19,12 @@ import {Activation} from 'app/gen-server/entity/Activation'; | ||||
| import {Activations} from 'app/gen-server/lib/Activations'; | ||||
| import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; | ||||
| import {RequestWithLogin} from 'app/server/lib/Authorizer'; | ||||
| import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession'; | ||||
| import {getDocSessionUser} from 'app/server/lib/DocSession'; | ||||
| import {expressWrap} from 'app/server/lib/expressWrap'; | ||||
| import {GristServer} from 'app/server/lib/GristServer'; | ||||
| import {hashId} from 'app/server/lib/hashingUtils'; | ||||
| import {LogMethods} from 'app/server/lib/LogMethods'; | ||||
| import {stringParam} from 'app/server/lib/requestUtils'; | ||||
| import {RequestOrSession, stringParam} from 'app/server/lib/requestUtils'; | ||||
| import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils'; | ||||
| import * as cookie from 'cookie'; | ||||
| import * as express from 'express'; | ||||
| @ -32,8 +32,6 @@ import fetch from 'node-fetch'; | ||||
| import merge = require('lodash/merge'); | ||||
| import pickBy = require('lodash/pickBy'); | ||||
| 
 | ||||
| type RequestOrSession = RequestWithLogin | OptDocSession | null; | ||||
| 
 | ||||
| interface RequestWithMatomoVisitorId extends RequestWithLogin { | ||||
|   /** | ||||
|    * Extracted from a cookie set by Matomo. | ||||
|  | ||||
							
								
								
									
										30
									
								
								app/server/lib/configureGristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										30
									
								
								app/server/lib/configureGristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,30 @@ | ||||
| import {appSettings} from 'app/server/lib/AppSettings'; | ||||
| import {GristAuditLogger} from 'app/server/lib/GristAuditLogger'; | ||||
| 
 | ||||
| export function configureGristAuditLogger() { | ||||
|   const options = checkGristAuditLogger(); | ||||
|   if (!options) { return undefined; } | ||||
| 
 | ||||
|   return new GristAuditLogger(options); | ||||
| } | ||||
| 
 | ||||
| export function checkGristAuditLogger() { | ||||
|   const settings = appSettings.section('auditLogger').section('http'); | ||||
|   const endpoint = settings.flag('endpoint').readString({ | ||||
|     envVar: 'GRIST_AUDIT_HTTP_ENDPOINT', | ||||
|   }); | ||||
|   if (!endpoint) { return undefined; } | ||||
| 
 | ||||
|   const payloadFormat = settings.flag('payloadFormat').readString({ | ||||
|     envVar: 'GRIST_AUDIT_HTTP_PAYLOAD_FORMAT', | ||||
|     defaultValue: 'grist', | ||||
|   }); | ||||
|   if (payloadFormat !== 'grist') { return undefined; } | ||||
| 
 | ||||
|   const authorizationHeader = settings.flag('authorizationHeader').readString({ | ||||
|     envVar: 'GRIST_AUDIT_HTTP_AUTHORIZATION_HEADER', | ||||
|     censor: true, | ||||
|   }); | ||||
| 
 | ||||
|   return {endpoint, authorizationHeader}; | ||||
| } | ||||
| @ -1,3 +1,4 @@ | ||||
| import { checkGristAuditLogger, configureGristAuditLogger } from 'app/server/lib/configureGristAuditLogger'; | ||||
| import { checkMinIOBucket, checkMinIOExternalStorage, | ||||
|          configureMinIOExternalStorage } from 'app/server/lib/configureMinIOExternalStorage'; | ||||
| import { makeSimpleCreator } from 'app/server/lib/ICreate'; | ||||
| @ -13,6 +14,13 @@ export const makeCoreCreator = () => makeSimpleCreator({ | ||||
|       create: configureMinIOExternalStorage, | ||||
|     }, | ||||
|   ], | ||||
|   auditLogger: [ | ||||
|     { | ||||
|       name: 'grist', | ||||
|       check: () => checkGristAuditLogger() !== undefined, | ||||
|       create: configureGristAuditLogger, | ||||
|     }, | ||||
|   ], | ||||
|   telemetry: { | ||||
|     create: (dbManager, gristServer) => new Telemetry(dbManager, gristServer), | ||||
|   } | ||||
|  | ||||
| @ -1,16 +1,19 @@ | ||||
| import {ApiError} from 'app/common/ApiError'; | ||||
| import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls'; | ||||
| import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls'; | ||||
| import * as gutil from 'app/common/gutil'; | ||||
| import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager'; | ||||
| import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; | ||||
| import {OptDocSession} from 'app/server/lib/DocSession'; | ||||
| import {RequestWithOrg} from 'app/server/lib/extractOrg'; | ||||
| import {RequestWithGrist} from 'app/server/lib/GristServer'; | ||||
| import log from 'app/server/lib/log'; | ||||
| import {Permit} from 'app/server/lib/Permit'; | ||||
| import {Request, Response} from 'express'; | ||||
| import { IncomingMessage } from 'http'; | ||||
| import {IncomingMessage} from 'http'; | ||||
| import {Writable} from 'stream'; | ||||
| import { TLSSocket } from 'tls'; | ||||
| import {TLSSocket} from 'tls'; | ||||
| 
 | ||||
| export type RequestOrSession = RequestWithLogin | OptDocSession | null; | ||||
| 
 | ||||
| // log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
 | ||||
| const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION); | ||||
|  | ||||
| @ -156,6 +156,7 @@ export async function main(port: number, serverTypes: ServerType[], | ||||
|       server.addHomeApi(); | ||||
|       server.addBillingApi(); | ||||
|       server.addNotifier(); | ||||
|       server.addAuditLogger(); | ||||
|       await server.addTelemetry(); | ||||
|       await server.addHousekeeper(); | ||||
|       await server.addLoginRoutes(); | ||||
| @ -170,6 +171,7 @@ export async function main(port: number, serverTypes: ServerType[], | ||||
| 
 | ||||
|     if (includeDocs) { | ||||
|       server.addJsonSupport(); | ||||
|       server.addAuditLogger(); | ||||
|       await server.addTelemetry(); | ||||
|       await server.addDoc(); | ||||
|     } | ||||
|  | ||||
| @ -97,6 +97,7 @@ | ||||
|     "mocha": "10.2.0", | ||||
|     "mocha-webdriver": "0.3.3", | ||||
|     "moment-locales-webpack-plugin": "^1.2.0", | ||||
|     "nock": "13.5.5", | ||||
|     "nodemon": "^2.0.4", | ||||
|     "otplib": "12.0.1", | ||||
|     "proper-lockfile": "4.1.2", | ||||
|  | ||||
| @ -34,6 +34,7 @@ async function activateServer(home: FlexServer, docManager: DocManager) { | ||||
|   home.addJsonSupport(); | ||||
|   await home.addLandingPages(); | ||||
|   home.addHomeApi(); | ||||
|   home.addAuditLogger(); | ||||
|   await home.addTelemetry(); | ||||
|   await home.addDoc(); | ||||
|   home.addApiErrorHandlers(); | ||||
|  | ||||
							
								
								
									
										117
									
								
								test/server/lib/GristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										117
									
								
								test/server/lib/GristAuditLogger.ts
									
									
									
									
									
										Normal file
									
								
							| @ -0,0 +1,117 @@ | ||||
| import {IAuditLogger} from 'app/server/lib/AuditLogger'; | ||||
| import {LogMethods} from 'app/server/lib/LogMethods'; | ||||
| import {assert} from 'chai'; | ||||
| import moment from 'moment-timezone'; | ||||
| import nock from 'nock'; | ||||
| import * as sinon from 'sinon'; | ||||
| import {TestServer} from 'test/gen-server/apiUtils'; | ||||
| import * as testUtils from 'test/server/testUtils'; | ||||
| 
 | ||||
| describe('GristAuditLogger', function() { | ||||
|   let auditLogger: IAuditLogger; | ||||
|   let oldEnv: testUtils.EnvironmentSnapshot; | ||||
|   let server: TestServer; | ||||
| 
 | ||||
|   before(async function() { | ||||
|     oldEnv = new testUtils.EnvironmentSnapshot(); | ||||
|     process.env.GRIST_AUDIT_HTTP_ENDPOINT = 'https://api.getgrist.com/events'; | ||||
|     process.env.GRIST_AUDIT_HTTP_AUTHORIZATION_HEADER = 'Grist bb48d1f8-8f6c-4065-8951-8543a8e70597'; | ||||
|     server = new TestServer(this); | ||||
|     await server.start(); | ||||
|     auditLogger = server.server.getAuditLogger(); | ||||
|   }); | ||||
| 
 | ||||
|   after(async function() { | ||||
|     await server.stop(); | ||||
|     oldEnv.restore(); | ||||
|   }); | ||||
| 
 | ||||
|   describe('logEventAsync', function() { | ||||
|     const sandbox: sinon.SinonSandbox = sinon.createSandbox(); | ||||
|     let logErrorCallArguments: any[] = []; | ||||
| 
 | ||||
|     before(async function() { | ||||
|       sandbox | ||||
|         .stub(LogMethods.prototype, 'error') | ||||
|         .callsFake((...args) => logErrorCallArguments.push([...args])); | ||||
|     }); | ||||
| 
 | ||||
|     after(async function() { | ||||
|       sandbox.restore(); | ||||
|     }); | ||||
| 
 | ||||
|     afterEach(function() { | ||||
|       logErrorCallArguments = []; | ||||
|       nock.cleanAll(); | ||||
|     }); | ||||
| 
 | ||||
|     it('logs audit events', async function() { | ||||
|       const timestamp = moment().toISOString(); | ||||
|       const scope = nock('https://api.getgrist.com') | ||||
|         .matchHeader('Authorization', 'Grist bb48d1f8-8f6c-4065-8951-8543a8e70597') | ||||
|         .post('/events', { | ||||
|           event: { | ||||
|             name: 'createDocument', | ||||
|             user: null, | ||||
|             details: { | ||||
|               id: 'docId', | ||||
|             }, | ||||
|           }, | ||||
|           timestamp, | ||||
|         }) | ||||
|         .reply(200); | ||||
|       await assert.isFulfilled( | ||||
|         auditLogger.logEventAsync(null, { | ||||
|           event: { | ||||
|             name: 'createDocument', | ||||
|             details: {id: 'docId'}, | ||||
|           }, | ||||
|           timestamp, | ||||
|         }) | ||||
|       ); | ||||
|       assert.isTrue(scope.isDone()); | ||||
|     }); | ||||
| 
 | ||||
|     it('throws on failure to log', async function() { | ||||
|       nock('https://api.getgrist.com') | ||||
|         .post('/events') | ||||
|         .reply(404, 'Not found'); | ||||
|       await assert.isRejected( | ||||
|         auditLogger.logEventAsync(null, { | ||||
|           event: { | ||||
|             name: 'createDocument', | ||||
|             details: {id: 'docId'}, | ||||
|           }, | ||||
|         }), | ||||
|         'received a non-200 response from https://api.getgrist.com/events: 404 Not found' | ||||
|       ); | ||||
|     }); | ||||
| 
 | ||||
|     it('throws if max pending requests exceeded', async function() { | ||||
|       nock('https://api.getgrist.com') | ||||
|         .persist() | ||||
|         .post('/events') | ||||
|         .delay(2000) | ||||
|         .reply(200); | ||||
|       // Queue up enough pending requests to reach the limit (25).
 | ||||
|       for (let i = 0; i < 25; i++) { | ||||
|         void auditLogger.logEvent(null, { | ||||
|           event: { | ||||
|             name: 'createDocument', | ||||
|             details: {id: 'docId'}, | ||||
|           }, | ||||
|         }); | ||||
|       } | ||||
|       await assert.isRejected( | ||||
|         auditLogger.logEventAsync(null, { | ||||
|           event: { | ||||
|             name: 'createDocument', | ||||
|             details: {id: 'docId'}, | ||||
|           }, | ||||
|         }), | ||||
|         'exceeded the maximum number of pending audit event calls (25)' | ||||
|       ); | ||||
|       nock.abortPendingRequests(); | ||||
|     }); | ||||
|   }); | ||||
| }); | ||||
							
								
								
									
										19
									
								
								yarn.lock
									
									
									
									
									
								
							
							
						
						
									
										19
									
								
								yarn.lock
									
									
									
									
									
								
							| @ -5139,6 +5139,11 @@ json-stable-stringify@~0.0.0: | ||||
|   dependencies: | ||||
|     jsonify "~0.0.0" | ||||
| 
 | ||||
| json-stringify-safe@^5.0.1: | ||||
|   version "5.0.1" | ||||
|   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" | ||||
|   integrity sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA== | ||||
| 
 | ||||
| json5@^1.0.1: | ||||
|   version "1.0.2" | ||||
|   resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" | ||||
| @ -5988,6 +5993,15 @@ nise@^5.1.5: | ||||
|     just-extend "^6.2.0" | ||||
|     path-to-regexp "^6.2.1" | ||||
| 
 | ||||
| nock@13.5.5: | ||||
|   version "13.5.5" | ||||
|   resolved "https://registry.yarnpkg.com/nock/-/nock-13.5.5.tgz#cd1caaca281d42be17d51946367a3d53a6af3e78" | ||||
|   integrity sha512-XKYnqUrCwXC8DGG1xX4YH5yNIrlh9c065uaMZZHUoeUUINTOyt+x/G+ezYk0Ft6ExSREVIs+qBJDK503viTfFA== | ||||
|   dependencies: | ||||
|     debug "^4.1.0" | ||||
|     json-stringify-safe "^5.0.1" | ||||
|     propagate "^2.0.0" | ||||
| 
 | ||||
| node-abort-controller@3.0.1: | ||||
|   version "3.0.1" | ||||
|   resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" | ||||
| @ -6631,6 +6645,11 @@ promise-retry@^2.0.1: | ||||
|     err-code "^2.0.2" | ||||
|     retry "^0.12.0" | ||||
| 
 | ||||
| propagate@^2.0.0: | ||||
|   version "2.0.1" | ||||
|   resolved "https://registry.yarnpkg.com/propagate/-/propagate-2.0.1.tgz#40cdedab18085c792334e64f0ac17256d38f9a45" | ||||
|   integrity sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag== | ||||
| 
 | ||||
| proper-lockfile@4.1.2: | ||||
|   version "4.1.2" | ||||
|   resolved "https://registry.yarnpkg.com/proper-lockfile/-/proper-lockfile-4.1.2.tgz#c8b9de2af6b2f1601067f98e01ac66baa223141f" | ||||
|  | ||||
		Loading…
	
		Reference in New Issue
	
	Block a user