(core) Add more audit logging data/events

Summary:
Adds a few additional audit events and enhances audit logging
to capture more data (request origin, active org, user type).

Test Plan: Server and manual tests.

Reviewers: jarek

Reviewed By: jarek

Subscribers: jarek

Differential Revision: https://phab.getgrist.com/D4348

app/common/AuditEvent.ts

@@ -1,31 +1,224 @@
 export interface AuditEvent<Name extends AuditEventName> {
+  /**
+   * The event.
+   */
   event: {
-    /** The event name. */
+    /**
+     * The name of the event.
+     */
     name: Name;
-    /** The user that triggered the event. */
+    /**
-    user: AuditEventUser | null;
+     * The user that triggered the event.
-    /** Additional event details. */
+     */
-    details: AuditEventDetails[Name] | null;
+    user: AuditEventUser;
+    /**
+     * The event details.
+     */
+    details: AuditEventDetails[Name] | {};
+    /**
+     * The context of the event.
+     */
+    context: AuditEventContext;
+    /**
+     * The source of the event.
+     */
+    source: AuditEventSource;
   };
-  /** ISO 8601 timestamp of when the event was logged. */
+  /**
+   * ISO 8601 timestamp of when the event occurred.
+   */
   timestamp: string;
 }
 
 export type AuditEventName =
-  | 'createDocument';
+  | 'createDocument'
+  | 'moveDocument'
+  | 'removeDocument'
+  | 'deleteDocument'
+  | 'restoreDocumentFromTrash'
+  | 'runSQLQuery';
 
-export interface AuditEventUser {
+export type AuditEventUser =
-  /** The user's id. */
+  | User
-  id: number | null;
+  | Anonymous
-  /** The user's email address. */
+  | Unknown;
-  email: string | null;
+
-  /** The user's name. */
+interface User {
-  name: string | null;
+  type: 'user';
+  id: number;
+  email: string;
+  name: string;
+}
+
+interface Anonymous {
+  type: 'anonymous';
+}
+
+interface Unknown {
+  type: 'unknown';
 }
 
 export interface AuditEventDetails {
+  /**
+   * A new document was created.
+   */
   createDocument: {
-    /** The ID of the document. */
+    /**
+     * The ID of the document.
+     */
     id: string;
+    /**
+     * The name of the document.
+     */
+    name?: string;
+  };
+  /**
+   * A document was moved to a new workspace.
+   */
+  moveDocument: {
+    /**
+     * The ID of the document.
+     */
+    id: string;
+    /**
+     * The previous workspace.
+     */
+    previous: {
+      /**
+       * The workspace the document was moved from.
+       */
+      workspace: {
+        /**
+         * The ID of the workspace.
+         */
+        id: number;
+        /**
+         * The name of the workspace.
+         */
+        name: string;
+      };
+    };
+    /**
+     * The current workspace.
+     */
+    current: {
+      /**
+       * The workspace the document was moved to.
+       */
+      workspace: {
+        /**
+         * The ID of the workspace.
+         */
+        id: number;
+        /**
+         * The name of the workspace.
+         */
+        name: string;
+      };
+    };
+  };
+  /**
+   * A document was moved to the trash.
+   */
+  removeDocument: {
+    /**
+     * The ID of the document.
+     */
+    id: string;
+    /**
+     * The name of the document.
+     */
+    name: string;
+  };
+  /**
+   * A document was permanently deleted.
+   */
+  deleteDocument: {
+    /**
+     * The ID of the document.
+     */
+    id: string;
+    /**
+     * The name of the document.
+     */
+    name: string;
+  };
+  /**
+   * A document was restored from the trash.
+   */
+  restoreDocumentFromTrash: {
+    /**
+     * The restored document.
+     */
+    document: {
+      /**
+       * The ID of the document.
+       */
+      id: string;
+      /**
+       * The name of the document.
+       */
+      name: string;
+    };
+    /**
+     * The workspace of the restored document.
+     */
+    workspace: {
+      /**
+       * The ID of the workspace.
+       */
+      id: number;
+      /**
+       * The name of the workspace.
+       */
+      name: string;
+    };
+  };
+  /**
+   * A SQL query was run against a document.
+   */
+  runSQLQuery: {
+    /**
+     * The SQL query.
+     */
+    query: string;
+    /**
+     * The arguments used for query parameters, if any.
+     */
+    arguments?: (string | number)[];
+    /**
+     * The duration in milliseconds until query execution should time out.
+     */
+    timeout?: number;
   };
 }
+
+export interface AuditEventContext {
+  /**
+   * The ID of the workspace the event occurred in.
+   */
+  workspaceId?: number;
+  /**
+   * The ID of the document the event occurred in.
+   */
+  documentId?: string;
+}
+
+export interface AuditEventSource {
+  /**
+   * The domain of the org tied to the originating request.
+   */
+  org?: string;
+  /**
+   * The IP address of the originating request.
+   */
+  ipAddress?: string;
+  /**
+   * The User-Agent HTTP header of the originating request.
+   */
+  userAgent?: string;
+  /**
+   * The ID of the session tied to the originating request.
+   */
+  sessionId?: string;
+}

app/gen-server/ApiServer.ts

@@ -8,8 +8,10 @@ import {ApiError} from 'app/common/ApiError';
 import {FullUser} from 'app/common/LoginSessionAPI';
 import {BasicRole} from 'app/common/roles';
 import {OrganizationProperties, PermissionDelta} from 'app/common/UserAPI';
+import {Document} from "app/gen-server/entity/Document";
 import {User} from 'app/gen-server/entity/User';
-import {BillingOptions, HomeDBManager, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {BillingOptions, HomeDBManager, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {PreviousAndCurrent, QueryResult} from 'app/gen-server/lib/homedb/Interfaces';
 import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession';
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -259,37 +261,10 @@ export class ApiServer {
     // POST /api/workspaces/:wid/docs
     // Create a new doc owned by the specific workspace.
     this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => {
-      const mreq = req as RequestWithLogin;
       const wsId = integerParam(req.params.wid, 'wid');
-      const query = await this._dbManager.addDocument(getScope(req), wsId, req.body);
+      const result = await this._dbManager.addDocument(getScope(req), wsId, req.body);
-      const docId = query.data!;
+      if (result.status === 200) { this._logCreateDocumentEvents(req, result.data!); }
-      this._gristServer.getTelemetry().logEvent(mreq, 'documentCreated', {
+      return sendReply(req, res, {...result, data: result.data?.id});
-        limited: {
-          docIdDigest: docId,
-          sourceDocIdDigest: undefined,
-          isImport: false,
-          fileType: undefined,
-          isSaved: true,
-        },
-        full: {
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      });
-      this._gristServer.getTelemetry().logEvent(mreq, 'createdDoc-Empty', {
-        full: {
-          docIdDigest: docId,
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      });
-      this._gristServer.getAuditLogger().logEvent(mreq, {
-        event: {
-          name: 'createDocument',
-          details: {id: docId},
-        },
-      });
-      return sendReply(req, res, query);
     }));
 
     // GET /api/templates/
@@ -334,7 +309,8 @@ export class ApiServer {
     // Recover the specified doc if it was previously soft-deleted and is
     // still available.
     this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => {
-      await this._dbManager.undeleteDocument(getDocScope(req));
+      const {status, data} = await this._dbManager.undeleteDocument(getDocScope(req));
+      if (status === 200) { this._logRestoreDocumentEvents(req, data!); }
       return sendOkReply(req, res);
     }));
 
@@ -375,9 +351,10 @@ export class ApiServer {
     // PATCH /api/docs/:did/move
     // Move the doc to the workspace specified in the body.
     this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => {
-      const workspaceId = req.body.workspace;
+      const workspaceId = integerParam(req.body.workspace, 'workspace');
-      const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId);
+      const result = await this._dbManager.moveDoc(getDocScope(req), workspaceId);
-      return sendReply(req, res, query);
+      if (result.status === 200) { this._logMoveDocumentEvents(req, result.data!); }
+      return sendReply(req, res, {...result, data: result.data?.current.id});
     }));
 
     this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => {
@@ -647,6 +624,57 @@ export class ApiServer {
     return result;
   }
 
+  private _logCreateDocumentEvents(req: Request, document: Document) {
+    const mreq = req as RequestWithLogin;
+    const {id, name, workspace: {id: workspaceId}} = document;
+    this._gristServer.getAuditLogger().logEvent(mreq, {
+      event: {
+        name: 'createDocument',
+        details: {id, name},
+        context: {workspaceId},
+      },
+    });
+    this._gristServer.getTelemetry().logEvent(mreq, 'documentCreated', {
+      limited: {
+        docIdDigest: id,
+        sourceDocIdDigest: undefined,
+        isImport: false,
+        fileType: undefined,
+        isSaved: true,
+      },
+      full: {
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+      },
+    });
+    this._gristServer.getTelemetry().logEvent(mreq, 'createdDoc-Empty', {
+      full: {
+        docIdDigest: id,
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+      },
+    });
+  }
+
+  private _logRestoreDocumentEvents(req: Request, document: Document) {
+    const {workspace} = document;
+    this._gristServer.getAuditLogger().logEvent(req as RequestWithLogin, {
+      event: {
+        name: 'restoreDocumentFromTrash',
+        details: {
+          document: {
+            id: document.id,
+            name: document.name,
+          },
+          workspace: {
+            id: workspace.id,
+            name: workspace.name,
+          },
+        },
+      },
+    });
+  }
+
   private _logInvitedDocUserTelemetryEvents(mreq: RequestWithLogin, delta: PermissionDelta) {
     if (!delta.users) { return; }
 
@@ -687,6 +715,32 @@ export class ApiServer {
       );
     }
   }
+
+  private _logMoveDocumentEvents(req: Request, {previous, current}: PreviousAndCurrent<Document>) {
+    this._gristServer.getAuditLogger().logEvent(req as RequestWithLogin, {
+      event: {
+        name: 'moveDocument',
+        details: {
+          id: current.id,
+          previous: {
+            workspace: {
+              id: previous.workspace.id,
+              name: previous.workspace.name,
+            },
+          },
+          current: {
+            workspace: {
+              id: current.workspace.id,
+              name: current.workspace.name,
+            },
+          },
+        },
+        context: {
+          workspaceId: previous.workspace.id,
+        },
+      },
+    });
+  }
 }
 
 /**

app/gen-server/lib/homedb/HomeDBManager.ts

@@ -41,7 +41,13 @@ import {User} from "app/gen-server/entity/User";
 import {Workspace} from "app/gen-server/entity/Workspace";
 import {Limit} from 'app/gen-server/entity/Limit';
 import {
-  AvailableUsers, GetUserOptions, NonGuestGroup, Resource, UserProfileChange
+  AvailableUsers,
+  GetUserOptions,
+  NonGuestGroup,
+  PreviousAndCurrent,
+  QueryResult,
+  Resource,
+  UserProfileChange,
 } from 'app/gen-server/lib/homedb/Interfaces';
 import {SUPPORT_EMAIL, UsersManager} from 'app/gen-server/lib/homedb/UsersManager';
 import {Permissions} from 'app/gen-server/lib/Permissions';
@@ -111,12 +117,6 @@ const listPublicSites = appSettings.section('access').flag('listPublicSites').re
 // which is a burden under heavy traffic.
 const DOC_AUTH_CACHE_TTL = 5000;
 
-export interface QueryResult<T> {
-  status: number;
-  data?: T;
-  errMessage?: string;
-}
-
 // Maps from userId to group name, or null to inherit.
 export interface UserIdDelta {
   [userId: string]: roles.NonGuestRole|null;
@@ -1496,8 +1496,12 @@ export class HomeDBManager extends EventEmitter {
   // by makeId().  The client should not be given control of the choice of docId.
   // This option is used during imports, where it is convenient not to add a row to the
   // document database until the document has actually been imported.
-  public async addDocument(scope: Scope, wsId: number, props: Partial<DocumentProperties>,
+  public async addDocument(
-                           docId?: string): Promise<QueryResult<string>> {
+    scope: Scope,
+    wsId: number,
+    props: Partial<DocumentProperties>,
+    docId?: string
+  ): Promise<QueryResult<Document>> {
     const name = props.name;
     if (!name) {
       return {
@@ -1577,7 +1581,12 @@ export class HomeDBManager extends EventEmitter {
       });
       // Saves the document as well as its new ACL Rules and Group.
       const groups = doc.aclRules.map(rule => rule.group);
-      const result = await manager.save([doc, ...doc.aclRules, ...doc.aliases, ...groups]);
+      const [data] = await manager.save<[Document, ...(AclRuleDoc|Alias|Group)[]]>([
+        doc,
+        ...doc.aclRules,
+        ...doc.aliases,
+        ...groups,
+      ]);
       // Ensure that the creator is in the ws and org's guests group. Creator already has
       // access to the workspace (he is at least an editor), but we need to be sure that
       // even if he is removed from the workspace, he will still have access to this doc.
@@ -1587,10 +1596,7 @@ export class HomeDBManager extends EventEmitter {
       // time), but they are ignoring any unique constraints errors.
       await this._repairWorkspaceGuests(scope, workspace.id, manager);
       await this._repairOrgGuests(scope, workspace.org.id, manager);
-      return {
+      return {status: 200, data};
-        status: 200,
-        data: (result[0] as Document).id
-      };
     });
   }
 
@@ -1751,9 +1757,9 @@ export class HomeDBManager extends EventEmitter {
   }
 
   // Checks that the user has REMOVE permissions to the given document. If not, throws an
-  // error. Otherwise deletes the given document. Returns an empty query result with
+  // error. Otherwise deletes the given document. Returns a query result with status 200
-  // status 200 on success.
+  // and the deleted document on success.
-  public async deleteDocument(scope: DocScope): Promise<QueryResult<number>> {
+  public async deleteDocument(scope: DocScope): Promise<QueryResult<Document>> {
     return await this._connection.transaction(async manager => {
       const {forkId} = parseUrlId(scope.urlId);
       if (forkId) {
@@ -1767,8 +1773,9 @@ export class HomeDBManager extends EventEmitter {
           return queryResult;
         }
         const fork: Document = queryResult.data;
-        await manager.remove([fork]);
+        const data = structuredClone(fork);
-        return {status: 200};
+        await manager.remove(fork);
+        return {status: 200, data};
       } else {
         const docQuery = this._doc(scope, {
           manager,
@@ -1785,22 +1792,23 @@ export class HomeDBManager extends EventEmitter {
           return queryResult;
         }
         const doc: Document = queryResult.data;
-        // Delete the doc and doc ACLs/groups.
+        const data = structuredClone(doc);
         const docGroups = doc.aclRules.map(docAcl => docAcl.group);
+        // Delete the doc and doc ACLs/groups.
         await manager.remove([doc, ...docGroups, ...doc.aclRules]);
         // Update guests of the workspace and org after removing this doc.
         await this._repairWorkspaceGuests(scope, doc.workspace.id, manager);
         await this._repairOrgGuests(scope, doc.workspace.org.id, manager);
-        return {status: 200};
+        return {status: 200, data};
       }
     });
   }
 
-  public softDeleteDocument(scope: DocScope): Promise<void> {
+  public softDeleteDocument(scope: DocScope): Promise<QueryResult<Document>> {
     return this._setDocumentRemovedAt(scope, new Date());
   }
 
-  public async undeleteDocument(scope: DocScope): Promise<void> {
+  public async undeleteDocument(scope: DocScope): Promise<QueryResult<Document>> {
     return this._setDocumentRemovedAt(scope, null);
   }
 
@@ -2263,7 +2271,7 @@ export class HomeDBManager extends EventEmitter {
   public async moveDoc(
     scope: DocScope,
     wsId: number
-  ): Promise<QueryResult<void>> {
+  ): Promise<QueryResult<PreviousAndCurrent<Document>>> {
     return await this._connection.transaction(async manager => {
       // Get the doc
       const docQuery = this._doc(scope, {
@@ -2285,6 +2293,7 @@ export class HomeDBManager extends EventEmitter {
         return docQueryResult;
       }
       const doc: Document = docQueryResult.data;
+      const previous = structuredClone(doc);
       if (doc.workspace.id === wsId) {
         return {
           status: 400,
@@ -2354,7 +2363,11 @@ export class HomeDBManager extends EventEmitter {
       doc.aliases = undefined as any;
       // Saves the document as well as its new ACL Rules and Groups and the
       // updated guest group in the workspace.
-      await manager.save([doc, ...doc.aclRules, ...docGroups]);
+      const [current] = await manager.save<[Document, ...(AclRuleDoc|Group)[]]>([
+        doc,
+        ...doc.aclRules,
+        ...docGroups,
+      ]);
       if (firstLevelUsers.length > 0) {
         // If the doc has first-level users, update the source and destination workspaces.
         await this._repairWorkspaceGuests(scope, oldWs.id, manager);
@@ -2365,9 +2378,7 @@ export class HomeDBManager extends EventEmitter {
           await this._repairOrgGuests(scope, doc.workspace.org.id, manager);
         }
       }
-      return {
+      return {status: 200, data: {previous, current}};
-        status: 200
-      };
     });
   }
 
@@ -4301,9 +4312,9 @@ export class HomeDBManager extends EventEmitter {
       if (!removedAt) {
         await this._checkRoomForAnotherDoc(doc.workspace, manager);
       }
-      await manager.createQueryBuilder()
+      doc.removedAt = removedAt;
-        .update(Document).set({removedAt}).where({id: doc.id})
+      const data = await manager.save(doc);
-        .execute();
+      return {status: 200, data};
     });
   }
 

app/gen-server/lib/homedb/Interfaces.ts

@@ -14,6 +14,11 @@ export interface QueryResult<T> {
   errMessage?: string;
 }
 
+export interface PreviousAndCurrent<T> {
+  previous: T;
+  current: T;
+}
+
 export interface GetUserOptions {
   manager?: EntityManager;
   profile?: UserProfile;

app/server/lib/ActiveDoc.ts

@@ -107,6 +107,15 @@ import {LogMethods} from "app/server/lib/LogMethods";
 import {ISandboxOptions} from 'app/server/lib/NSandbox';
 import {NullSandbox, UnavailableSandboxMethodError} from 'app/server/lib/NullSandbox';
 import {DocRequests} from 'app/server/lib/Requests';
+import {
+  getAltSessionId,
+  getDocSessionAccess,
+  getDocSessionAccessOrNull,
+  getDocSessionUsage,
+  getFullUser,
+  getLogMeta,
+  getUserId,
+} from 'app/server/lib/sessionUtils';
 import {shortDesc} from 'app/server/lib/shortDesc';
 import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader';
 import {DocTriggers} from "app/server/lib/Triggers";
@@ -127,21 +136,12 @@ import {ActionHistoryImpl} from './ActionHistoryImpl';
 import {ActiveDocImport, FileImportOptions} from './ActiveDocImport';
 import {DocClients} from './DocClients';
 import {DocPluginManager} from './DocPluginManager';
-import {
+import {DocSession, makeExceptionalDocSession, OptDocSession} from './DocSession';
-  DocSession,
-  getDocSessionAccess,
-  getDocSessionAltSessionId,
-  getDocSessionUsage,
-  getDocSessionUser,
-  getDocSessionUserId,
-  makeExceptionalDocSession,
-  OptDocSession
-} from './DocSession';
 import {createAttachmentsIndex, DocStorage, REMOVE_UNUSED_ATTACHMENTS_DELAY} from './DocStorage';
 import {expandQuery, getFormulaErrorForExpandQuery} from './ExpandedQuery';
 import {GranularAccess, GranularAccessForBundle} from './GranularAccess';
 import {OnDemandActions} from './OnDemandActions';
-import {getLogMetaFromDocSession, getPubSubPrefix, getTelemetryMetaFromDocSession} from './serverUtils';
+import {getPubSubPrefix} from './serverUtils';
 import {findOrAddAllEnvelope, Sharing} from './Sharing';
 import cloneDeep = require('lodash/cloneDeep');
 import flatten = require('lodash/flatten');
@@ -455,7 +455,7 @@ export class ActiveDoc extends EventEmitter {
   // Constructs metadata for logging, given a Client or an OptDocSession.
   public getLogMeta(docSession: OptDocSession|null, docMethod?: string): log.ILogMeta {
     return {
-      ...(docSession ? getLogMetaFromDocSession(docSession) : {}),
+      ...getLogMeta(docSession),
       docId: this._docName,
       ...(docMethod ? {docMethod} : {}),
     };
@@ -819,7 +819,7 @@ export class ActiveDoc extends EventEmitter {
    * It returns the list of rowIds for the rows created in the _grist_Attachments table.
    */
   public async addAttachments(docSession: OptDocSession, uploadId: number): Promise<number[]> {
-    const userId = getDocSessionUserId(docSession);
+    const userId = getUserId(docSession);
     const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId));
     try {
       // We'll assert that the upload won't cause limits to be exceeded, retrying once after
@@ -1212,7 +1212,7 @@ export class ActiveDoc extends EventEmitter {
     const options = sanitizeApplyUAOptions(unsanitizedOptions);
     const actionBundles = await this._actionHistory.getActions(actionNums);
     let fromOwnHistory: boolean = true;
-    const user = getDocSessionUser(docSession);
+    const user = getFullUser(docSession);
     let oldestSource: number = Date.now();
     for (const [index, bundle] of actionBundles.entries()) {
       const actionNum = actionNums[index];
@@ -1403,7 +1403,7 @@ export class ActiveDoc extends EventEmitter {
    */
   public async fork(docSession: OptDocSession): Promise<ForkResult> {
     const dbManager = this._getHomeDbManagerOrFail();
-    const user = getDocSessionUser(docSession);
+    const user = getFullUser(docSession);
     // For now, fork only if user can read everything (or is owner).
     // TODO: allow forks with partial content.
     if (!user || !await this.canDownload(docSession)) {
@@ -1471,7 +1471,7 @@ export class ActiveDoc extends EventEmitter {
 
   public async getAccessToken(docSession: OptDocSession, options: AccessTokenOptions): Promise<AccessTokenResult> {
     const tokens = this._docManager.gristServer.getAccessTokens();
-    const userId = getDocSessionUserId(docSession);
+    const userId = getUserId(docSession);
     const docId = this.docName;
     const access = getDocSessionAccess(docSession);
     // If we happen to be using a "readOnly" connection, max out at "readOnly"
@@ -1585,7 +1585,7 @@ export class ActiveDoc extends EventEmitter {
     };
     const isShared = new Set<string>();
 
-    const userId = getDocSessionUserId(docSession);
+    const userId = getUserId(docSession);
     if (!userId) { throw new Error('Cannot determine user'); }
 
     const parsed = parseUrlId(this.docName);
@@ -2747,9 +2747,9 @@ export class ActiveDoc extends EventEmitter {
   }
 
   private _getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel {
-    const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined;
+    const altSessionId = getAltSessionId(docSession);
     return merge(
-      docSession ? getTelemetryMetaFromDocSession(docSession) : {},
+      getTelemetryMeta(docSession),
       altSessionId ? {altSessionId} : {},
       {
         limited: {
@@ -3048,3 +3048,23 @@ export function createSandbox(options: {
     sandboxOptions,
   });
 }
+
+/**
+ * Extract telemetry metadata from session.
+ */
+function getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel {
+  if (!docSession) { return {}; }
+
+  const access = getDocSessionAccessOrNull(docSession);
+  const user = getFullUser(docSession);
+  const {client} = docSession;
+  return {
+    limited: {
+      access,
+    },
+    full: {
+      ...(user ? {userId: user.id} : {}),
+      ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
+    },
+  };
+}

app/server/lib/Assistance.ts

@@ -6,16 +6,16 @@ import {
   AssistanceContext,
   AssistanceMessage,
   AssistanceRequest,
-  AssistanceResponse
+  AssistanceResponse,
 } from 'app/common/AssistancePrompts';
 import {delay} from 'app/common/delay';
 import {DocAction} from 'app/common/DocActions';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession';
+import {OptDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
+import {getFullUser, getLogMeta} from 'app/server/lib/sessionUtils';
+import {createHash} from 'crypto';
 import fetch from 'node-fetch';
-import {createHash} from "crypto";
-import {getLogMetaFromDocSession} from "./serverUtils";
 
 // These are mocked/replaced in tests.
 // fetch is also replacing in the runCompletion script to add caching.
@@ -559,13 +559,13 @@ async function completionToResponse(
 }
 
 function getUserHash(session: OptDocSession): string {
-  const user = getDocSessionUser(session);
+  const user = getFullUser(session);
   // Make it a bit harder to guess the user ID.
   const salt = "7a8sb6987asdb678asd687sad6boas7f8b6aso7fd";
   const hashSource = `${user?.id} ${user?.ref} ${salt}`;
   const hash = createHash('sha256').update(hashSource).digest('base64');
   // So that if we get feedback about a user ID hash, we can
   // search for the hash in the logs to find the original user ID.
-  log.rawInfo("getUserHash", {...getLogMetaFromDocSession(session), userRef: user?.ref, hash});
+  log.rawInfo("getUserHash", {...getLogMeta(session), userRef: user?.ref, hash});
   return hash;
 }

app/server/lib/AuditLogger.ts

@@ -1,5 +1,5 @@
-import {AuditEventDetails, AuditEventName} from 'app/common/AuditEvent';
+import {AuditEvent, AuditEventContext, AuditEventDetails, AuditEventName} from 'app/common/AuditEvent';
-import {RequestOrSession} from 'app/server/lib/requestUtils';
+import {RequestOrSession} from 'app/server/lib/sessionUtils';
 
 export interface IAuditLogger {
   /**
@@ -7,16 +7,16 @@ export interface IAuditLogger {
    */
   logEvent<Name extends AuditEventName>(
     requestOrSession: RequestOrSession,
-    props: AuditEventProperties<Name>
+    properties: AuditEventProperties<Name>
   ): void;
   /**
-   * Asynchronous variant of `logEvent`.
+   * Logs an audit event.
    *
-   * Throws on failure to log an event.
+   * Throws a `LogAuditEventError` on failure.
    */
   logEventAsync<Name extends AuditEventName>(
     requestOrSession: RequestOrSession,
-    props: AuditEventProperties<Name>
+    properties: AuditEventProperties<Name>
   ): Promise<void>;
 }
 
@@ -30,6 +30,10 @@ export interface AuditEventProperties<Name extends AuditEventName> {
      * Additional event details.
      */
     details?: AuditEventDetails[Name];
+    /**
+     * The context of the event.
+     */
+    context?: AuditEventContext;
   };
   /**
    * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured.
@@ -38,3 +42,15 @@ export interface AuditEventProperties<Name extends AuditEventName> {
    */
   timestamp?: string;
 }
+
+export class LogAuditEventError<Name extends AuditEventName> extends Error {
+  public name = 'LogAuditEventError';
+
+  constructor(public auditEvent: AuditEvent<Name>, ...params: any[]) {
+    super(...params);
+
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, LogAuditEventError);
+    }
+  }
+}

app/server/lib/Authorizer.ts

@@ -18,7 +18,7 @@ import {makeId} from 'app/server/lib/idUtils';
 import log from 'app/server/lib/log';
 import {IPermitStore, Permit} from 'app/server/lib/Permit';
 import {AccessTokenInfo} from 'app/server/lib/AccessTokens';
-import {allowHost, getOriginUrl, optStringParam} from 'app/server/lib/requestUtils';
+import {allowHost, buildXForwardedForHeader, getOriginUrl, optStringParam} from 'app/server/lib/requestUtils';
 import * as cookie from 'cookie';
 import {NextFunction, Request, RequestHandler, Response} from 'express';
 import {IncomingMessage} from 'http';
@@ -704,6 +704,7 @@ export function getTransitiveHeaders(
   const PermitHeader = req.get('Permit');
   const Organization = (req as RequestWithOrg).org;
   const XRequestedWith = req.get('X-Requested-With');
+  const UserAgent = req.get('User-Agent');
   const Origin = req.get('Origin');  // Pass along the original Origin since it may
                                      // play a role in granular access control.
 
@@ -713,6 +714,8 @@ export function getTransitiveHeaders(
     ...(Organization ? { Organization } : undefined),
     ...(PermitHeader ? { Permit: PermitHeader } : undefined),
     ...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined),
+    ...(UserAgent ? { 'User-Agent': UserAgent } : undefined),
+    ...buildXForwardedForHeader(req),
     ...((includeOrigin && Origin) ? { Origin } : undefined),
   };
   const extraHeader = process.env.GRIST_FORWARD_AUTH_HEADER;

app/server/lib/Client.ts

@@ -1,28 +1,29 @@
 import {ApiError} from 'app/common/ApiError';
 import {BrowserSettings} from 'app/common/BrowserSettings';
-import {delay} from 'app/common/delay';
 import {CommClientConnect, CommMessage, CommResponse, CommResponseError} from 'app/common/CommTypes';
+import {delay} from 'app/common/delay';
+import {normalizeEmail} from 'app/common/emails';
 import {ErrorWithCode} from 'app/common/ErrorWithCode';
 import {FullUser, UserProfile} from 'app/common/LoginSessionAPI';
 import {TelemetryMetadata} from 'app/common/Telemetry';
 import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI';
-import {normalizeEmail} from 'app/common/emails';
 import {User} from 'app/gen-server/entity/User';
-import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {Authorizer} from 'app/server/lib/Authorizer';
 import {ScopedSession} from 'app/server/lib/BrowserSession';
 import type {Comm} from 'app/server/lib/Comm';
 import {DocSession} from 'app/server/lib/DocSession';
-import log from 'app/server/lib/log';
-import {LogMethods} from "app/server/lib/LogMethods";
-import {MemoryPool} from 'app/server/lib/MemoryPool';
-import {shortDesc} from 'app/server/lib/shortDesc';
-import {fromCallback} from 'app/server/lib/serverUtils';
-import {i18n} from 'i18next';
-import * as crypto from 'crypto';
-import moment from 'moment';
 import {GristServerSocket} from 'app/server/lib/GristServerSocket';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
+import log from 'app/server/lib/log';
+import {LogMethods} from 'app/server/lib/LogMethods';
+import {MemoryPool} from 'app/server/lib/MemoryPool';
+import {fromCallback} from 'app/server/lib/serverUtils';
+import {shortDesc} from 'app/server/lib/shortDesc';
+import * as crypto from 'crypto';
+import {IncomingMessage} from 'http';
+import {i18n} from 'i18next';
+import moment from 'moment';
 
 // How many messages and bytes to accumulate for a disconnected client before booting it.
 // The benefit is that a client who temporarily disconnects and reconnects without missing much,
@@ -97,7 +98,8 @@ export class Client {
   private _missedMessagesTotalLength: number = 0;
   private _destroyTimer: NodeJS.Timer|null = null;
   private _destroyed: boolean = false;
-  private _websocket: GristServerSocket|null;
+  private _websocket: GristServerSocket|null = null;
+  private _req: IncomingMessage|null = null;
   private _org: string|null = null;
   private _profile: UserProfile|null = null;
   private _user: FullUser|undefined = undefined;
@@ -130,8 +132,15 @@ export class Client {
     return this._locale;
   }
 
-  public setConnection(websocket: GristServerSocket, counter: string|null, browserSettings: BrowserSettings) {
+  public setConnection(options: {
+    websocket: GristServerSocket;
+    req: IncomingMessage;
+    counter: string|null;
+    browserSettings: BrowserSettings;
+  }) {
+    const {websocket, req, counter, browserSettings} = options;
     this._websocket = websocket;
+    this._req = req;
     this._counter = counter;
     this.browserSettings = browserSettings;
 
@@ -140,6 +149,10 @@ export class Client {
     websocket.onmessage = (msg: string) => this._onMessage(msg);
   }
 
+  public getConnectionRequest(): IncomingMessage|null {
+    return this._req;
+  }
+
   /**
    * Returns DocSession for the given docFD, or throws an exception if this doc is not open.
    */

app/server/lib/Comm.ts

@@ -232,7 +232,7 @@ export class Comm extends EventEmitter {
     client.setSession(scopedSession);                 // Add a Session object to the client.
     client.setOrg((req as RequestWithOrg).org || "");
     client.setProfile(profile);
-    client.setConnection(websocket, counter, browserSettings);
+    client.setConnection({websocket, req, counter, browserSettings});
 
     await client.sendConnectMessage(newClient, reuseClient, lastSeqId, {
       serverVersion: this._serverVersion || version.gitcommit,

app/server/lib/DocApi.ts

@@ -26,11 +26,11 @@ import {SchemaTypes} from "app/common/schema";
 import {SortFunc} from 'app/common/SortFunc';
 import {Sort} from 'app/common/SortSpec';
 import {MetaRowRecord} from 'app/common/TableData';
-import {TelemetryMetadataByLevel} from "app/common/Telemetry";
 import {WebhookFields} from "app/common/Triggers";
 import TriggersTI from 'app/common/Triggers-ti';
 import {DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
 import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {QueryResult} from 'app/gen-server/lib/homedb/Interfaces';
 import * as Types from "app/plugin/DocApiTypes";
 import DocApiTypesTI from "app/plugin/DocApiTypes-ti";
 import {GristObjCode} from "app/plugin/GristData";
@@ -54,8 +54,11 @@ import {
   RequestWithLogin
 } from 'app/server/lib/Authorizer';
 import {DocManager} from "app/server/lib/DocManager";
-import {docSessionFromRequest, getDocSessionShare, makeExceptionalDocSession,
+import {
-        OptDocSession} from "app/server/lib/DocSession";
+  docSessionFromRequest,
+  makeExceptionalDocSession,
+  OptDocSession,
+} from "app/server/lib/DocSession";
 import {DocWorker} from "app/server/lib/DocWorker";
 import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
 import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
@@ -85,6 +88,7 @@ import {
 } from 'app/server/lib/requestUtils';
 import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
 import {localeFromRequest} from "app/server/lib/ServerLocale";
+import {getDocSessionShare} from "app/server/lib/sessionUtils";
 import {isUrlAllowed, WebhookAction, WebHookSecret} from "app/server/lib/Triggers";
 import {fetchDoc, globalUploadSet, handleOptionalUpload, handleUpload,
         makeAccessId} from "app/server/lib/uploads";
@@ -993,14 +997,16 @@ export class DocWorkerApi {
     // DELETE /api/docs/:docId
     // Delete the specified doc.
     this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => {
-      await this._removeDoc(req, res, true);
+      const {status, data} = await this._removeDoc(req, res, true);
+      if (status === 200) { this._logDeleteDocumentEvents(req, data!); }
     }));
 
     // POST /api/docs/:docId/remove
     // Soft-delete the specified doc.  If query parameter "permanent" is set,
     // delete permanently.
     this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => {
-      await this._removeDoc(req, res, isParameterOn(req.query.permanent));
+      const {status, data} = await this._removeDoc(req, res, isParameterOn(req.query.permanent));
+      if (status === 200) { this._logRemoveDocumentEvents(req, data!); }
     }));
 
     this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => {
@@ -1253,11 +1259,7 @@ export class DocWorkerApi {
           },
         },
       });
-      this._logCreatedFileImportDocTelemetryEvent(req, {
+      this._logImportDocumentEvents(mreq, result);
-        full: {
-          docIdDigest: result.id,
-        },
-      });
       res.json(result);
     }));
 
@@ -1402,11 +1404,7 @@ export class DocWorkerApi {
           },
         });
         docId = result.id;
-        this._logCreatedFileImportDocTelemetryEvent(req, {
+        this._logImportDocumentEvents(mreq, result);
-          full: {
-            docIdDigest: docId,
-          },
-        });
       } else if (workspaceId !== undefined) {
         docId = await this._createNewSavedDoc(req, {
           workspaceId: workspaceId,
@@ -1661,7 +1659,7 @@ export class DocWorkerApi {
     }
 
     // Then, import the copy to the workspace.
-    const result = await this._docManager.importDocToWorkspace(mreq, {
+    const {id, title: name} = await this._docManager.importDocToWorkspace(mreq, {
       userId,
       uploadId: uploadResult.uploadId,
       documentName,
@@ -1677,31 +1675,9 @@ export class DocWorkerApi {
         },
       },
     });
-
+    this._logDuplicateDocumentEvents(mreq, {id: sourceDocumentId}, {id, name})
-    const sourceDocument = await this._dbManager.getRawDocById(sourceDocumentId);
+      .catch(e => log.error('DocApi failed to log duplicate document events', e));
-    const isTemplateCopy = sourceDocument.type === 'template';
+    return id;
-    if (isTemplateCopy) {
-      this._grist.getTelemetry().logEvent(mreq, 'copiedTemplate', {
-        full: {
-          templateId: parseUrlId(sourceDocument.urlId || sourceDocument.id).trunkId,
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      });
-    }
-    this._grist.getTelemetry().logEvent(
-      mreq,
-      `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`,
-      {
-        full: {
-          docIdDigest: result.id,
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      }
-    );
-
-    return result.id;
   }
 
   private async _createNewSavedDoc(req: Request, options: {
@@ -1712,31 +1688,13 @@ export class DocWorkerApi {
     const {status, data, errMessage} = await this._dbManager.addDocument(getScope(req), workspaceId, {
       name: documentName ?? 'Untitled document',
     });
-    const docId = data!;
     if (status !== 200) {
       throw new ApiError(errMessage || 'unable to create document', status);
     }
-    this._logDocumentCreatedTelemetryEvent(req, {
+
-      limited: {
+    const {id, name, workspace} = data!;
-        docIdDigest: docId,
+    this._logCreateDocumentEvents(req, {id, name, workspaceId: workspace.id});
-        sourceDocIdDigest: undefined,
+    return id;
-        isImport: false,
-        fileType: undefined,
-        isSaved: true,
-      },
-    });
-    this._logCreatedEmptyDocTelemetryEvent(req, {
-      full: {
-        docIdDigest: docId,
-      },
-    });
-    this._grist.getAuditLogger().logEvent(req as RequestWithLogin, {
-      event: {
-        name: 'createDocument',
-        details: {id: docId},
-      },
-    });
-    return docId;
   }
 
   private async _createNewUnsavedDoc(req: Request, options: {
@@ -1752,64 +1710,13 @@ export class DocWorkerApi {
       trunkDocId: NEW_DOCUMENT_CODE,
       trunkUrlId: NEW_DOCUMENT_CODE,
     });
-    const docId = result.docId;
+    const id = result.docId;
-    await this._docManager.createNamedDoc(
+    const name = await this._docManager.createNamedDoc(
       makeExceptionalDocSession('nascent', {req: mreq, browserSettings}),
-      docId
+      id
     );
-    this._logDocumentCreatedTelemetryEvent(req, {
+    this._logCreateDocumentEvents(req as RequestWithLogin, {id, name});
-      limited: {
+    return id;
-        docIdDigest: docId,
-        sourceDocIdDigest: undefined,
-        isImport: false,
-        fileType: undefined,
-        isSaved: false,
-      },
-    });
-    this._logCreatedEmptyDocTelemetryEvent(req, {
-      full: {
-        docIdDigest: docId,
-      },
-    });
-    this._grist.getAuditLogger().logEvent(mreq, {
-      event: {
-        name: 'createDocument',
-        details: {id: docId},
-      },
-    });
-    return docId;
-  }
-
-  private _logDocumentCreatedTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
-    const mreq = req as RequestWithLogin;
-    this._grist.getTelemetry().logEvent(mreq, 'documentCreated', _.merge({
-      full: {
-        userId: mreq.userId,
-        altSessionId: mreq.altSessionId,
-      },
-    }, metadata));
-  }
-
-  private _logCreatedEmptyDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
-    this._logCreatedDocTelemetryEvent(req, 'createdDoc-Empty', metadata);
-  }
-
-  private _logCreatedFileImportDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
-    this._logCreatedDocTelemetryEvent(req, 'createdDoc-FileImport', metadata);
-  }
-
-  private _logCreatedDocTelemetryEvent(
-    req: Request,
-    event: 'createdDoc-Empty' | 'createdDoc-FileImport',
-    metadata: TelemetryMetadataByLevel,
-  ) {
-    const mreq = req as RequestWithLogin;
-    this._grist.getTelemetry().logEvent(mreq, event, _.merge({
-      full: {
-        userId: mreq.userId,
-        altSessionId: mreq.altSessionId,
-      },
-    }, metadata));
   }
 
   /**
@@ -2091,10 +1998,10 @@ export class DocWorkerApi {
     return result;
   }
 
-  private async _removeDoc(req: Request, res: Response, permanent: boolean) {
+  private async _removeDoc(req: Request, res: Response, permanent: boolean): Promise<QueryResult<Document>> {
-    const mreq = req as RequestWithLogin;
     const scope = getDocScope(req);
     const docId = getDocId(req);
+    let result: QueryResult<Document>;
     if (permanent) {
       const {forkId} = parseUrlId(docId);
       if (!forkId) {
@@ -2110,22 +2017,16 @@ export class DocWorkerApi {
       ];
       await Promise.all(docsToDelete.map(docName => this._docManager.deleteDoc(null, docName, true)));
       // Permanently delete from database.
-      const query = await this._dbManager.deleteDocument(scope);
+      result = await this._dbManager.deleteDocument(scope);
-      this._dbManager.checkQueryResult(query);
+      this._dbManager.checkQueryResult(result);
-      this._grist.getTelemetry().logEvent(mreq, 'deletedDoc', {
+      await sendReply(req, res, {...result, data: result.data!.id});
-        full: {
-          docIdDigest: docId,
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      });
-      await sendReply(req, res, query);
     } else {
-      await this._dbManager.softDeleteDocument(scope);
+      result = await this._dbManager.softDeleteDocument(scope);
       await sendOkReply(req, res);
     }
     await this._dbManager.flushSingleDocAuthCache(scope, docId);
     await this._docManager.interruptDocClients(docId);
+    return result;
   }
 
   private async _runSql(activeDoc: ActiveDoc, req: RequestWithLogin, res: Response,
@@ -2170,6 +2071,7 @@ export class DocWorkerApi {
     try {
       const records = await activeDoc.docStorage.all(wrappedStatement,
                                                      ...(options.args || []));
+      this._logRunSQLQueryEvents(req, options);
       res.status(200).json({
         statement,
         records: records.map(
@@ -2194,6 +2096,130 @@ export class DocWorkerApi {
       clearTimeout(interrupt);
     }
   }
+
+  private _logCreateDocumentEvents(
+    req: Request,
+    document: {id: string; name?: string; workspaceId?: number}
+  ) {
+    const mreq = req as RequestWithLogin;
+    const {id, name, workspaceId} = document;
+    this._grist.getAuditLogger().logEvent(mreq, {
+      event: {
+        name: 'createDocument',
+        details: {id, name},
+        context: {workspaceId},
+      },
+    });
+    this._grist.getTelemetry().logEvent(mreq, 'documentCreated', {
+      limited: {
+        docIdDigest: id,
+        sourceDocIdDigest: undefined,
+        isImport: false,
+        fileType: undefined,
+        isSaved: workspaceId !== undefined,
+      },
+      full: {
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+      },
+    });
+    this._grist.getTelemetry().logEvent(mreq, 'createdDoc-Empty', {
+      limited: {
+        docIdDigest: id,
+        sourceDocIdDigest: undefined,
+        isImport: false,
+        fileType: undefined,
+        isSaved: workspaceId !== undefined,
+      },
+      full: {
+        docIdDigest: id,
+        userId: mreq.userId,
+        altSessionId: mreq.altSessionId,
+      },
+    });
+  }
+
+  private _logRemoveDocumentEvents(req: RequestWithLogin, document: Document) {
+    const {id, name, workspace: {id: workspaceId}} = document;
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'removeDocument',
+        details: {id, name},
+        context: {workspaceId},
+      },
+    });
+  }
+
+  private _logDeleteDocumentEvents(req: RequestWithLogin, {id, name}: Document) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'deleteDocument',
+        details: {id, name},
+      },
+    });
+    this._grist.getTelemetry().logEvent(req, 'deletedDoc', {
+      full: {
+        docIdDigest: id,
+        userId: req.userId,
+        altSessionId: req.altSessionId,
+      },
+    });
+  }
+
+  private _logImportDocumentEvents(
+    req: RequestWithLogin,
+    {id}: {id: string}
+  ) {
+    this._grist.getTelemetry().logEvent(req, 'createdDoc-FileImport', {
+      full: {
+        docIdDigest: id,
+        userId: req.userId,
+        altSessionId: req.altSessionId,
+      },
+    });
+  }
+
+  private async _logDuplicateDocumentEvents(
+    req: RequestWithLogin,
+    originalDocument: {id: string},
+    newDocument: {id: string; name: string}
+  ) {
+    const document = await this._dbManager.getRawDocById(originalDocument.id);
+    const isTemplateCopy = document.type === 'template';
+    if (isTemplateCopy) {
+      this._grist.getTelemetry().logEvent(req, 'copiedTemplate', {
+        full: {
+          templateId: parseUrlId(document.urlId || document.id).trunkId,
+          userId: req.userId,
+          altSessionId: req.altSessionId,
+        },
+      });
+    }
+    this._grist.getTelemetry().logEvent(
+      req,
+      `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`,
+      {
+        full: {
+          docIdDigest: newDocument.id,
+          userId: req.userId,
+          altSessionId: req.altSessionId,
+        },
+      }
+    );
+  }
+
+  private _logRunSQLQueryEvents(
+    req: RequestWithLogin,
+    {sql: query, args, timeout}: Types.SqlPost
+  ) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'runSQLQuery',
+        details: {query, arguments: args, timeout},
+        context: {documentId: getDocId(req)},
+      },
+    });
+  }
 }
 
 export function addDocApiRoutes(

app/server/lib/DocManager.ts

@@ -19,17 +19,13 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
-import {
+import {makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession';
-  getDocSessionCachedDoc,
-  makeExceptionalDocSession,
-  makeOptDocSession,
-  OptDocSession
-} from 'app/server/lib/DocSession';
 import * as docUtils from 'app/server/lib/docUtils';
 import {GristServer} from 'app/server/lib/GristServer';
 import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
 import {makeForkIds, makeId} from 'app/server/lib/idUtils';
 import {checkAllegedGristDoc} from 'app/server/lib/serverUtils';
+import {getDocSessionCachedDoc} from 'app/server/lib/sessionUtils';
 import log from 'app/server/lib/log';
 import {ActiveDoc} from './ActiveDoc';
 import {PluginManager} from './PluginManager';
@@ -246,7 +242,6 @@ export class DocManager extends EventEmitter {
       register,
       userId,
     });
-
     this.gristServer.getTelemetry().logEvent(mreq, 'documentCreated', merge({
       limited: {
         docIdDigest: docCreationInfo.id,
@@ -254,13 +249,6 @@ export class DocManager extends EventEmitter {
         isSaved: workspaceId !== undefined,
       },
     }, telemetryMetadata));
-    this.gristServer.getAuditLogger().logEvent(mreq, {
-      event: {
-        name: 'createDocument',
-        details: {id: docCreationInfo.id},
-      },
-    });
-
     return docCreationInfo;
     // The imported document is associated with the worker that did the import.
     // We could break that association (see /api/docs/:docId/assign for how) if

app/server/lib/DocSession.ts

@@ -1,10 +1,6 @@
 import {BrowserSettings} from 'app/common/BrowserSettings';
-import {DocumentUsage} from 'app/common/DocUsage';
-import {Role} from 'app/common/roles';
-import {FullUser} from 'app/common/UserAPI';
-import {Document} from 'app/gen-server/entity/Document';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {Authorizer, getUser, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
 
 /**
@@ -82,125 +78,3 @@ export class DocSession implements OptDocSession {
   // Browser settings (like timezone) obtained from the Client.
   public get browserSettings(): BrowserSettings { return this.client.browserSettings; }
 }
-
-/**
- * Extract userId from OptDocSession.  Use Authorizer if available (for web socket
- * sessions), or get it from the Request if that is available (for rest api calls),
- * or from the Client if that is available.  Returns null if userId information is
- * not available or not cached.
- */
-export function getDocSessionUserId(docSession: OptDocSession): number|null {
-  if (docSession.authorizer) {
-    return docSession.authorizer.getUserId();
-  }
-  if (docSession.req) {
-    return getUserId(docSession.req);
-  }
-  if (docSession.client) {
-    return docSession.client.getCachedUserId();
-  }
-  return null;
-}
-
-export function getDocSessionAltSessionId(docSession: OptDocSession): string|null {
-  if (docSession.req) {
-    return docSession.req.altSessionId || null;
-  }
-  if (docSession.client) {
-    return docSession.client.getAltSessionId() || null;
-  }
-  return null;
-}
-
-/**
- * Get as much of user profile as we can (id, name, email).
- */
-export function getDocSessionUser(docSession: OptDocSession): FullUser|null {
-  if (docSession.authorizer) {
-    return docSession.authorizer.getUser();
-  }
-  if (docSession.req) {
-    const user = getUser(docSession.req);
-    const email = user.loginEmail;
-    if (email) {
-      return {id: user.id, name: user.name, email, ref: user.ref, locale: user.options?.locale};
-    }
-  }
-  if (docSession.client) {
-    const id = docSession.client.getCachedUserId();
-    const ref = docSession.client.getCachedUserRef();
-    const profile = docSession.client.getProfile();
-    if (id && profile) {
-      return {
-        id,
-        ref,
-        ...profile
-      };
-    }
-  }
-  return null;
-}
-
-/**
- * Extract user's role from OptDocSession.  Method depends on whether using web
- * sockets or rest api.  Assumes that access has already been checked by wrappers
- * for api methods and that cached access information is therefore available.
- */
-export function getDocSessionAccess(docSession: OptDocSession): Role {
-  // "nascent" DocSessions are for when a document is being created, and user is
-  // its only owner as yet.
-  // "system" DocSessions are for access without access control.
-  if (docSession.mode === 'nascent' || docSession.mode === 'system') { return 'owners'; }
-  // "plugin" DocSessions are for access from plugins, which is currently quite crude,
-  // and granted only to editors.
-  if (docSession.mode === 'plugin') { return 'editors'; }
-  if (docSession.authorizer) {
-    const access = docSession.authorizer.getCachedAuth().access;
-    if (!access) { throw new Error('getDocSessionAccess expected authorizer.getCachedAuth'); }
-    return access;
-  }
-  if (docSession.req) {
-    const access =  docSession.req.docAuth?.access;
-    if (!access) { throw new Error('getDocSessionAccess expected req.docAuth.access'); }
-    return access;
-  }
-  throw new Error('getDocSessionAccess could not find access information in DocSession');
-}
-
-export function getDocSessionShare(docSession: OptDocSession): string|null {
-  return _getCachedDoc(docSession)?.linkId || null;
-}
-
-/**
- * Get document usage seen in db when we were last checking document
- * access. Not necessarily a live value when using a websocket
- * (although we do recheck access periodically).
- */
-export function getDocSessionUsage(docSession: OptDocSession): DocumentUsage|null {
-  return _getCachedDoc(docSession)?.usage || null;
-}
-
-export function _getCachedDoc(docSession: OptDocSession): Document|null {
-  if (docSession.authorizer) {
-    return docSession.authorizer.getCachedAuth().cachedDoc || null;
-  }
-  if (docSession.req) {
-    return docSession.req.docAuth?.cachedDoc || null;
-  }
-  return null;
-}
-
-export function getDocSessionAccessOrNull(docSession: OptDocSession): Role|null {
-  try {
-    return getDocSessionAccess(docSession);
-  } catch (err) {
-    return null;
-  }
-}
-
-/**
- * Get cached information about the document, if available.  May be stale.
- */
-export function getDocSessionCachedDoc(docSession: OptDocSession): Document|undefined {
-  return (docSession.req as RequestWithLogin)?.docAuth?.cachedDoc;
-}

app/server/lib/FlexServer.ts

@@ -915,9 +915,9 @@ export class FlexServer implements GristServer {
   }
 
   public addAuditLogger() {
-    if (this._check('audit-logger')) { return; }
+    if (this._check('audit-logger', 'homedb')) { return; }
 
-    this._auditLogger = this.create.AuditLogger();
+    this._auditLogger = this.create.AuditLogger(this._dbManager);
   }
 
   public async addTelemetry() {

app/server/lib/GranularAccess.ts

@@ -38,8 +38,7 @@ import { FullUser, UserAccessData } from 'app/common/UserAPI';
 import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { GristObjCode } from 'app/plugin/GristData';
 import { DocClients } from 'app/server/lib/DocClients';
-import { getDocSessionAccess, getDocSessionAltSessionId, getDocSessionShare,
+import { OptDocSession } from 'app/server/lib/DocSession';
-         getDocSessionUser, OptDocSession } from 'app/server/lib/DocSession';
 import { DocStorage, REMOVE_UNUSED_ATTACHMENTS_DELAY } from 'app/server/lib/DocStorage';
 import log from 'app/server/lib/log';
 import { IPermissionInfo, MixedPermissionSetWithContext,
@@ -47,6 +46,12 @@ import { IPermissionInfo, MixedPermissionSetWithContext,
 import { TablePermissionSetWithContext } from 'app/server/lib/PermissionInfo';
 import { integerParam } from 'app/server/lib/requestUtils';
 import { getRelatedRows, getRowIdsFromDocAction } from 'app/server/lib/RowAccess';
+import {
+  getAltSessionId,
+  getDocSessionAccess,
+  getDocSessionShare,
+  getFullUser,
+} from 'app/server/lib/sessionUtils';
 import cloneDeep = require('lodash/cloneDeep');
 import fromPairs = require('lodash/fromPairs');
 import get = require('lodash/get');
@@ -379,7 +384,7 @@ export class GranularAccess implements GranularAccessForBundle {
         fullUser = attrs.override.user;
       }
     } else {
-      fullUser = getDocSessionUser(docSession);
+      fullUser = getFullUser(docSession);
     }
     const user = new User();
     user.Access = access;
@@ -395,7 +400,7 @@ export class GranularAccess implements GranularAccessForBundle {
     // Include origin info if accessed via the rest api.
     // TODO: could also get this for websocket access, just via a different route.
     user.Origin = docSession.req?.get('origin') || null;
-    user.SessionID = isAnonymous ? `a${getDocSessionAltSessionId(docSession)}` : `u${user.UserID}`;
+    user.SessionID = isAnonymous ? `a${getAltSessionId(docSession)}` : `u${user.UserID}`;
     user.IsLoggedIn = !isAnonymous;
     user.UserRef = fullUser?.ref || null; // Empty string should be treated as null.
 

app/server/lib/HTTPAuditLogger.ts

@@ -1,11 +1,19 @@
-import {AuditEvent, AuditEventName, AuditEventUser} from 'app/common/AuditEvent';
+import {AuditEvent, AuditEventName, AuditEventSource, AuditEventUser} from 'app/common/AuditEvent';
-import {AuditEventProperties, IAuditLogger} from 'app/server/lib/AuditLogger';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
-import {getDocSessionUser} from 'app/server/lib/DocSession';
+import {AuditEventProperties, IAuditLogger, LogAuditEventError} from 'app/server/lib/AuditLogger';
-import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods';
+import {LogMethods} from 'app/server/lib/LogMethods';
-import {RequestOrSession} from 'app/server/lib/requestUtils';
+import {getOriginIpAddress} from 'app/server/lib/requestUtils';
-import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
+import {
+  getAltSessionId,
+  getFullUser,
+  getLogMeta,
+  getOrg,
+  getRequest,
+  RequestOrSession,
+} from 'app/server/lib/sessionUtils';
 import moment from 'moment-timezone';
 import fetch from 'node-fetch';
+import {inspect} from 'util';
 
 interface HTTPAuditLoggerOptions {
   /**
@@ -31,35 +39,40 @@ const MAX_PENDING_REQUESTS = 25;
  * See `GristAuditLogger` for an example.
  */
 export abstract class HTTPAuditLogger implements IAuditLogger {
-  private _endpoint = this._options.endpoint;
-  private _authorizationHeader = this._options.authorizationHeader;
   private _numPendingRequests = 0;
-  private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession: RequestOrSession | undefined) =>
+  private readonly _endpoint = this._options.endpoint;
+  private readonly _authorizationHeader = this._options.authorizationHeader;
+  private readonly _logger = new LogMethods<RequestOrSession | undefined>('AuditLogger ', (requestOrSession) =>
     getLogMeta(requestOrSession));
 
-  constructor(private _options: HTTPAuditLoggerOptions) {}
+  constructor(private _db: HomeDBManager, private _options: HTTPAuditLoggerOptions) {}
 
   /**
    * Logs an audit event.
    */
   public logEvent<Name extends AuditEventName>(
     requestOrSession: RequestOrSession,
-    event: AuditEventProperties<Name>
+    properties: AuditEventProperties<Name>
   ): void {
-    this._logEventOrThrow(requestOrSession, event)
+    this._logEventOrThrow(requestOrSession, properties)
-      .catch((e) => this._logger.error(requestOrSession, `failed to log audit event`, event, e));
+      .catch((e) => {
+        this._logger.error(requestOrSession, `failed to log audit event`, e);
+        this._logger.warn(requestOrSession, 'skipping audit event ', inspect(e.auditEvent, {
+          depth: Infinity,
+        }));
+      });
   }
 
   /**
-   * Asynchronous variant of `logEvent`.
+   * Logs an audit event.
    *
-   * Throws on failure to log an event.
+   * Throws a LogAuditEventError on failure.
    */
   public async logEventAsync<Name extends AuditEventName>(
     requestOrSession: RequestOrSession,
-    event: AuditEventProperties<Name>
+    properties: AuditEventProperties<Name>
   ): Promise<void> {
-    await this._logEventOrThrow(requestOrSession, event);
+    await this._logEventOrThrow(requestOrSession, properties);
   }
 
   /**
@@ -69,10 +82,14 @@ export abstract class HTTPAuditLogger implements IAuditLogger {
 
   private async _logEventOrThrow<Name extends AuditEventName>(
     requestOrSession: RequestOrSession,
-    {event: {name, details}, timestamp}: AuditEventProperties<Name>
+    properties: AuditEventProperties<Name>
   ) {
+    const event: AuditEvent<Name> = this._buildAuditEvent(requestOrSession, properties);
     if (this._numPendingRequests === MAX_PENDING_REQUESTS) {
-      throw new Error(`exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`);
+      throw new LogAuditEventError(
+        event,
+        `exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`
+      );
     }
 
     try {
@@ -83,53 +100,58 @@ export abstract class HTTPAuditLogger implements IAuditLogger {
           ...(this._authorizationHeader ? {'Authorization': this._authorizationHeader} : undefined),
           'Content-Type': 'application/json',
         },
-        body: this.toJSON({
+        body: this.toJSON(event),
-          event: {
-            name,
-            user: getAuditEventUser(requestOrSession),
-            details: details ?? null,
-          },
-          timestamp: timestamp ?? moment().toISOString(),
-        }),
       });
       if (!resp.ok) {
         throw new Error(`received a non-200 response from ${resp.url}: ${resp.status} ${await resp.text()}`);
       }
+    } catch (e) {
+      throw new LogAuditEventError(
+        event,
+        e?.message ?? `failed to POST audit event to ${this._endpoint}`,
+        {cause: e}
+      );
     } finally {
       this._numPendingRequests -= 1;
     }
   }
-}
 
-function getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser | null {
+  private _buildAuditEvent<Name extends AuditEventName>(
-  if (!requestOrSession) { return null; }
+    requestOrSession: RequestOrSession,
-
+    properties: AuditEventProperties<Name>
-  if ('get' in requestOrSession) {
+  ): AuditEvent<Name> {
+    const {event: {name, details = {}, context = {}}, timestamp = moment().toISOString()} = properties;
     return {
-      id: requestOrSession.userId ?? null,
+      event: {
-      email: requestOrSession.user?.loginEmail ?? null,
+        name,
-      name: requestOrSession.user?.name ?? null,
+        user: this._getAuditEventUser(requestOrSession),
+        details,
+        context,
+        source: getAuditEventSource(requestOrSession),
+      },
+      timestamp,
     };
-  } else {
+  }
-    const user = getDocSessionUser(requestOrSession);
-    if (!user) { return null; }
 
+  private _getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser {
+    const user = getFullUser(requestOrSession);
+    if (!user) {
+      return {type: 'unknown'};
+    } else if (user.id === this._db.getAnonymousUserId()) {
+      return {type: 'anonymous'};
+    } else {
       const {id, email, name} = user;
-    return {id, email, name};
+      return {type: 'user', id, email, name};
+    }
   }
 }
 
-function getLogMeta(requestOrSession?: RequestOrSession): ILogMeta {
+function getAuditEventSource(requestOrSession: RequestOrSession): AuditEventSource {
-  if (!requestOrSession) { return {}; }
+  const request = getRequest(requestOrSession);
-
-  if ('get' in requestOrSession) {
   return {
-      org: requestOrSession.org,
+    org: getOrg(requestOrSession) || undefined,
-      email: requestOrSession.user?.loginEmail,
+    ipAddress: request ? getOriginIpAddress(request) : undefined,
-      userId: requestOrSession.userId,
+    userAgent: request?.headers['user-agent'] || undefined,
-      altSessionId: requestOrSession.altSessionId,
+    sessionId: getAltSessionId(requestOrSession) || undefined,
   };
-  } else {
-    return getLogMetaFromDocSession(requestOrSession);
-  }
 }

app/server/lib/ICreate.ts

@@ -70,7 +70,7 @@ export interface ICreate {
 
   Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling;
   Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier;
-  AuditLogger(): IAuditLogger;
+  AuditLogger(dbManager: HomeDBManager): IAuditLogger;
   Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry;
   Shell?(): IShell;  // relevant to electron version of Grist only.
 
@@ -120,7 +120,7 @@ export interface ICreateBillingOptions {
 export interface ICreateAuditLoggerOptions {
   name: 'grist'|'hec';
   check(): boolean;
-  create(): IAuditLogger|undefined;
+  create(dbManager: HomeDBManager): IAuditLogger|undefined;
 }
 
 export interface ICreateTelemetryOptions {
@@ -177,8 +177,8 @@ export function makeSimpleCreator(opts: {
       }
       return undefined;
     },
-    AuditLogger() {
+    AuditLogger(dbManager) {
-      return auditLogger?.find(({check}) => check())?.create() ?? createDummyAuditLogger();
+      return auditLogger?.find(({check}) => check())?.create(dbManager) ?? createDummyAuditLogger();
     },
     Telemetry(dbManager, gristConfig) {
       return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry();

app/server/lib/Telemetry.ts

@@ -19,13 +19,12 @@ import {Activation} from 'app/gen-server/entity/Activation';
 import {Activations} from 'app/gen-server/lib/Activations';
 import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
-import {getDocSessionUser} from 'app/server/lib/DocSession';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {GristServer} from 'app/server/lib/GristServer';
 import {hashId} from 'app/server/lib/hashingUtils';
 import {LogMethods} from 'app/server/lib/LogMethods';
-import {RequestOrSession, stringParam} from 'app/server/lib/requestUtils';
+import {stringParam} from 'app/server/lib/requestUtils';
-import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils';
+import {getFullUser, getLogMeta, isRequest, RequestOrSession} from 'app/server/lib/sessionUtils';
 import * as cookie from 'cookie';
 import * as express from 'express';
 import fetch from 'node-fetch';
@@ -73,8 +72,8 @@ export class Telemetry implements ITelemetry {
   private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL ||
     'https://telemetry.getgrist.com/api/telemetry';
   private _numPendingForwardEventRequests = 0;
-  private readonly _logger = new LogMethods('Telemetry ', (requestOrSession: RequestOrSession | undefined) =>
+  private readonly _logger = new LogMethods<RequestOrSession | undefined>('Telemetry ', (requestOrSession) =>
-    this._getLogMeta(requestOrSession));
+    getLogMeta(requestOrSession));
   private readonly _telemetryLogger = new LogMethods<string>('Telemetry ', (eventType) => ({
     eventType,
   }));
@@ -273,14 +272,14 @@ export class Telemetry implements ITelemetry {
     if (requestOrSession) {
       let email: string | undefined;
       let org: string | undefined;
-      if ('get' in requestOrSession) {
+      if (isRequest(requestOrSession)) {
         email = requestOrSession.user?.loginEmail;
         org = requestOrSession.org;
         if (isAnonymousUser) {
           visitorId = this._getAndSetMatomoVisitorId(requestOrSession);
         }
       } else {
-        email = getDocSessionUser(requestOrSession)?.email;
+        email = getFullUser(requestOrSession)?.email;
         org = requestOrSession.client?.getOrg() ?? requestOrSession.req?.org;
       }
       if (email) {
@@ -378,21 +377,6 @@ export class Telemetry implements ITelemetry {
       throw new ApiError('Telemetry is not ready', 500);
     }
   }
-
-  private _getLogMeta(requestOrSession?: RequestOrSession) {
-    if (!requestOrSession) { return {}; }
-
-    if ('get' in requestOrSession) {
-      return {
-        org: requestOrSession.org,
-        email: requestOrSession.user?.loginEmail,
-        userId: requestOrSession.userId,
-        altSessionId: requestOrSession.altSessionId,
-      };
-    } else {
-      return getLogMetaFromDocSession(requestOrSession);
-    }
-  }
 }
 
 export async function getTelemetryPrefs(

app/server/lib/configureGristAuditLogger.ts

@@ -1,11 +1,12 @@
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {appSettings} from 'app/server/lib/AppSettings';
 import {GristAuditLogger} from 'app/server/lib/GristAuditLogger';
 
-export function configureGristAuditLogger() {
+export function configureGristAuditLogger(db: HomeDBManager) {
   const options = checkGristAuditLogger();
   if (!options) { return undefined; }
 
-  return new GristAuditLogger(options);
+  return new GristAuditLogger(db, options);
 }
 
 export function checkGristAuditLogger() {

app/server/lib/requestUtils.ts

@@ -1,9 +1,9 @@
 import {ApiError} from 'app/common/ApiError';
 import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
-import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {DocScope, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
+import {QueryResult} from 'app/gen-server/lib/homedb/Interfaces';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
-import {OptDocSession} from 'app/server/lib/DocSession';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {RequestWithGrist} from 'app/server/lib/GristServer';
 import log from 'app/server/lib/log';
@@ -13,8 +13,6 @@ import {IncomingMessage} from 'http';
 import {Writable} from 'stream';
 import {TLSSocket} from 'tls';
 
-export type RequestOrSession = RequestWithLogin | OptDocSession | null;
-
 // log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
 const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION);
 
@@ -347,6 +345,38 @@ export function getOriginUrl(req: IncomingMessage) {
   return `${protocol}://${host}`;
 }
 
+/**
+ * Returns the original request IP address.
+ *
+ * If the request was made through a proxy or load balancer, the IP address
+ * is read from forwarded headers. See:
+ *
+ *  - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
+ *  - https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html
+ */
+export function getOriginIpAddress(req: IncomingMessage) {
+  return (
+    // May contain multiple comma-separated values; the first one is the original.
+    (req.headers['x-forwarded-for'] as string | undefined)
+      ?.split(',')
+      .map(value => value.trim())[0] ||
+    req.socket?.remoteAddress ||
+    undefined
+  );
+}
+
+/**
+ * Returns the request's "X-Forwarded-For" header, with the request's IP address
+ * appended to its value.
+ *
+ * If the header is absent from the request, a new header will be returned.
+ */
+export function buildXForwardedForHeader(req: Request): {'X-Forwarded-For': string}|undefined {
+  const values = req.get('X-Forwarded-For')?.split(',').map(value => value.trim()) ?? [];
+  if (req.socket.remoteAddress) { values.push(req.socket.remoteAddress); }
+  return values.length > 0 ? { 'X-Forwarded-For': values.join(', ') } : undefined;
+}
+
 /**
  * Get the protocol to use in Grist URLs that are intended to be reachable
  * from a user's browser. Use the protocol in APP_HOME_URL if available,

app/server/lib/serverUtils.ts

@@ -1,16 +1,15 @@
-import bluebird from 'bluebird';
-import { ChildProcess } from 'child_process';
-import * as net from 'net';
-import * as path from 'path';
-import { ConnectionOptions } from 'typeorm';
-import uuidv4 from 'uuid/v4';
-import {AbortSignal} from 'node-abort-controller';
-
 import {EngineCode} from 'app/common/DocumentSettings';
-import {TelemetryMetadataByLevel} from 'app/common/Telemetry';
+import {OptDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
+import {getLogMeta} from 'app/server/lib/sessionUtils';
 import {OpenMode, SQLiteDB} from 'app/server/lib/SQLiteDB';
-import {getDocSessionAccessOrNull, getDocSessionUser, OptDocSession} from './DocSession';
+import bluebird from 'bluebird';
+import {ChildProcess} from 'child_process';
+import * as net from 'net';
+import {AbortSignal} from 'node-abort-controller';
+import * as path from 'path';
+import {ConnectionOptions} from 'typeorm';
+import uuidv4 from 'uuid/v4';
 
 // This method previously lived in this file. Re-export to avoid changing imports all over.
 export {timeoutReached} from 'app/common/gutil';
@@ -130,8 +129,11 @@ export async function checkAllegedGristDoc(docSession: OptDocSession, fname: str
     const integrityCheckResults = await db.all('PRAGMA integrity_check');
     if (integrityCheckResults.length !== 1 || integrityCheckResults[0].integrity_check !== 'ok') {
       const uuid = uuidv4();
-      log.info('Integrity check failure on import', {uuid, integrityCheckResults,
+      log.info('Integrity check failure on import', {
-                                                     ...getLogMetaFromDocSession(docSession)});
+        uuid,
+        integrityCheckResults,
+        ...getLogMeta(docSession),
+      });
       throw new Error(`Document failed integrity checks - is it corrupted? Event ID: ${uuid}`);
     }
   } finally {
@@ -139,39 +141,6 @@ export async function checkAllegedGristDoc(docSession: OptDocSession, fname: str
   }
 }
 
-/**
- * Extract access, userId, email, and client (if applicable) from session, for logging purposes.
- */
-export function getLogMetaFromDocSession(docSession: OptDocSession) {
-  const client = docSession.client;
-  const access = getDocSessionAccessOrNull(docSession);
-  const user = getDocSessionUser(docSession);
-  const email = user?.loginEmail || user?.email;
-  return {
-    access,
-    ...(user ? {userId: user.id, email} : {}),
-    ...(client ? client.getLogMeta() : {}),   // Client if present will repeat and add to user info.
-  };
-}
-
-/**
- * Extract telemetry metadata from session.
- */
-export function getTelemetryMetaFromDocSession(docSession: OptDocSession): TelemetryMetadataByLevel {
-  const client = docSession.client;
-  const access = getDocSessionAccessOrNull(docSession);
-  const user = getDocSessionUser(docSession);
-  return {
-    limited: {
-      access,
-    },
-    full: {
-      ...(user ? {userId: user.id} : {}),
-      ...(client ? client.getFullTelemetryMeta() : {}),   // Client if present will repeat and add to user info.
-    },
-  };
-}
-
 /**
  * Only offer choices of engine on experimental deployments (staging/dev).
  */

app/server/lib/sessionUtils.ts

@@ -0,0 +1,255 @@
+import {DocumentUsage} from 'app/common/DocUsage';
+import {FullUser} from 'app/common/LoginSessionAPI';
+import {Role} from 'app/common/roles';
+import {Document} from 'app/gen-server/entity/Document';
+import {getUserId as getRequestUserId, getUser, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {OptDocSession} from 'app/server/lib/DocSession';
+import {ILogMeta} from 'app/server/lib/log';
+import {IncomingMessage} from 'http';
+
+export type RequestOrSession = RequestWithLogin | OptDocSession | null;
+
+export function isRequest(
+  requestOrSession: RequestOrSession
+): requestOrSession is RequestWithLogin {
+  return Boolean(requestOrSession && 'get' in requestOrSession);
+}
+
+/**
+ * Extract the raw `IncomingMessage` from `requestOrSession`, if available.
+ */
+export function getRequest(requestOrSession: RequestOrSession): IncomingMessage | null {
+  if (!requestOrSession) { return null; }
+
+  // The location of the request depends on the context, which include REST
+  // API calls to document endpoints and WebSocket sessions.
+  if (isRequest(requestOrSession)) {
+    return requestOrSession;
+  } else if (requestOrSession.req) {
+    // A REST API call to a document endpoint.
+    return requestOrSession.req;
+  } else if (requestOrSession.client) {
+    // A WebSocket session.
+    return requestOrSession.client.getConnectionRequest();
+  } else {
+    return null;
+  }
+}
+
+export function getAltSessionId(requestOrSession: RequestOrSession): string | null {
+  if (!requestOrSession) { return null; }
+
+  if (isRequest(requestOrSession)) {
+    return requestOrSession.altSessionId || null;
+  } else {
+    return getDocSessionAltSessionId(requestOrSession);
+  }
+}
+
+function getDocSessionAltSessionId(docSession: OptDocSession): string|null {
+  if (docSession.req) {
+    return docSession.req.altSessionId || null;
+  }
+  if (docSession.client) {
+    return docSession.client.getAltSessionId() || null;
+  }
+  return null;
+}
+
+export function getUserId(requestOrSession: RequestOrSession): number|null {
+  if (!requestOrSession) { return null; }
+
+  if (isRequest(requestOrSession)) {
+    return getRequestUserId(requestOrSession);
+  } else {
+    return getDocSessionUserId(requestOrSession);
+  }
+}
+
+/**
+ * Extract userId from OptDocSession.  Use Authorizer if available (for web socket
+ * sessions), or get it from the Request if that is available (for rest api calls),
+ * or from the Client if that is available.  Returns null if userId information is
+ * not available or not cached.
+ */
+function getDocSessionUserId(docSession: OptDocSession): number|null {
+  if (docSession.authorizer) {
+    return docSession.authorizer.getUserId();
+  }
+  if (docSession.req) {
+    return getUserId(docSession.req);
+  }
+  if (docSession.client) {
+    return docSession.client.getCachedUserId();
+  }
+  return null;
+}
+
+/**
+ * Get as much of user profile as we can (id, name, email).
+ */
+export function getFullUser(requestOrSession: RequestOrSession): FullUser | null {
+  if (!requestOrSession) { return null; }
+
+  if (isRequest(requestOrSession)) {
+    return getRequestFullUser(requestOrSession);
+  } else {
+    return getDocSessionFullUser(requestOrSession);
+  }
+}
+
+function getRequestFullUser(request: RequestWithLogin): FullUser|null {
+  const user = getUser(request);
+  if (!user.loginEmail) { return null; }
+
+  const {id, name, loginEmail: email, ref, options} = user;
+  return {
+    id,
+    name,
+    email,
+    ref,
+    locale: options?.locale,
+  };
+}
+
+function getDocSessionFullUser(docSession: OptDocSession): FullUser|null {
+  if (docSession.authorizer) {
+    return docSession.authorizer.getUser();
+  }
+  if (docSession.req) {
+    return getRequestFullUser(docSession.req);
+  }
+  if (docSession.client) {
+    const id = docSession.client.getCachedUserId();
+    const ref = docSession.client.getCachedUserRef();
+    const profile = docSession.client.getProfile();
+    if (id && profile) {
+      return {
+        id,
+        ref,
+        ...profile
+      };
+    }
+  }
+  return null;
+}
+
+export function getOrg(requestOrSession: RequestOrSession): string | null {
+  if (!requestOrSession) { return null; }
+
+  if (isRequest(requestOrSession)) {
+    return requestOrSession.org || null;
+  } else {
+    return getDocSessionOrg(requestOrSession);
+  }
+}
+
+function getDocSessionOrg(docSession: OptDocSession): string | null {
+  if (docSession.req) {
+    return docSession.req.org || null;
+  }
+  if (docSession.client) {
+    return docSession.client.getOrg() || null;
+  }
+  return null;
+}
+
+/**
+ * Extract access, userId, email, and client (if applicable) from
+ * `requestOrSession`, for logging purposes.
+ */
+export function getLogMeta(requestOrSession: RequestOrSession | undefined): ILogMeta {
+  if (!requestOrSession) { return {}; }
+
+  if (isRequest(requestOrSession)) {
+    return getRequestLogMeta(requestOrSession);
+  } else {
+    return getDocSessionLogMeta(requestOrSession);
+  }
+}
+
+function getRequestLogMeta(request: RequestWithLogin): ILogMeta {
+  const {org, user, userId, altSessionId} = request;
+  return {
+    org,
+    email: user?.loginEmail,
+    userId,
+    altSessionId,
+  };
+}
+
+function getDocSessionLogMeta(docSession: OptDocSession): ILogMeta {
+  const client = docSession.client;
+  const access = getDocSessionAccessOrNull(docSession);
+  const user = getDocSessionFullUser(docSession);
+  const email = user?.loginEmail || user?.email;
+  return {
+    access,
+    ...(user ? {userId: user.id, email} : {}),
+    ...(client ? client.getLogMeta() : {}),   // Client if present will repeat and add to user info.
+  };
+}
+
+/**
+ * Extract user's role from OptDocSession.  Method depends on whether using web
+ * sockets or rest api.  Assumes that access has already been checked by wrappers
+ * for api methods and that cached access information is therefore available.
+ */
+export function getDocSessionAccess(docSession: OptDocSession): Role {
+  // "nascent" DocSessions are for when a document is being created, and user is
+  // its only owner as yet.
+  // "system" DocSessions are for access without access control.
+  if (docSession.mode === 'nascent' || docSession.mode === 'system') { return 'owners'; }
+  // "plugin" DocSessions are for access from plugins, which is currently quite crude,
+  // and granted only to editors.
+  if (docSession.mode === 'plugin') { return 'editors'; }
+  if (docSession.authorizer) {
+    const access = docSession.authorizer.getCachedAuth().access;
+    if (!access) { throw new Error('getDocSessionAccess expected authorizer.getCachedAuth'); }
+    return access;
+  }
+  if (docSession.req) {
+    const access =  docSession.req.docAuth?.access;
+    if (!access) { throw new Error('getDocSessionAccess expected req.docAuth.access'); }
+    return access;
+  }
+  throw new Error('getDocSessionAccess could not find access information in DocSession');
+}
+
+export function getDocSessionShare(docSession: OptDocSession): string|null {
+  return _getCachedDoc(docSession)?.linkId || null;
+}
+
+/**
+ * Get document usage seen in db when we were last checking document
+ * access. Not necessarily a live value when using a websocket
+ * (although we do recheck access periodically).
+ */
+export function getDocSessionUsage(docSession: OptDocSession): DocumentUsage|null {
+  return _getCachedDoc(docSession)?.usage || null;
+}
+
+export function _getCachedDoc(docSession: OptDocSession): Document|null {
+  if (docSession.authorizer) {
+    return docSession.authorizer.getCachedAuth().cachedDoc || null;
+  }
+  if (docSession.req) {
+    return docSession.req.docAuth?.cachedDoc || null;
+  }
+  return null;
+}
+
+export function getDocSessionAccessOrNull(docSession: OptDocSession): Role|null {
+  try {
+    return getDocSessionAccess(docSession);
+  } catch (err) {
+    return null;
+  }
+}
+
+/**
+ * Get cached information about the document, if available.  May be stale.
+ */
+export function getDocSessionCachedDoc(docSession: OptDocSession): Document | undefined {
+  return (docSession.req as RequestWithLogin)?.docAuth?.cachedDoc;
+}

test/server/lib/GristAuditLogger.ts

@@ -52,10 +52,13 @@ describe('GristAuditLogger', function() {
         .post('/events', {
           event: {
             name: 'createDocument',
-            user: null,
+            user: {type: 'unknown'},
             details: {
               id: 'docId',
+              name: 'docName',
             },
+            context: {},
+            source: {},
           },
           timestamp,
         })
@@ -64,7 +67,7 @@ describe('GristAuditLogger', function() {
         auditLogger.logEventAsync(null, {
           event: {
             name: 'createDocument',
-            details: {id: 'docId'},
+            details: {id: 'docId', name: 'docName'},
           },
           timestamp,
         })
@@ -80,7 +83,7 @@ describe('GristAuditLogger', function() {
         auditLogger.logEventAsync(null, {
           event: {
             name: 'createDocument',
-            details: {id: 'docId'},
+            details: {id: 'docId', name: 'docName'},
           },
         }),
         'received a non-200 response from https://api.getgrist.com/events: 404 Not found'
@@ -98,7 +101,7 @@ describe('GristAuditLogger', function() {
         void auditLogger.logEvent(null, {
           event: {
             name: 'createDocument',
-            details: {id: 'docId'},
+            details: {id: 'docId', name: 'docName'},
           },
         });
       }
@@ -106,7 +109,7 @@ describe('GristAuditLogger', function() {
         auditLogger.logEventAsync(null, {
           event: {
             name: 'createDocument',
-            details: {id: 'docId'},
+            details: {id: 'docId', name: 'docName'},
           },
         }),
         'exceeded the maximum number of pending audit event calls (25)'