From 8b1d1c5d25f37be4990203f6303fdd7cf085b521 Mon Sep 17 00:00:00 2001 From: George Gevoian Date: Mon, 23 Sep 2024 11:04:22 -0400 Subject: [PATCH] (core) Add more audit logging data/events Summary: Adds a few additional audit events and enhances audit logging to capture more data (request origin, active org, user type). Test Plan: Server and manual tests. Reviewers: jarek Reviewed By: jarek Subscribers: jarek Differential Revision: https://phab.getgrist.com/D4348 --- app/common/AuditEvent.ts | 223 ++++++++++++++- app/gen-server/ApiServer.ts | 124 ++++++--- app/gen-server/lib/homedb/HomeDBManager.ts | 73 ++--- app/gen-server/lib/homedb/Interfaces.ts | 5 + app/server/lib/ActiveDoc.ts | 58 ++-- app/server/lib/Assistance.ts | 12 +- app/server/lib/AuditLogger.ts | 28 +- app/server/lib/Authorizer.ts | 5 +- app/server/lib/Client.ts | 39 ++- app/server/lib/Comm.ts | 2 +- app/server/lib/DocApi.ts | 290 +++++++++++--------- app/server/lib/DocManager.ts | 16 +- app/server/lib/DocSession.ts | 128 +-------- app/server/lib/FlexServer.ts | 4 +- app/server/lib/GranularAccess.ts | 13 +- app/server/lib/HTTPAuditLogger.ts | 128 +++++---- app/server/lib/ICreate.ts | 8 +- app/server/lib/Telemetry.ts | 28 +- app/server/lib/configureGristAuditLogger.ts | 5 +- app/server/lib/requestUtils.ts | 38 ++- app/server/lib/serverUtils.ts | 59 +--- app/server/lib/sessionUtils.ts | 255 +++++++++++++++++ test/server/lib/GristAuditLogger.ts | 13 +- 23 files changed, 1013 insertions(+), 541 deletions(-) create mode 100644 app/server/lib/sessionUtils.ts diff --git a/app/common/AuditEvent.ts b/app/common/AuditEvent.ts index 47a5da3a..e88cbf2f 100644 --- a/app/common/AuditEvent.ts +++ b/app/common/AuditEvent.ts @@ -1,31 +1,224 @@ export interface AuditEvent { + /** + * The event. + */ event: { - /** The event name. */ + /** + * The name of the event. + */ name: Name; - /** The user that triggered the event. */ - user: AuditEventUser | null; - /** Additional event details. */ - details: AuditEventDetails[Name] | null; + /** + * The user that triggered the event. + */ + user: AuditEventUser; + /** + * The event details. + */ + details: AuditEventDetails[Name] | {}; + /** + * The context of the event. + */ + context: AuditEventContext; + /** + * The source of the event. + */ + source: AuditEventSource; }; - /** ISO 8601 timestamp of when the event was logged. */ + /** + * ISO 8601 timestamp of when the event occurred. + */ timestamp: string; } export type AuditEventName = - | 'createDocument'; + | 'createDocument' + | 'moveDocument' + | 'removeDocument' + | 'deleteDocument' + | 'restoreDocumentFromTrash' + | 'runSQLQuery'; -export interface AuditEventUser { - /** The user's id. */ - id: number | null; - /** The user's email address. */ - email: string | null; - /** The user's name. */ - name: string | null; +export type AuditEventUser = + | User + | Anonymous + | Unknown; + +interface User { + type: 'user'; + id: number; + email: string; + name: string; +} + +interface Anonymous { + type: 'anonymous'; +} + +interface Unknown { + type: 'unknown'; } export interface AuditEventDetails { + /** + * A new document was created. + */ createDocument: { - /** The ID of the document. */ + /** + * The ID of the document. + */ id: string; + /** + * The name of the document. + */ + name?: string; + }; + /** + * A document was moved to a new workspace. + */ + moveDocument: { + /** + * The ID of the document. + */ + id: string; + /** + * The previous workspace. + */ + previous: { + /** + * The workspace the document was moved from. + */ + workspace: { + /** + * The ID of the workspace. + */ + id: number; + /** + * The name of the workspace. + */ + name: string; + }; + }; + /** + * The current workspace. + */ + current: { + /** + * The workspace the document was moved to. + */ + workspace: { + /** + * The ID of the workspace. + */ + id: number; + /** + * The name of the workspace. + */ + name: string; + }; + }; + }; + /** + * A document was moved to the trash. + */ + removeDocument: { + /** + * The ID of the document. + */ + id: string; + /** + * The name of the document. + */ + name: string; + }; + /** + * A document was permanently deleted. + */ + deleteDocument: { + /** + * The ID of the document. + */ + id: string; + /** + * The name of the document. + */ + name: string; + }; + /** + * A document was restored from the trash. + */ + restoreDocumentFromTrash: { + /** + * The restored document. + */ + document: { + /** + * The ID of the document. + */ + id: string; + /** + * The name of the document. + */ + name: string; + }; + /** + * The workspace of the restored document. + */ + workspace: { + /** + * The ID of the workspace. + */ + id: number; + /** + * The name of the workspace. + */ + name: string; + }; + }; + /** + * A SQL query was run against a document. + */ + runSQLQuery: { + /** + * The SQL query. + */ + query: string; + /** + * The arguments used for query parameters, if any. + */ + arguments?: (string | number)[]; + /** + * The duration in milliseconds until query execution should time out. + */ + timeout?: number; }; } + +export interface AuditEventContext { + /** + * The ID of the workspace the event occurred in. + */ + workspaceId?: number; + /** + * The ID of the document the event occurred in. + */ + documentId?: string; +} + +export interface AuditEventSource { + /** + * The domain of the org tied to the originating request. + */ + org?: string; + /** + * The IP address of the originating request. + */ + ipAddress?: string; + /** + * The User-Agent HTTP header of the originating request. + */ + userAgent?: string; + /** + * The ID of the session tied to the originating request. + */ + sessionId?: string; +} diff --git a/app/gen-server/ApiServer.ts b/app/gen-server/ApiServer.ts index ba55cf04..3acc243d 100644 --- a/app/gen-server/ApiServer.ts +++ b/app/gen-server/ApiServer.ts @@ -8,8 +8,10 @@ import {ApiError} from 'app/common/ApiError'; import {FullUser} from 'app/common/LoginSessionAPI'; import {BasicRole} from 'app/common/roles'; import {OrganizationProperties, PermissionDelta} from 'app/common/UserAPI'; +import {Document} from "app/gen-server/entity/Document"; import {User} from 'app/gen-server/entity/User'; -import {BillingOptions, HomeDBManager, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {BillingOptions, HomeDBManager, Scope} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {PreviousAndCurrent, QueryResult} from 'app/gen-server/lib/homedb/Interfaces'; import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer'; import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession'; import {expressWrap} from 'app/server/lib/expressWrap'; @@ -259,37 +261,10 @@ export class ApiServer { // POST /api/workspaces/:wid/docs // Create a new doc owned by the specific workspace. this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => { - const mreq = req as RequestWithLogin; const wsId = integerParam(req.params.wid, 'wid'); - const query = await this._dbManager.addDocument(getScope(req), wsId, req.body); - const docId = query.data!; - this._gristServer.getTelemetry().logEvent(mreq, 'documentCreated', { - limited: { - docIdDigest: docId, - sourceDocIdDigest: undefined, - isImport: false, - fileType: undefined, - isSaved: true, - }, - full: { - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }); - this._gristServer.getTelemetry().logEvent(mreq, 'createdDoc-Empty', { - full: { - docIdDigest: docId, - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }); - this._gristServer.getAuditLogger().logEvent(mreq, { - event: { - name: 'createDocument', - details: {id: docId}, - }, - }); - return sendReply(req, res, query); + const result = await this._dbManager.addDocument(getScope(req), wsId, req.body); + if (result.status === 200) { this._logCreateDocumentEvents(req, result.data!); } + return sendReply(req, res, {...result, data: result.data?.id}); })); // GET /api/templates/ @@ -334,7 +309,8 @@ export class ApiServer { // Recover the specified doc if it was previously soft-deleted and is // still available. this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => { - await this._dbManager.undeleteDocument(getDocScope(req)); + const {status, data} = await this._dbManager.undeleteDocument(getDocScope(req)); + if (status === 200) { this._logRestoreDocumentEvents(req, data!); } return sendOkReply(req, res); })); @@ -375,9 +351,10 @@ export class ApiServer { // PATCH /api/docs/:did/move // Move the doc to the workspace specified in the body. this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => { - const workspaceId = req.body.workspace; - const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId); - return sendReply(req, res, query); + const workspaceId = integerParam(req.body.workspace, 'workspace'); + const result = await this._dbManager.moveDoc(getDocScope(req), workspaceId); + if (result.status === 200) { this._logMoveDocumentEvents(req, result.data!); } + return sendReply(req, res, {...result, data: result.data?.current.id}); })); this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => { @@ -647,6 +624,57 @@ export class ApiServer { return result; } + private _logCreateDocumentEvents(req: Request, document: Document) { + const mreq = req as RequestWithLogin; + const {id, name, workspace: {id: workspaceId}} = document; + this._gristServer.getAuditLogger().logEvent(mreq, { + event: { + name: 'createDocument', + details: {id, name}, + context: {workspaceId}, + }, + }); + this._gristServer.getTelemetry().logEvent(mreq, 'documentCreated', { + limited: { + docIdDigest: id, + sourceDocIdDigest: undefined, + isImport: false, + fileType: undefined, + isSaved: true, + }, + full: { + userId: mreq.userId, + altSessionId: mreq.altSessionId, + }, + }); + this._gristServer.getTelemetry().logEvent(mreq, 'createdDoc-Empty', { + full: { + docIdDigest: id, + userId: mreq.userId, + altSessionId: mreq.altSessionId, + }, + }); + } + + private _logRestoreDocumentEvents(req: Request, document: Document) { + const {workspace} = document; + this._gristServer.getAuditLogger().logEvent(req as RequestWithLogin, { + event: { + name: 'restoreDocumentFromTrash', + details: { + document: { + id: document.id, + name: document.name, + }, + workspace: { + id: workspace.id, + name: workspace.name, + }, + }, + }, + }); + } + private _logInvitedDocUserTelemetryEvents(mreq: RequestWithLogin, delta: PermissionDelta) { if (!delta.users) { return; } @@ -687,6 +715,32 @@ export class ApiServer { ); } } + + private _logMoveDocumentEvents(req: Request, {previous, current}: PreviousAndCurrent) { + this._gristServer.getAuditLogger().logEvent(req as RequestWithLogin, { + event: { + name: 'moveDocument', + details: { + id: current.id, + previous: { + workspace: { + id: previous.workspace.id, + name: previous.workspace.name, + }, + }, + current: { + workspace: { + id: current.workspace.id, + name: current.workspace.name, + }, + }, + }, + context: { + workspaceId: previous.workspace.id, + }, + }, + }); + } } /** diff --git a/app/gen-server/lib/homedb/HomeDBManager.ts b/app/gen-server/lib/homedb/HomeDBManager.ts index 24423d70..e9c4cc1c 100644 --- a/app/gen-server/lib/homedb/HomeDBManager.ts +++ b/app/gen-server/lib/homedb/HomeDBManager.ts @@ -41,7 +41,13 @@ import {User} from "app/gen-server/entity/User"; import {Workspace} from "app/gen-server/entity/Workspace"; import {Limit} from 'app/gen-server/entity/Limit'; import { - AvailableUsers, GetUserOptions, NonGuestGroup, Resource, UserProfileChange + AvailableUsers, + GetUserOptions, + NonGuestGroup, + PreviousAndCurrent, + QueryResult, + Resource, + UserProfileChange, } from 'app/gen-server/lib/homedb/Interfaces'; import {SUPPORT_EMAIL, UsersManager} from 'app/gen-server/lib/homedb/UsersManager'; import {Permissions} from 'app/gen-server/lib/Permissions'; @@ -111,12 +117,6 @@ const listPublicSites = appSettings.section('access').flag('listPublicSites').re // which is a burden under heavy traffic. const DOC_AUTH_CACHE_TTL = 5000; -export interface QueryResult { - status: number; - data?: T; - errMessage?: string; -} - // Maps from userId to group name, or null to inherit. export interface UserIdDelta { [userId: string]: roles.NonGuestRole|null; @@ -1496,8 +1496,12 @@ export class HomeDBManager extends EventEmitter { // by makeId(). The client should not be given control of the choice of docId. // This option is used during imports, where it is convenient not to add a row to the // document database until the document has actually been imported. - public async addDocument(scope: Scope, wsId: number, props: Partial, - docId?: string): Promise> { + public async addDocument( + scope: Scope, + wsId: number, + props: Partial, + docId?: string + ): Promise> { const name = props.name; if (!name) { return { @@ -1577,7 +1581,12 @@ export class HomeDBManager extends EventEmitter { }); // Saves the document as well as its new ACL Rules and Group. const groups = doc.aclRules.map(rule => rule.group); - const result = await manager.save([doc, ...doc.aclRules, ...doc.aliases, ...groups]); + const [data] = await manager.save<[Document, ...(AclRuleDoc|Alias|Group)[]]>([ + doc, + ...doc.aclRules, + ...doc.aliases, + ...groups, + ]); // Ensure that the creator is in the ws and org's guests group. Creator already has // access to the workspace (he is at least an editor), but we need to be sure that // even if he is removed from the workspace, he will still have access to this doc. @@ -1587,10 +1596,7 @@ export class HomeDBManager extends EventEmitter { // time), but they are ignoring any unique constraints errors. await this._repairWorkspaceGuests(scope, workspace.id, manager); await this._repairOrgGuests(scope, workspace.org.id, manager); - return { - status: 200, - data: (result[0] as Document).id - }; + return {status: 200, data}; }); } @@ -1751,9 +1757,9 @@ export class HomeDBManager extends EventEmitter { } // Checks that the user has REMOVE permissions to the given document. If not, throws an - // error. Otherwise deletes the given document. Returns an empty query result with - // status 200 on success. - public async deleteDocument(scope: DocScope): Promise> { + // error. Otherwise deletes the given document. Returns a query result with status 200 + // and the deleted document on success. + public async deleteDocument(scope: DocScope): Promise> { return await this._connection.transaction(async manager => { const {forkId} = parseUrlId(scope.urlId); if (forkId) { @@ -1767,8 +1773,9 @@ export class HomeDBManager extends EventEmitter { return queryResult; } const fork: Document = queryResult.data; - await manager.remove([fork]); - return {status: 200}; + const data = structuredClone(fork); + await manager.remove(fork); + return {status: 200, data}; } else { const docQuery = this._doc(scope, { manager, @@ -1785,22 +1792,23 @@ export class HomeDBManager extends EventEmitter { return queryResult; } const doc: Document = queryResult.data; - // Delete the doc and doc ACLs/groups. + const data = structuredClone(doc); const docGroups = doc.aclRules.map(docAcl => docAcl.group); + // Delete the doc and doc ACLs/groups. await manager.remove([doc, ...docGroups, ...doc.aclRules]); // Update guests of the workspace and org after removing this doc. await this._repairWorkspaceGuests(scope, doc.workspace.id, manager); await this._repairOrgGuests(scope, doc.workspace.org.id, manager); - return {status: 200}; + return {status: 200, data}; } }); } - public softDeleteDocument(scope: DocScope): Promise { + public softDeleteDocument(scope: DocScope): Promise> { return this._setDocumentRemovedAt(scope, new Date()); } - public async undeleteDocument(scope: DocScope): Promise { + public async undeleteDocument(scope: DocScope): Promise> { return this._setDocumentRemovedAt(scope, null); } @@ -2263,7 +2271,7 @@ export class HomeDBManager extends EventEmitter { public async moveDoc( scope: DocScope, wsId: number - ): Promise> { + ): Promise>> { return await this._connection.transaction(async manager => { // Get the doc const docQuery = this._doc(scope, { @@ -2285,6 +2293,7 @@ export class HomeDBManager extends EventEmitter { return docQueryResult; } const doc: Document = docQueryResult.data; + const previous = structuredClone(doc); if (doc.workspace.id === wsId) { return { status: 400, @@ -2354,7 +2363,11 @@ export class HomeDBManager extends EventEmitter { doc.aliases = undefined as any; // Saves the document as well as its new ACL Rules and Groups and the // updated guest group in the workspace. - await manager.save([doc, ...doc.aclRules, ...docGroups]); + const [current] = await manager.save<[Document, ...(AclRuleDoc|Group)[]]>([ + doc, + ...doc.aclRules, + ...docGroups, + ]); if (firstLevelUsers.length > 0) { // If the doc has first-level users, update the source and destination workspaces. await this._repairWorkspaceGuests(scope, oldWs.id, manager); @@ -2365,9 +2378,7 @@ export class HomeDBManager extends EventEmitter { await this._repairOrgGuests(scope, doc.workspace.org.id, manager); } } - return { - status: 200 - }; + return {status: 200, data: {previous, current}}; }); } @@ -4301,9 +4312,9 @@ export class HomeDBManager extends EventEmitter { if (!removedAt) { await this._checkRoomForAnotherDoc(doc.workspace, manager); } - await manager.createQueryBuilder() - .update(Document).set({removedAt}).where({id: doc.id}) - .execute(); + doc.removedAt = removedAt; + const data = await manager.save(doc); + return {status: 200, data}; }); } diff --git a/app/gen-server/lib/homedb/Interfaces.ts b/app/gen-server/lib/homedb/Interfaces.ts index 22eec249..6a0b5ff5 100644 --- a/app/gen-server/lib/homedb/Interfaces.ts +++ b/app/gen-server/lib/homedb/Interfaces.ts @@ -14,6 +14,11 @@ export interface QueryResult { errMessage?: string; } +export interface PreviousAndCurrent { + previous: T; + current: T; +} + export interface GetUserOptions { manager?: EntityManager; profile?: UserProfile; diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 4a1ff82e..b6810ae9 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -107,6 +107,15 @@ import {LogMethods} from "app/server/lib/LogMethods"; import {ISandboxOptions} from 'app/server/lib/NSandbox'; import {NullSandbox, UnavailableSandboxMethodError} from 'app/server/lib/NullSandbox'; import {DocRequests} from 'app/server/lib/Requests'; +import { + getAltSessionId, + getDocSessionAccess, + getDocSessionAccessOrNull, + getDocSessionUsage, + getFullUser, + getLogMeta, + getUserId, +} from 'app/server/lib/sessionUtils'; import {shortDesc} from 'app/server/lib/shortDesc'; import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader'; import {DocTriggers} from "app/server/lib/Triggers"; @@ -127,21 +136,12 @@ import {ActionHistoryImpl} from './ActionHistoryImpl'; import {ActiveDocImport, FileImportOptions} from './ActiveDocImport'; import {DocClients} from './DocClients'; import {DocPluginManager} from './DocPluginManager'; -import { - DocSession, - getDocSessionAccess, - getDocSessionAltSessionId, - getDocSessionUsage, - getDocSessionUser, - getDocSessionUserId, - makeExceptionalDocSession, - OptDocSession -} from './DocSession'; +import {DocSession, makeExceptionalDocSession, OptDocSession} from './DocSession'; import {createAttachmentsIndex, DocStorage, REMOVE_UNUSED_ATTACHMENTS_DELAY} from './DocStorage'; import {expandQuery, getFormulaErrorForExpandQuery} from './ExpandedQuery'; import {GranularAccess, GranularAccessForBundle} from './GranularAccess'; import {OnDemandActions} from './OnDemandActions'; -import {getLogMetaFromDocSession, getPubSubPrefix, getTelemetryMetaFromDocSession} from './serverUtils'; +import {getPubSubPrefix} from './serverUtils'; import {findOrAddAllEnvelope, Sharing} from './Sharing'; import cloneDeep = require('lodash/cloneDeep'); import flatten = require('lodash/flatten'); @@ -455,7 +455,7 @@ export class ActiveDoc extends EventEmitter { // Constructs metadata for logging, given a Client or an OptDocSession. public getLogMeta(docSession: OptDocSession|null, docMethod?: string): log.ILogMeta { return { - ...(docSession ? getLogMetaFromDocSession(docSession) : {}), + ...getLogMeta(docSession), docId: this._docName, ...(docMethod ? {docMethod} : {}), }; @@ -819,7 +819,7 @@ export class ActiveDoc extends EventEmitter { * It returns the list of rowIds for the rows created in the _grist_Attachments table. */ public async addAttachments(docSession: OptDocSession, uploadId: number): Promise { - const userId = getDocSessionUserId(docSession); + const userId = getUserId(docSession); const upload: UploadInfo = globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)); try { // We'll assert that the upload won't cause limits to be exceeded, retrying once after @@ -1212,7 +1212,7 @@ export class ActiveDoc extends EventEmitter { const options = sanitizeApplyUAOptions(unsanitizedOptions); const actionBundles = await this._actionHistory.getActions(actionNums); let fromOwnHistory: boolean = true; - const user = getDocSessionUser(docSession); + const user = getFullUser(docSession); let oldestSource: number = Date.now(); for (const [index, bundle] of actionBundles.entries()) { const actionNum = actionNums[index]; @@ -1403,7 +1403,7 @@ export class ActiveDoc extends EventEmitter { */ public async fork(docSession: OptDocSession): Promise { const dbManager = this._getHomeDbManagerOrFail(); - const user = getDocSessionUser(docSession); + const user = getFullUser(docSession); // For now, fork only if user can read everything (or is owner). // TODO: allow forks with partial content. if (!user || !await this.canDownload(docSession)) { @@ -1471,7 +1471,7 @@ export class ActiveDoc extends EventEmitter { public async getAccessToken(docSession: OptDocSession, options: AccessTokenOptions): Promise { const tokens = this._docManager.gristServer.getAccessTokens(); - const userId = getDocSessionUserId(docSession); + const userId = getUserId(docSession); const docId = this.docName; const access = getDocSessionAccess(docSession); // If we happen to be using a "readOnly" connection, max out at "readOnly" @@ -1585,7 +1585,7 @@ export class ActiveDoc extends EventEmitter { }; const isShared = new Set(); - const userId = getDocSessionUserId(docSession); + const userId = getUserId(docSession); if (!userId) { throw new Error('Cannot determine user'); } const parsed = parseUrlId(this.docName); @@ -2747,9 +2747,9 @@ export class ActiveDoc extends EventEmitter { } private _getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel { - const altSessionId = docSession ? getDocSessionAltSessionId(docSession) : undefined; + const altSessionId = getAltSessionId(docSession); return merge( - docSession ? getTelemetryMetaFromDocSession(docSession) : {}, + getTelemetryMeta(docSession), altSessionId ? {altSessionId} : {}, { limited: { @@ -3048,3 +3048,23 @@ export function createSandbox(options: { sandboxOptions, }); } + +/** + * Extract telemetry metadata from session. + */ +function getTelemetryMeta(docSession: OptDocSession|null): TelemetryMetadataByLevel { + if (!docSession) { return {}; } + + const access = getDocSessionAccessOrNull(docSession); + const user = getFullUser(docSession); + const {client} = docSession; + return { + limited: { + access, + }, + full: { + ...(user ? {userId: user.id} : {}), + ...(client ? client.getFullTelemetryMeta() : {}), // Client if present will repeat and add to user info. + }, + }; +} diff --git a/app/server/lib/Assistance.ts b/app/server/lib/Assistance.ts index e57b0e7b..40ed3579 100644 --- a/app/server/lib/Assistance.ts +++ b/app/server/lib/Assistance.ts @@ -6,16 +6,16 @@ import { AssistanceContext, AssistanceMessage, AssistanceRequest, - AssistanceResponse + AssistanceResponse, } from 'app/common/AssistancePrompts'; import {delay} from 'app/common/delay'; import {DocAction} from 'app/common/DocActions'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; -import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession'; +import {OptDocSession} from 'app/server/lib/DocSession'; import log from 'app/server/lib/log'; +import {getFullUser, getLogMeta} from 'app/server/lib/sessionUtils'; +import {createHash} from 'crypto'; import fetch from 'node-fetch'; -import {createHash} from "crypto"; -import {getLogMetaFromDocSession} from "./serverUtils"; // These are mocked/replaced in tests. // fetch is also replacing in the runCompletion script to add caching. @@ -559,13 +559,13 @@ async function completionToResponse( } function getUserHash(session: OptDocSession): string { - const user = getDocSessionUser(session); + const user = getFullUser(session); // Make it a bit harder to guess the user ID. const salt = "7a8sb6987asdb678asd687sad6boas7f8b6aso7fd"; const hashSource = `${user?.id} ${user?.ref} ${salt}`; const hash = createHash('sha256').update(hashSource).digest('base64'); // So that if we get feedback about a user ID hash, we can // search for the hash in the logs to find the original user ID. - log.rawInfo("getUserHash", {...getLogMetaFromDocSession(session), userRef: user?.ref, hash}); + log.rawInfo("getUserHash", {...getLogMeta(session), userRef: user?.ref, hash}); return hash; } diff --git a/app/server/lib/AuditLogger.ts b/app/server/lib/AuditLogger.ts index 38cc7e00..11d9cbf9 100644 --- a/app/server/lib/AuditLogger.ts +++ b/app/server/lib/AuditLogger.ts @@ -1,5 +1,5 @@ -import {AuditEventDetails, AuditEventName} from 'app/common/AuditEvent'; -import {RequestOrSession} from 'app/server/lib/requestUtils'; +import {AuditEvent, AuditEventContext, AuditEventDetails, AuditEventName} from 'app/common/AuditEvent'; +import {RequestOrSession} from 'app/server/lib/sessionUtils'; export interface IAuditLogger { /** @@ -7,16 +7,16 @@ export interface IAuditLogger { */ logEvent( requestOrSession: RequestOrSession, - props: AuditEventProperties + properties: AuditEventProperties ): void; /** - * Asynchronous variant of `logEvent`. + * Logs an audit event. * - * Throws on failure to log an event. + * Throws a `LogAuditEventError` on failure. */ logEventAsync( requestOrSession: RequestOrSession, - props: AuditEventProperties + properties: AuditEventProperties ): Promise; } @@ -30,6 +30,10 @@ export interface AuditEventProperties { * Additional event details. */ details?: AuditEventDetails[Name]; + /** + * The context of the event. + */ + context?: AuditEventContext; }; /** * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured. @@ -38,3 +42,15 @@ export interface AuditEventProperties { */ timestamp?: string; } + +export class LogAuditEventError extends Error { + public name = 'LogAuditEventError'; + + constructor(public auditEvent: AuditEvent, ...params: any[]) { + super(...params); + + if (Error.captureStackTrace) { + Error.captureStackTrace(this, LogAuditEventError); + } + } +} diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts index 0386a6d3..8d2eefd9 100644 --- a/app/server/lib/Authorizer.ts +++ b/app/server/lib/Authorizer.ts @@ -18,7 +18,7 @@ import {makeId} from 'app/server/lib/idUtils'; import log from 'app/server/lib/log'; import {IPermitStore, Permit} from 'app/server/lib/Permit'; import {AccessTokenInfo} from 'app/server/lib/AccessTokens'; -import {allowHost, getOriginUrl, optStringParam} from 'app/server/lib/requestUtils'; +import {allowHost, buildXForwardedForHeader, getOriginUrl, optStringParam} from 'app/server/lib/requestUtils'; import * as cookie from 'cookie'; import {NextFunction, Request, RequestHandler, Response} from 'express'; import {IncomingMessage} from 'http'; @@ -704,6 +704,7 @@ export function getTransitiveHeaders( const PermitHeader = req.get('Permit'); const Organization = (req as RequestWithOrg).org; const XRequestedWith = req.get('X-Requested-With'); + const UserAgent = req.get('User-Agent'); const Origin = req.get('Origin'); // Pass along the original Origin since it may // play a role in granular access control. @@ -713,6 +714,8 @@ export function getTransitiveHeaders( ...(Organization ? { Organization } : undefined), ...(PermitHeader ? { Permit: PermitHeader } : undefined), ...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined), + ...(UserAgent ? { 'User-Agent': UserAgent } : undefined), + ...buildXForwardedForHeader(req), ...((includeOrigin && Origin) ? { Origin } : undefined), }; const extraHeader = process.env.GRIST_FORWARD_AUTH_HEADER; diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts index ce2a9b0b..a16e93a2 100644 --- a/app/server/lib/Client.ts +++ b/app/server/lib/Client.ts @@ -1,28 +1,29 @@ import {ApiError} from 'app/common/ApiError'; import {BrowserSettings} from 'app/common/BrowserSettings'; -import {delay} from 'app/common/delay'; import {CommClientConnect, CommMessage, CommResponse, CommResponseError} from 'app/common/CommTypes'; +import {delay} from 'app/common/delay'; +import {normalizeEmail} from 'app/common/emails'; import {ErrorWithCode} from 'app/common/ErrorWithCode'; import {FullUser, UserProfile} from 'app/common/LoginSessionAPI'; import {TelemetryMetadata} from 'app/common/Telemetry'; import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI'; -import {normalizeEmail} from 'app/common/emails'; import {User} from 'app/gen-server/entity/User'; -import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; import {Authorizer} from 'app/server/lib/Authorizer'; import {ScopedSession} from 'app/server/lib/BrowserSession'; import type {Comm} from 'app/server/lib/Comm'; import {DocSession} from 'app/server/lib/DocSession'; -import log from 'app/server/lib/log'; -import {LogMethods} from "app/server/lib/LogMethods"; -import {MemoryPool} from 'app/server/lib/MemoryPool'; -import {shortDesc} from 'app/server/lib/shortDesc'; -import {fromCallback} from 'app/server/lib/serverUtils'; -import {i18n} from 'i18next'; -import * as crypto from 'crypto'; -import moment from 'moment'; import {GristServerSocket} from 'app/server/lib/GristServerSocket'; +import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; +import log from 'app/server/lib/log'; +import {LogMethods} from 'app/server/lib/LogMethods'; +import {MemoryPool} from 'app/server/lib/MemoryPool'; +import {fromCallback} from 'app/server/lib/serverUtils'; +import {shortDesc} from 'app/server/lib/shortDesc'; +import * as crypto from 'crypto'; +import {IncomingMessage} from 'http'; +import {i18n} from 'i18next'; +import moment from 'moment'; // How many messages and bytes to accumulate for a disconnected client before booting it. // The benefit is that a client who temporarily disconnects and reconnects without missing much, @@ -97,7 +98,8 @@ export class Client { private _missedMessagesTotalLength: number = 0; private _destroyTimer: NodeJS.Timer|null = null; private _destroyed: boolean = false; - private _websocket: GristServerSocket|null; + private _websocket: GristServerSocket|null = null; + private _req: IncomingMessage|null = null; private _org: string|null = null; private _profile: UserProfile|null = null; private _user: FullUser|undefined = undefined; @@ -130,8 +132,15 @@ export class Client { return this._locale; } - public setConnection(websocket: GristServerSocket, counter: string|null, browserSettings: BrowserSettings) { + public setConnection(options: { + websocket: GristServerSocket; + req: IncomingMessage; + counter: string|null; + browserSettings: BrowserSettings; + }) { + const {websocket, req, counter, browserSettings} = options; this._websocket = websocket; + this._req = req; this._counter = counter; this.browserSettings = browserSettings; @@ -140,6 +149,10 @@ export class Client { websocket.onmessage = (msg: string) => this._onMessage(msg); } + public getConnectionRequest(): IncomingMessage|null { + return this._req; + } + /** * Returns DocSession for the given docFD, or throws an exception if this doc is not open. */ diff --git a/app/server/lib/Comm.ts b/app/server/lib/Comm.ts index b414661a..d4174fc1 100644 --- a/app/server/lib/Comm.ts +++ b/app/server/lib/Comm.ts @@ -232,7 +232,7 @@ export class Comm extends EventEmitter { client.setSession(scopedSession); // Add a Session object to the client. client.setOrg((req as RequestWithOrg).org || ""); client.setProfile(profile); - client.setConnection(websocket, counter, browserSettings); + client.setConnection({websocket, req, counter, browserSettings}); await client.sendConnectMessage(newClient, reuseClient, lastSeqId, { serverVersion: this._serverVersion || version.gitcommit, diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts index ba2062b0..26148903 100644 --- a/app/server/lib/DocApi.ts +++ b/app/server/lib/DocApi.ts @@ -26,11 +26,11 @@ import {SchemaTypes} from "app/common/schema"; import {SortFunc} from 'app/common/SortFunc'; import {Sort} from 'app/common/SortSpec'; import {MetaRowRecord} from 'app/common/TableData'; -import {TelemetryMetadataByLevel} from "app/common/Telemetry"; import {WebhookFields} from "app/common/Triggers"; import TriggersTI from 'app/common/Triggers-ti'; import {DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI'; import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {QueryResult} from 'app/gen-server/lib/homedb/Interfaces'; import * as Types from "app/plugin/DocApiTypes"; import DocApiTypesTI from "app/plugin/DocApiTypes-ti"; import {GristObjCode} from "app/plugin/GristData"; @@ -54,8 +54,11 @@ import { RequestWithLogin } from 'app/server/lib/Authorizer'; import {DocManager} from "app/server/lib/DocManager"; -import {docSessionFromRequest, getDocSessionShare, makeExceptionalDocSession, - OptDocSession} from "app/server/lib/DocSession"; +import { + docSessionFromRequest, + makeExceptionalDocSession, + OptDocSession, +} from "app/server/lib/DocSession"; import {DocWorker} from "app/server/lib/DocWorker"; import {IDocWorkerMap} from "app/server/lib/DocWorkerMap"; import {DownloadOptions, parseExportParameters} from "app/server/lib/Export"; @@ -85,6 +88,7 @@ import { } from 'app/server/lib/requestUtils'; import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters'; import {localeFromRequest} from "app/server/lib/ServerLocale"; +import {getDocSessionShare} from "app/server/lib/sessionUtils"; import {isUrlAllowed, WebhookAction, WebHookSecret} from "app/server/lib/Triggers"; import {fetchDoc, globalUploadSet, handleOptionalUpload, handleUpload, makeAccessId} from "app/server/lib/uploads"; @@ -993,14 +997,16 @@ export class DocWorkerApi { // DELETE /api/docs/:docId // Delete the specified doc. this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => { - await this._removeDoc(req, res, true); + const {status, data} = await this._removeDoc(req, res, true); + if (status === 200) { this._logDeleteDocumentEvents(req, data!); } })); // POST /api/docs/:docId/remove // Soft-delete the specified doc. If query parameter "permanent" is set, // delete permanently. this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => { - await this._removeDoc(req, res, isParameterOn(req.query.permanent)); + const {status, data} = await this._removeDoc(req, res, isParameterOn(req.query.permanent)); + if (status === 200) { this._logRemoveDocumentEvents(req, data!); } })); this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => { @@ -1253,11 +1259,7 @@ export class DocWorkerApi { }, }, }); - this._logCreatedFileImportDocTelemetryEvent(req, { - full: { - docIdDigest: result.id, - }, - }); + this._logImportDocumentEvents(mreq, result); res.json(result); })); @@ -1402,11 +1404,7 @@ export class DocWorkerApi { }, }); docId = result.id; - this._logCreatedFileImportDocTelemetryEvent(req, { - full: { - docIdDigest: docId, - }, - }); + this._logImportDocumentEvents(mreq, result); } else if (workspaceId !== undefined) { docId = await this._createNewSavedDoc(req, { workspaceId: workspaceId, @@ -1661,7 +1659,7 @@ export class DocWorkerApi { } // Then, import the copy to the workspace. - const result = await this._docManager.importDocToWorkspace(mreq, { + const {id, title: name} = await this._docManager.importDocToWorkspace(mreq, { userId, uploadId: uploadResult.uploadId, documentName, @@ -1677,31 +1675,9 @@ export class DocWorkerApi { }, }, }); - - const sourceDocument = await this._dbManager.getRawDocById(sourceDocumentId); - const isTemplateCopy = sourceDocument.type === 'template'; - if (isTemplateCopy) { - this._grist.getTelemetry().logEvent(mreq, 'copiedTemplate', { - full: { - templateId: parseUrlId(sourceDocument.urlId || sourceDocument.id).trunkId, - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }); - } - this._grist.getTelemetry().logEvent( - mreq, - `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`, - { - full: { - docIdDigest: result.id, - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - } - ); - - return result.id; + this._logDuplicateDocumentEvents(mreq, {id: sourceDocumentId}, {id, name}) + .catch(e => log.error('DocApi failed to log duplicate document events', e)); + return id; } private async _createNewSavedDoc(req: Request, options: { @@ -1712,31 +1688,13 @@ export class DocWorkerApi { const {status, data, errMessage} = await this._dbManager.addDocument(getScope(req), workspaceId, { name: documentName ?? 'Untitled document', }); - const docId = data!; if (status !== 200) { throw new ApiError(errMessage || 'unable to create document', status); } - this._logDocumentCreatedTelemetryEvent(req, { - limited: { - docIdDigest: docId, - sourceDocIdDigest: undefined, - isImport: false, - fileType: undefined, - isSaved: true, - }, - }); - this._logCreatedEmptyDocTelemetryEvent(req, { - full: { - docIdDigest: docId, - }, - }); - this._grist.getAuditLogger().logEvent(req as RequestWithLogin, { - event: { - name: 'createDocument', - details: {id: docId}, - }, - }); - return docId; + + const {id, name, workspace} = data!; + this._logCreateDocumentEvents(req, {id, name, workspaceId: workspace.id}); + return id; } private async _createNewUnsavedDoc(req: Request, options: { @@ -1752,64 +1710,13 @@ export class DocWorkerApi { trunkDocId: NEW_DOCUMENT_CODE, trunkUrlId: NEW_DOCUMENT_CODE, }); - const docId = result.docId; - await this._docManager.createNamedDoc( + const id = result.docId; + const name = await this._docManager.createNamedDoc( makeExceptionalDocSession('nascent', {req: mreq, browserSettings}), - docId + id ); - this._logDocumentCreatedTelemetryEvent(req, { - limited: { - docIdDigest: docId, - sourceDocIdDigest: undefined, - isImport: false, - fileType: undefined, - isSaved: false, - }, - }); - this._logCreatedEmptyDocTelemetryEvent(req, { - full: { - docIdDigest: docId, - }, - }); - this._grist.getAuditLogger().logEvent(mreq, { - event: { - name: 'createDocument', - details: {id: docId}, - }, - }); - return docId; - } - - private _logDocumentCreatedTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) { - const mreq = req as RequestWithLogin; - this._grist.getTelemetry().logEvent(mreq, 'documentCreated', _.merge({ - full: { - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }, metadata)); - } - - private _logCreatedEmptyDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) { - this._logCreatedDocTelemetryEvent(req, 'createdDoc-Empty', metadata); - } - - private _logCreatedFileImportDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) { - this._logCreatedDocTelemetryEvent(req, 'createdDoc-FileImport', metadata); - } - - private _logCreatedDocTelemetryEvent( - req: Request, - event: 'createdDoc-Empty' | 'createdDoc-FileImport', - metadata: TelemetryMetadataByLevel, - ) { - const mreq = req as RequestWithLogin; - this._grist.getTelemetry().logEvent(mreq, event, _.merge({ - full: { - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }, metadata)); + this._logCreateDocumentEvents(req as RequestWithLogin, {id, name}); + return id; } /** @@ -2091,10 +1998,10 @@ export class DocWorkerApi { return result; } - private async _removeDoc(req: Request, res: Response, permanent: boolean) { - const mreq = req as RequestWithLogin; + private async _removeDoc(req: Request, res: Response, permanent: boolean): Promise> { const scope = getDocScope(req); const docId = getDocId(req); + let result: QueryResult; if (permanent) { const {forkId} = parseUrlId(docId); if (!forkId) { @@ -2110,22 +2017,16 @@ export class DocWorkerApi { ]; await Promise.all(docsToDelete.map(docName => this._docManager.deleteDoc(null, docName, true))); // Permanently delete from database. - const query = await this._dbManager.deleteDocument(scope); - this._dbManager.checkQueryResult(query); - this._grist.getTelemetry().logEvent(mreq, 'deletedDoc', { - full: { - docIdDigest: docId, - userId: mreq.userId, - altSessionId: mreq.altSessionId, - }, - }); - await sendReply(req, res, query); + result = await this._dbManager.deleteDocument(scope); + this._dbManager.checkQueryResult(result); + await sendReply(req, res, {...result, data: result.data!.id}); } else { - await this._dbManager.softDeleteDocument(scope); + result = await this._dbManager.softDeleteDocument(scope); await sendOkReply(req, res); } await this._dbManager.flushSingleDocAuthCache(scope, docId); await this._docManager.interruptDocClients(docId); + return result; } private async _runSql(activeDoc: ActiveDoc, req: RequestWithLogin, res: Response, @@ -2170,6 +2071,7 @@ export class DocWorkerApi { try { const records = await activeDoc.docStorage.all(wrappedStatement, ...(options.args || [])); + this._logRunSQLQueryEvents(req, options); res.status(200).json({ statement, records: records.map( @@ -2194,6 +2096,130 @@ export class DocWorkerApi { clearTimeout(interrupt); } } + + private _logCreateDocumentEvents( + req: Request, + document: {id: string; name?: string; workspaceId?: number} + ) { + const mreq = req as RequestWithLogin; + const {id, name, workspaceId} = document; + this._grist.getAuditLogger().logEvent(mreq, { + event: { + name: 'createDocument', + details: {id, name}, + context: {workspaceId}, + }, + }); + this._grist.getTelemetry().logEvent(mreq, 'documentCreated', { + limited: { + docIdDigest: id, + sourceDocIdDigest: undefined, + isImport: false, + fileType: undefined, + isSaved: workspaceId !== undefined, + }, + full: { + userId: mreq.userId, + altSessionId: mreq.altSessionId, + }, + }); + this._grist.getTelemetry().logEvent(mreq, 'createdDoc-Empty', { + limited: { + docIdDigest: id, + sourceDocIdDigest: undefined, + isImport: false, + fileType: undefined, + isSaved: workspaceId !== undefined, + }, + full: { + docIdDigest: id, + userId: mreq.userId, + altSessionId: mreq.altSessionId, + }, + }); + } + + private _logRemoveDocumentEvents(req: RequestWithLogin, document: Document) { + const {id, name, workspace: {id: workspaceId}} = document; + this._grist.getAuditLogger().logEvent(req, { + event: { + name: 'removeDocument', + details: {id, name}, + context: {workspaceId}, + }, + }); + } + + private _logDeleteDocumentEvents(req: RequestWithLogin, {id, name}: Document) { + this._grist.getAuditLogger().logEvent(req, { + event: { + name: 'deleteDocument', + details: {id, name}, + }, + }); + this._grist.getTelemetry().logEvent(req, 'deletedDoc', { + full: { + docIdDigest: id, + userId: req.userId, + altSessionId: req.altSessionId, + }, + }); + } + + private _logImportDocumentEvents( + req: RequestWithLogin, + {id}: {id: string} + ) { + this._grist.getTelemetry().logEvent(req, 'createdDoc-FileImport', { + full: { + docIdDigest: id, + userId: req.userId, + altSessionId: req.altSessionId, + }, + }); + } + + private async _logDuplicateDocumentEvents( + req: RequestWithLogin, + originalDocument: {id: string}, + newDocument: {id: string; name: string} + ) { + const document = await this._dbManager.getRawDocById(originalDocument.id); + const isTemplateCopy = document.type === 'template'; + if (isTemplateCopy) { + this._grist.getTelemetry().logEvent(req, 'copiedTemplate', { + full: { + templateId: parseUrlId(document.urlId || document.id).trunkId, + userId: req.userId, + altSessionId: req.altSessionId, + }, + }); + } + this._grist.getTelemetry().logEvent( + req, + `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`, + { + full: { + docIdDigest: newDocument.id, + userId: req.userId, + altSessionId: req.altSessionId, + }, + } + ); + } + + private _logRunSQLQueryEvents( + req: RequestWithLogin, + {sql: query, args, timeout}: Types.SqlPost + ) { + this._grist.getAuditLogger().logEvent(req, { + event: { + name: 'runSQLQuery', + details: {query, arguments: args, timeout}, + context: {documentId: getDocId(req)}, + }, + }); + } } export function addDocApiRoutes( diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts index 46046904..6a9f7c6e 100644 --- a/app/server/lib/DocManager.ts +++ b/app/server/lib/DocManager.ts @@ -19,17 +19,13 @@ import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer'; import {Client} from 'app/server/lib/Client'; -import { - getDocSessionCachedDoc, - makeExceptionalDocSession, - makeOptDocSession, - OptDocSession -} from 'app/server/lib/DocSession'; +import {makeExceptionalDocSession, makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession'; import * as docUtils from 'app/server/lib/docUtils'; import {GristServer} from 'app/server/lib/GristServer'; import {IDocStorageManager} from 'app/server/lib/IDocStorageManager'; import {makeForkIds, makeId} from 'app/server/lib/idUtils'; import {checkAllegedGristDoc} from 'app/server/lib/serverUtils'; +import {getDocSessionCachedDoc} from 'app/server/lib/sessionUtils'; import log from 'app/server/lib/log'; import {ActiveDoc} from './ActiveDoc'; import {PluginManager} from './PluginManager'; @@ -246,7 +242,6 @@ export class DocManager extends EventEmitter { register, userId, }); - this.gristServer.getTelemetry().logEvent(mreq, 'documentCreated', merge({ limited: { docIdDigest: docCreationInfo.id, @@ -254,13 +249,6 @@ export class DocManager extends EventEmitter { isSaved: workspaceId !== undefined, }, }, telemetryMetadata)); - this.gristServer.getAuditLogger().logEvent(mreq, { - event: { - name: 'createDocument', - details: {id: docCreationInfo.id}, - }, - }); - return docCreationInfo; // The imported document is associated with the worker that did the import. // We could break that association (see /api/docs/:docId/assign for how) if diff --git a/app/server/lib/DocSession.ts b/app/server/lib/DocSession.ts index 2036a01d..2efa465d 100644 --- a/app/server/lib/DocSession.ts +++ b/app/server/lib/DocSession.ts @@ -1,10 +1,6 @@ import {BrowserSettings} from 'app/common/BrowserSettings'; -import {DocumentUsage} from 'app/common/DocUsage'; -import {Role} from 'app/common/roles'; -import {FullUser} from 'app/common/UserAPI'; -import {Document} from 'app/gen-server/entity/Document'; import {ActiveDoc} from 'app/server/lib/ActiveDoc'; -import {Authorizer, getUser, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer'; import {Client} from 'app/server/lib/Client'; /** @@ -82,125 +78,3 @@ export class DocSession implements OptDocSession { // Browser settings (like timezone) obtained from the Client. public get browserSettings(): BrowserSettings { return this.client.browserSettings; } } - -/** - * Extract userId from OptDocSession. Use Authorizer if available (for web socket - * sessions), or get it from the Request if that is available (for rest api calls), - * or from the Client if that is available. Returns null if userId information is - * not available or not cached. - */ -export function getDocSessionUserId(docSession: OptDocSession): number|null { - if (docSession.authorizer) { - return docSession.authorizer.getUserId(); - } - if (docSession.req) { - return getUserId(docSession.req); - } - if (docSession.client) { - return docSession.client.getCachedUserId(); - } - return null; -} - -export function getDocSessionAltSessionId(docSession: OptDocSession): string|null { - if (docSession.req) { - return docSession.req.altSessionId || null; - } - if (docSession.client) { - return docSession.client.getAltSessionId() || null; - } - return null; -} - -/** - * Get as much of user profile as we can (id, name, email). - */ -export function getDocSessionUser(docSession: OptDocSession): FullUser|null { - if (docSession.authorizer) { - return docSession.authorizer.getUser(); - } - if (docSession.req) { - const user = getUser(docSession.req); - const email = user.loginEmail; - if (email) { - return {id: user.id, name: user.name, email, ref: user.ref, locale: user.options?.locale}; - } - } - if (docSession.client) { - const id = docSession.client.getCachedUserId(); - const ref = docSession.client.getCachedUserRef(); - const profile = docSession.client.getProfile(); - if (id && profile) { - return { - id, - ref, - ...profile - }; - } - } - return null; -} - -/** - * Extract user's role from OptDocSession. Method depends on whether using web - * sockets or rest api. Assumes that access has already been checked by wrappers - * for api methods and that cached access information is therefore available. - */ -export function getDocSessionAccess(docSession: OptDocSession): Role { - // "nascent" DocSessions are for when a document is being created, and user is - // its only owner as yet. - // "system" DocSessions are for access without access control. - if (docSession.mode === 'nascent' || docSession.mode === 'system') { return 'owners'; } - // "plugin" DocSessions are for access from plugins, which is currently quite crude, - // and granted only to editors. - if (docSession.mode === 'plugin') { return 'editors'; } - if (docSession.authorizer) { - const access = docSession.authorizer.getCachedAuth().access; - if (!access) { throw new Error('getDocSessionAccess expected authorizer.getCachedAuth'); } - return access; - } - if (docSession.req) { - const access = docSession.req.docAuth?.access; - if (!access) { throw new Error('getDocSessionAccess expected req.docAuth.access'); } - return access; - } - throw new Error('getDocSessionAccess could not find access information in DocSession'); -} - -export function getDocSessionShare(docSession: OptDocSession): string|null { - return _getCachedDoc(docSession)?.linkId || null; -} - -/** - * Get document usage seen in db when we were last checking document - * access. Not necessarily a live value when using a websocket - * (although we do recheck access periodically). - */ -export function getDocSessionUsage(docSession: OptDocSession): DocumentUsage|null { - return _getCachedDoc(docSession)?.usage || null; -} - -export function _getCachedDoc(docSession: OptDocSession): Document|null { - if (docSession.authorizer) { - return docSession.authorizer.getCachedAuth().cachedDoc || null; - } - if (docSession.req) { - return docSession.req.docAuth?.cachedDoc || null; - } - return null; -} - -export function getDocSessionAccessOrNull(docSession: OptDocSession): Role|null { - try { - return getDocSessionAccess(docSession); - } catch (err) { - return null; - } -} - -/** - * Get cached information about the document, if available. May be stale. - */ -export function getDocSessionCachedDoc(docSession: OptDocSession): Document|undefined { - return (docSession.req as RequestWithLogin)?.docAuth?.cachedDoc; -} diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts index 4bdf5369..e529ddb6 100644 --- a/app/server/lib/FlexServer.ts +++ b/app/server/lib/FlexServer.ts @@ -915,9 +915,9 @@ export class FlexServer implements GristServer { } public addAuditLogger() { - if (this._check('audit-logger')) { return; } + if (this._check('audit-logger', 'homedb')) { return; } - this._auditLogger = this.create.AuditLogger(); + this._auditLogger = this.create.AuditLogger(this._dbManager); } public async addTelemetry() { diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts index 95af9aac..315fcd47 100644 --- a/app/server/lib/GranularAccess.ts +++ b/app/server/lib/GranularAccess.ts @@ -38,8 +38,7 @@ import { FullUser, UserAccessData } from 'app/common/UserAPI'; import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager'; import { GristObjCode } from 'app/plugin/GristData'; import { DocClients } from 'app/server/lib/DocClients'; -import { getDocSessionAccess, getDocSessionAltSessionId, getDocSessionShare, - getDocSessionUser, OptDocSession } from 'app/server/lib/DocSession'; +import { OptDocSession } from 'app/server/lib/DocSession'; import { DocStorage, REMOVE_UNUSED_ATTACHMENTS_DELAY } from 'app/server/lib/DocStorage'; import log from 'app/server/lib/log'; import { IPermissionInfo, MixedPermissionSetWithContext, @@ -47,6 +46,12 @@ import { IPermissionInfo, MixedPermissionSetWithContext, import { TablePermissionSetWithContext } from 'app/server/lib/PermissionInfo'; import { integerParam } from 'app/server/lib/requestUtils'; import { getRelatedRows, getRowIdsFromDocAction } from 'app/server/lib/RowAccess'; +import { + getAltSessionId, + getDocSessionAccess, + getDocSessionShare, + getFullUser, +} from 'app/server/lib/sessionUtils'; import cloneDeep = require('lodash/cloneDeep'); import fromPairs = require('lodash/fromPairs'); import get = require('lodash/get'); @@ -379,7 +384,7 @@ export class GranularAccess implements GranularAccessForBundle { fullUser = attrs.override.user; } } else { - fullUser = getDocSessionUser(docSession); + fullUser = getFullUser(docSession); } const user = new User(); user.Access = access; @@ -395,7 +400,7 @@ export class GranularAccess implements GranularAccessForBundle { // Include origin info if accessed via the rest api. // TODO: could also get this for websocket access, just via a different route. user.Origin = docSession.req?.get('origin') || null; - user.SessionID = isAnonymous ? `a${getDocSessionAltSessionId(docSession)}` : `u${user.UserID}`; + user.SessionID = isAnonymous ? `a${getAltSessionId(docSession)}` : `u${user.UserID}`; user.IsLoggedIn = !isAnonymous; user.UserRef = fullUser?.ref || null; // Empty string should be treated as null. diff --git a/app/server/lib/HTTPAuditLogger.ts b/app/server/lib/HTTPAuditLogger.ts index 62e349c7..be7bbdaf 100644 --- a/app/server/lib/HTTPAuditLogger.ts +++ b/app/server/lib/HTTPAuditLogger.ts @@ -1,11 +1,19 @@ -import {AuditEvent, AuditEventName, AuditEventUser} from 'app/common/AuditEvent'; -import {AuditEventProperties, IAuditLogger} from 'app/server/lib/AuditLogger'; -import {getDocSessionUser} from 'app/server/lib/DocSession'; -import {ILogMeta, LogMethods} from 'app/server/lib/LogMethods'; -import {RequestOrSession} from 'app/server/lib/requestUtils'; -import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils'; +import {AuditEvent, AuditEventName, AuditEventSource, AuditEventUser} from 'app/common/AuditEvent'; +import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {AuditEventProperties, IAuditLogger, LogAuditEventError} from 'app/server/lib/AuditLogger'; +import {LogMethods} from 'app/server/lib/LogMethods'; +import {getOriginIpAddress} from 'app/server/lib/requestUtils'; +import { + getAltSessionId, + getFullUser, + getLogMeta, + getOrg, + getRequest, + RequestOrSession, +} from 'app/server/lib/sessionUtils'; import moment from 'moment-timezone'; import fetch from 'node-fetch'; +import {inspect} from 'util'; interface HTTPAuditLoggerOptions { /** @@ -31,35 +39,40 @@ const MAX_PENDING_REQUESTS = 25; * See `GristAuditLogger` for an example. */ export abstract class HTTPAuditLogger implements IAuditLogger { - private _endpoint = this._options.endpoint; - private _authorizationHeader = this._options.authorizationHeader; private _numPendingRequests = 0; - private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession: RequestOrSession | undefined) => + private readonly _endpoint = this._options.endpoint; + private readonly _authorizationHeader = this._options.authorizationHeader; + private readonly _logger = new LogMethods('AuditLogger ', (requestOrSession) => getLogMeta(requestOrSession)); - constructor(private _options: HTTPAuditLoggerOptions) {} + constructor(private _db: HomeDBManager, private _options: HTTPAuditLoggerOptions) {} /** * Logs an audit event. */ public logEvent( requestOrSession: RequestOrSession, - event: AuditEventProperties + properties: AuditEventProperties ): void { - this._logEventOrThrow(requestOrSession, event) - .catch((e) => this._logger.error(requestOrSession, `failed to log audit event`, event, e)); + this._logEventOrThrow(requestOrSession, properties) + .catch((e) => { + this._logger.error(requestOrSession, `failed to log audit event`, e); + this._logger.warn(requestOrSession, 'skipping audit event ', inspect(e.auditEvent, { + depth: Infinity, + })); + }); } /** - * Asynchronous variant of `logEvent`. + * Logs an audit event. * - * Throws on failure to log an event. + * Throws a LogAuditEventError on failure. */ public async logEventAsync( requestOrSession: RequestOrSession, - event: AuditEventProperties + properties: AuditEventProperties ): Promise { - await this._logEventOrThrow(requestOrSession, event); + await this._logEventOrThrow(requestOrSession, properties); } /** @@ -69,10 +82,14 @@ export abstract class HTTPAuditLogger implements IAuditLogger { private async _logEventOrThrow( requestOrSession: RequestOrSession, - {event: {name, details}, timestamp}: AuditEventProperties + properties: AuditEventProperties ) { + const event: AuditEvent = this._buildAuditEvent(requestOrSession, properties); if (this._numPendingRequests === MAX_PENDING_REQUESTS) { - throw new Error(`exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})`); + throw new LogAuditEventError( + event, + `exceeded the maximum number of pending audit event calls (${MAX_PENDING_REQUESTS})` + ); } try { @@ -83,53 +100,58 @@ export abstract class HTTPAuditLogger implements IAuditLogger { ...(this._authorizationHeader ? {'Authorization': this._authorizationHeader} : undefined), 'Content-Type': 'application/json', }, - body: this.toJSON({ - event: { - name, - user: getAuditEventUser(requestOrSession), - details: details ?? null, - }, - timestamp: timestamp ?? moment().toISOString(), - }), + body: this.toJSON(event), }); if (!resp.ok) { throw new Error(`received a non-200 response from ${resp.url}: ${resp.status} ${await resp.text()}`); } + } catch (e) { + throw new LogAuditEventError( + event, + e?.message ?? `failed to POST audit event to ${this._endpoint}`, + {cause: e} + ); } finally { this._numPendingRequests -= 1; } } -} -function getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser | null { - if (!requestOrSession) { return null; } - - if ('get' in requestOrSession) { + private _buildAuditEvent( + requestOrSession: RequestOrSession, + properties: AuditEventProperties + ): AuditEvent { + const {event: {name, details = {}, context = {}}, timestamp = moment().toISOString()} = properties; return { - id: requestOrSession.userId ?? null, - email: requestOrSession.user?.loginEmail ?? null, - name: requestOrSession.user?.name ?? null, + event: { + name, + user: this._getAuditEventUser(requestOrSession), + details, + context, + source: getAuditEventSource(requestOrSession), + }, + timestamp, }; - } else { - const user = getDocSessionUser(requestOrSession); - if (!user) { return null; } + } - const {id, email, name} = user; - return {id, email, name}; + private _getAuditEventUser(requestOrSession: RequestOrSession): AuditEventUser { + const user = getFullUser(requestOrSession); + if (!user) { + return {type: 'unknown'}; + } else if (user.id === this._db.getAnonymousUserId()) { + return {type: 'anonymous'}; + } else { + const {id, email, name} = user; + return {type: 'user', id, email, name}; + } } } -function getLogMeta(requestOrSession?: RequestOrSession): ILogMeta { - if (!requestOrSession) { return {}; } - - if ('get' in requestOrSession) { - return { - org: requestOrSession.org, - email: requestOrSession.user?.loginEmail, - userId: requestOrSession.userId, - altSessionId: requestOrSession.altSessionId, - }; - } else { - return getLogMetaFromDocSession(requestOrSession); - } +function getAuditEventSource(requestOrSession: RequestOrSession): AuditEventSource { + const request = getRequest(requestOrSession); + return { + org: getOrg(requestOrSession) || undefined, + ipAddress: request ? getOriginIpAddress(request) : undefined, + userAgent: request?.headers['user-agent'] || undefined, + sessionId: getAltSessionId(requestOrSession) || undefined, + }; } diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts index 1886ad81..77d2a8a2 100644 --- a/app/server/lib/ICreate.ts +++ b/app/server/lib/ICreate.ts @@ -70,7 +70,7 @@ export interface ICreate { Billing(dbManager: HomeDBManager, gristConfig: GristServer): IBilling; Notifier(dbManager: HomeDBManager, gristConfig: GristServer): INotifier; - AuditLogger(): IAuditLogger; + AuditLogger(dbManager: HomeDBManager): IAuditLogger; Telemetry(dbManager: HomeDBManager, gristConfig: GristServer): ITelemetry; Shell?(): IShell; // relevant to electron version of Grist only. @@ -120,7 +120,7 @@ export interface ICreateBillingOptions { export interface ICreateAuditLoggerOptions { name: 'grist'|'hec'; check(): boolean; - create(): IAuditLogger|undefined; + create(dbManager: HomeDBManager): IAuditLogger|undefined; } export interface ICreateTelemetryOptions { @@ -177,8 +177,8 @@ export function makeSimpleCreator(opts: { } return undefined; }, - AuditLogger() { - return auditLogger?.find(({check}) => check())?.create() ?? createDummyAuditLogger(); + AuditLogger(dbManager) { + return auditLogger?.find(({check}) => check())?.create(dbManager) ?? createDummyAuditLogger(); }, Telemetry(dbManager, gristConfig) { return telemetry?.create(dbManager, gristConfig) ?? createDummyTelemetry(); diff --git a/app/server/lib/Telemetry.ts b/app/server/lib/Telemetry.ts index 0cabb127..982d9cf9 100644 --- a/app/server/lib/Telemetry.ts +++ b/app/server/lib/Telemetry.ts @@ -19,13 +19,12 @@ import {Activation} from 'app/gen-server/entity/Activation'; import {Activations} from 'app/gen-server/lib/Activations'; import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {RequestWithLogin} from 'app/server/lib/Authorizer'; -import {getDocSessionUser} from 'app/server/lib/DocSession'; import {expressWrap} from 'app/server/lib/expressWrap'; import {GristServer} from 'app/server/lib/GristServer'; import {hashId} from 'app/server/lib/hashingUtils'; import {LogMethods} from 'app/server/lib/LogMethods'; -import {RequestOrSession, stringParam} from 'app/server/lib/requestUtils'; -import {getLogMetaFromDocSession} from 'app/server/lib/serverUtils'; +import {stringParam} from 'app/server/lib/requestUtils'; +import {getFullUser, getLogMeta, isRequest, RequestOrSession} from 'app/server/lib/sessionUtils'; import * as cookie from 'cookie'; import * as express from 'express'; import fetch from 'node-fetch'; @@ -73,8 +72,8 @@ export class Telemetry implements ITelemetry { private readonly _forwardTelemetryEventsUrl = process.env.GRIST_TELEMETRY_URL || 'https://telemetry.getgrist.com/api/telemetry'; private _numPendingForwardEventRequests = 0; - private readonly _logger = new LogMethods('Telemetry ', (requestOrSession: RequestOrSession | undefined) => - this._getLogMeta(requestOrSession)); + private readonly _logger = new LogMethods('Telemetry ', (requestOrSession) => + getLogMeta(requestOrSession)); private readonly _telemetryLogger = new LogMethods('Telemetry ', (eventType) => ({ eventType, })); @@ -273,14 +272,14 @@ export class Telemetry implements ITelemetry { if (requestOrSession) { let email: string | undefined; let org: string | undefined; - if ('get' in requestOrSession) { + if (isRequest(requestOrSession)) { email = requestOrSession.user?.loginEmail; org = requestOrSession.org; if (isAnonymousUser) { visitorId = this._getAndSetMatomoVisitorId(requestOrSession); } } else { - email = getDocSessionUser(requestOrSession)?.email; + email = getFullUser(requestOrSession)?.email; org = requestOrSession.client?.getOrg() ?? requestOrSession.req?.org; } if (email) { @@ -378,21 +377,6 @@ export class Telemetry implements ITelemetry { throw new ApiError('Telemetry is not ready', 500); } } - - private _getLogMeta(requestOrSession?: RequestOrSession) { - if (!requestOrSession) { return {}; } - - if ('get' in requestOrSession) { - return { - org: requestOrSession.org, - email: requestOrSession.user?.loginEmail, - userId: requestOrSession.userId, - altSessionId: requestOrSession.altSessionId, - }; - } else { - return getLogMetaFromDocSession(requestOrSession); - } - } } export async function getTelemetryPrefs( diff --git a/app/server/lib/configureGristAuditLogger.ts b/app/server/lib/configureGristAuditLogger.ts index 45f53726..120bb6ee 100644 --- a/app/server/lib/configureGristAuditLogger.ts +++ b/app/server/lib/configureGristAuditLogger.ts @@ -1,11 +1,12 @@ +import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager'; import {appSettings} from 'app/server/lib/AppSettings'; import {GristAuditLogger} from 'app/server/lib/GristAuditLogger'; -export function configureGristAuditLogger() { +export function configureGristAuditLogger(db: HomeDBManager) { const options = checkGristAuditLogger(); if (!options) { return undefined; } - return new GristAuditLogger(options); + return new GristAuditLogger(db, options); } export function checkGristAuditLogger() { diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts index 842e9b65..5ed0ae4d 100644 --- a/app/server/lib/requestUtils.ts +++ b/app/server/lib/requestUtils.ts @@ -1,9 +1,9 @@ import {ApiError} from 'app/common/ApiError'; import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls'; import * as gutil from 'app/common/gutil'; -import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {DocScope, Scope} from 'app/gen-server/lib/homedb/HomeDBManager'; +import {QueryResult} from 'app/gen-server/lib/homedb/Interfaces'; import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer'; -import {OptDocSession} from 'app/server/lib/DocSession'; import {RequestWithOrg} from 'app/server/lib/extractOrg'; import {RequestWithGrist} from 'app/server/lib/GristServer'; import log from 'app/server/lib/log'; @@ -13,8 +13,6 @@ import {IncomingMessage} from 'http'; import {Writable} from 'stream'; import {TLSSocket} from 'tls'; -export type RequestOrSession = RequestWithLogin | OptDocSession | null; - // log api details outside of dev environment (when GRIST_HOSTED_VERSION is set) const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION); @@ -347,6 +345,38 @@ export function getOriginUrl(req: IncomingMessage) { return `${protocol}://${host}`; } +/** + * Returns the original request IP address. + * + * If the request was made through a proxy or load balancer, the IP address + * is read from forwarded headers. See: + * + * - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For + * - https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/x-forwarded-headers.html + */ +export function getOriginIpAddress(req: IncomingMessage) { + return ( + // May contain multiple comma-separated values; the first one is the original. + (req.headers['x-forwarded-for'] as string | undefined) + ?.split(',') + .map(value => value.trim())[0] || + req.socket?.remoteAddress || + undefined + ); +} + +/** + * Returns the request's "X-Forwarded-For" header, with the request's IP address + * appended to its value. + * + * If the header is absent from the request, a new header will be returned. + */ +export function buildXForwardedForHeader(req: Request): {'X-Forwarded-For': string}|undefined { + const values = req.get('X-Forwarded-For')?.split(',').map(value => value.trim()) ?? []; + if (req.socket.remoteAddress) { values.push(req.socket.remoteAddress); } + return values.length > 0 ? { 'X-Forwarded-For': values.join(', ') } : undefined; +} + /** * Get the protocol to use in Grist URLs that are intended to be reachable * from a user's browser. Use the protocol in APP_HOME_URL if available, diff --git a/app/server/lib/serverUtils.ts b/app/server/lib/serverUtils.ts index ad14b708..43859b08 100644 --- a/app/server/lib/serverUtils.ts +++ b/app/server/lib/serverUtils.ts @@ -1,16 +1,15 @@ -import bluebird from 'bluebird'; -import { ChildProcess } from 'child_process'; -import * as net from 'net'; -import * as path from 'path'; -import { ConnectionOptions } from 'typeorm'; -import uuidv4 from 'uuid/v4'; -import {AbortSignal} from 'node-abort-controller'; - import {EngineCode} from 'app/common/DocumentSettings'; -import {TelemetryMetadataByLevel} from 'app/common/Telemetry'; +import {OptDocSession} from 'app/server/lib/DocSession'; import log from 'app/server/lib/log'; +import {getLogMeta} from 'app/server/lib/sessionUtils'; import {OpenMode, SQLiteDB} from 'app/server/lib/SQLiteDB'; -import {getDocSessionAccessOrNull, getDocSessionUser, OptDocSession} from './DocSession'; +import bluebird from 'bluebird'; +import {ChildProcess} from 'child_process'; +import * as net from 'net'; +import {AbortSignal} from 'node-abort-controller'; +import * as path from 'path'; +import {ConnectionOptions} from 'typeorm'; +import uuidv4 from 'uuid/v4'; // This method previously lived in this file. Re-export to avoid changing imports all over. export {timeoutReached} from 'app/common/gutil'; @@ -130,8 +129,11 @@ export async function checkAllegedGristDoc(docSession: OptDocSession, fname: str const integrityCheckResults = await db.all('PRAGMA integrity_check'); if (integrityCheckResults.length !== 1 || integrityCheckResults[0].integrity_check !== 'ok') { const uuid = uuidv4(); - log.info('Integrity check failure on import', {uuid, integrityCheckResults, - ...getLogMetaFromDocSession(docSession)}); + log.info('Integrity check failure on import', { + uuid, + integrityCheckResults, + ...getLogMeta(docSession), + }); throw new Error(`Document failed integrity checks - is it corrupted? Event ID: ${uuid}`); } } finally { @@ -139,39 +141,6 @@ export async function checkAllegedGristDoc(docSession: OptDocSession, fname: str } } -/** - * Extract access, userId, email, and client (if applicable) from session, for logging purposes. - */ -export function getLogMetaFromDocSession(docSession: OptDocSession) { - const client = docSession.client; - const access = getDocSessionAccessOrNull(docSession); - const user = getDocSessionUser(docSession); - const email = user?.loginEmail || user?.email; - return { - access, - ...(user ? {userId: user.id, email} : {}), - ...(client ? client.getLogMeta() : {}), // Client if present will repeat and add to user info. - }; -} - -/** - * Extract telemetry metadata from session. - */ -export function getTelemetryMetaFromDocSession(docSession: OptDocSession): TelemetryMetadataByLevel { - const client = docSession.client; - const access = getDocSessionAccessOrNull(docSession); - const user = getDocSessionUser(docSession); - return { - limited: { - access, - }, - full: { - ...(user ? {userId: user.id} : {}), - ...(client ? client.getFullTelemetryMeta() : {}), // Client if present will repeat and add to user info. - }, - }; -} - /** * Only offer choices of engine on experimental deployments (staging/dev). */ diff --git a/app/server/lib/sessionUtils.ts b/app/server/lib/sessionUtils.ts new file mode 100644 index 00000000..88c7bc9b --- /dev/null +++ b/app/server/lib/sessionUtils.ts @@ -0,0 +1,255 @@ +import {DocumentUsage} from 'app/common/DocUsage'; +import {FullUser} from 'app/common/LoginSessionAPI'; +import {Role} from 'app/common/roles'; +import {Document} from 'app/gen-server/entity/Document'; +import {getUserId as getRequestUserId, getUser, RequestWithLogin} from 'app/server/lib/Authorizer'; +import {OptDocSession} from 'app/server/lib/DocSession'; +import {ILogMeta} from 'app/server/lib/log'; +import {IncomingMessage} from 'http'; + +export type RequestOrSession = RequestWithLogin | OptDocSession | null; + +export function isRequest( + requestOrSession: RequestOrSession +): requestOrSession is RequestWithLogin { + return Boolean(requestOrSession && 'get' in requestOrSession); +} + +/** + * Extract the raw `IncomingMessage` from `requestOrSession`, if available. + */ +export function getRequest(requestOrSession: RequestOrSession): IncomingMessage | null { + if (!requestOrSession) { return null; } + + // The location of the request depends on the context, which include REST + // API calls to document endpoints and WebSocket sessions. + if (isRequest(requestOrSession)) { + return requestOrSession; + } else if (requestOrSession.req) { + // A REST API call to a document endpoint. + return requestOrSession.req; + } else if (requestOrSession.client) { + // A WebSocket session. + return requestOrSession.client.getConnectionRequest(); + } else { + return null; + } +} + +export function getAltSessionId(requestOrSession: RequestOrSession): string | null { + if (!requestOrSession) { return null; } + + if (isRequest(requestOrSession)) { + return requestOrSession.altSessionId || null; + } else { + return getDocSessionAltSessionId(requestOrSession); + } +} + +function getDocSessionAltSessionId(docSession: OptDocSession): string|null { + if (docSession.req) { + return docSession.req.altSessionId || null; + } + if (docSession.client) { + return docSession.client.getAltSessionId() || null; + } + return null; +} + +export function getUserId(requestOrSession: RequestOrSession): number|null { + if (!requestOrSession) { return null; } + + if (isRequest(requestOrSession)) { + return getRequestUserId(requestOrSession); + } else { + return getDocSessionUserId(requestOrSession); + } +} + +/** + * Extract userId from OptDocSession. Use Authorizer if available (for web socket + * sessions), or get it from the Request if that is available (for rest api calls), + * or from the Client if that is available. Returns null if userId information is + * not available or not cached. + */ +function getDocSessionUserId(docSession: OptDocSession): number|null { + if (docSession.authorizer) { + return docSession.authorizer.getUserId(); + } + if (docSession.req) { + return getUserId(docSession.req); + } + if (docSession.client) { + return docSession.client.getCachedUserId(); + } + return null; +} + +/** + * Get as much of user profile as we can (id, name, email). + */ +export function getFullUser(requestOrSession: RequestOrSession): FullUser | null { + if (!requestOrSession) { return null; } + + if (isRequest(requestOrSession)) { + return getRequestFullUser(requestOrSession); + } else { + return getDocSessionFullUser(requestOrSession); + } +} + +function getRequestFullUser(request: RequestWithLogin): FullUser|null { + const user = getUser(request); + if (!user.loginEmail) { return null; } + + const {id, name, loginEmail: email, ref, options} = user; + return { + id, + name, + email, + ref, + locale: options?.locale, + }; +} + +function getDocSessionFullUser(docSession: OptDocSession): FullUser|null { + if (docSession.authorizer) { + return docSession.authorizer.getUser(); + } + if (docSession.req) { + return getRequestFullUser(docSession.req); + } + if (docSession.client) { + const id = docSession.client.getCachedUserId(); + const ref = docSession.client.getCachedUserRef(); + const profile = docSession.client.getProfile(); + if (id && profile) { + return { + id, + ref, + ...profile + }; + } + } + return null; +} + +export function getOrg(requestOrSession: RequestOrSession): string | null { + if (!requestOrSession) { return null; } + + if (isRequest(requestOrSession)) { + return requestOrSession.org || null; + } else { + return getDocSessionOrg(requestOrSession); + } +} + +function getDocSessionOrg(docSession: OptDocSession): string | null { + if (docSession.req) { + return docSession.req.org || null; + } + if (docSession.client) { + return docSession.client.getOrg() || null; + } + return null; +} + +/** + * Extract access, userId, email, and client (if applicable) from + * `requestOrSession`, for logging purposes. + */ +export function getLogMeta(requestOrSession: RequestOrSession | undefined): ILogMeta { + if (!requestOrSession) { return {}; } + + if (isRequest(requestOrSession)) { + return getRequestLogMeta(requestOrSession); + } else { + return getDocSessionLogMeta(requestOrSession); + } +} + +function getRequestLogMeta(request: RequestWithLogin): ILogMeta { + const {org, user, userId, altSessionId} = request; + return { + org, + email: user?.loginEmail, + userId, + altSessionId, + }; +} + +function getDocSessionLogMeta(docSession: OptDocSession): ILogMeta { + const client = docSession.client; + const access = getDocSessionAccessOrNull(docSession); + const user = getDocSessionFullUser(docSession); + const email = user?.loginEmail || user?.email; + return { + access, + ...(user ? {userId: user.id, email} : {}), + ...(client ? client.getLogMeta() : {}), // Client if present will repeat and add to user info. + }; +} + +/** + * Extract user's role from OptDocSession. Method depends on whether using web + * sockets or rest api. Assumes that access has already been checked by wrappers + * for api methods and that cached access information is therefore available. + */ +export function getDocSessionAccess(docSession: OptDocSession): Role { + // "nascent" DocSessions are for when a document is being created, and user is + // its only owner as yet. + // "system" DocSessions are for access without access control. + if (docSession.mode === 'nascent' || docSession.mode === 'system') { return 'owners'; } + // "plugin" DocSessions are for access from plugins, which is currently quite crude, + // and granted only to editors. + if (docSession.mode === 'plugin') { return 'editors'; } + if (docSession.authorizer) { + const access = docSession.authorizer.getCachedAuth().access; + if (!access) { throw new Error('getDocSessionAccess expected authorizer.getCachedAuth'); } + return access; + } + if (docSession.req) { + const access = docSession.req.docAuth?.access; + if (!access) { throw new Error('getDocSessionAccess expected req.docAuth.access'); } + return access; + } + throw new Error('getDocSessionAccess could not find access information in DocSession'); +} + +export function getDocSessionShare(docSession: OptDocSession): string|null { + return _getCachedDoc(docSession)?.linkId || null; +} + +/** + * Get document usage seen in db when we were last checking document + * access. Not necessarily a live value when using a websocket + * (although we do recheck access periodically). + */ +export function getDocSessionUsage(docSession: OptDocSession): DocumentUsage|null { + return _getCachedDoc(docSession)?.usage || null; +} + +export function _getCachedDoc(docSession: OptDocSession): Document|null { + if (docSession.authorizer) { + return docSession.authorizer.getCachedAuth().cachedDoc || null; + } + if (docSession.req) { + return docSession.req.docAuth?.cachedDoc || null; + } + return null; +} + +export function getDocSessionAccessOrNull(docSession: OptDocSession): Role|null { + try { + return getDocSessionAccess(docSession); + } catch (err) { + return null; + } +} + +/** + * Get cached information about the document, if available. May be stale. + */ +export function getDocSessionCachedDoc(docSession: OptDocSession): Document | undefined { + return (docSession.req as RequestWithLogin)?.docAuth?.cachedDoc; +} diff --git a/test/server/lib/GristAuditLogger.ts b/test/server/lib/GristAuditLogger.ts index bfa6f7c3..6cec0481 100644 --- a/test/server/lib/GristAuditLogger.ts +++ b/test/server/lib/GristAuditLogger.ts @@ -52,10 +52,13 @@ describe('GristAuditLogger', function() { .post('/events', { event: { name: 'createDocument', - user: null, + user: {type: 'unknown'}, details: { id: 'docId', + name: 'docName', }, + context: {}, + source: {}, }, timestamp, }) @@ -64,7 +67,7 @@ describe('GristAuditLogger', function() { auditLogger.logEventAsync(null, { event: { name: 'createDocument', - details: {id: 'docId'}, + details: {id: 'docId', name: 'docName'}, }, timestamp, }) @@ -80,7 +83,7 @@ describe('GristAuditLogger', function() { auditLogger.logEventAsync(null, { event: { name: 'createDocument', - details: {id: 'docId'}, + details: {id: 'docId', name: 'docName'}, }, }), 'received a non-200 response from https://api.getgrist.com/events: 404 Not found' @@ -98,7 +101,7 @@ describe('GristAuditLogger', function() { void auditLogger.logEvent(null, { event: { name: 'createDocument', - details: {id: 'docId'}, + details: {id: 'docId', name: 'docName'}, }, }); } @@ -106,7 +109,7 @@ describe('GristAuditLogger', function() { auditLogger.logEventAsync(null, { event: { name: 'createDocument', - details: {id: 'docId'}, + details: {id: 'docId', name: 'docName'}, }, }), 'exceeded the maximum number of pending audit event calls (25)'