(core) Add remaining audit log events

Summary: Adds the remaining batch of audit log events, and a CLI utility to generate documentation for installation and site audit events. Test Plan: Manual. Reviewers: jarek Reviewed By: jarek Differential Revision: https://phab.getgrist.com/D4356
2026-03-02 04:09:24 +00:00 · 2024-09-30 13:11:01 -04:00
parent 1927c87413
commit bda7935714
15 changed files with 1837 additions and 311 deletions
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -36,6 +36,7 @@ import {
 import {ApiError} from 'app/common/ApiError';
 import {mapGetOrSet, MapWithTTL} from 'app/common/AsyncCreate';
 import {AttachmentColumns, gatherAttachmentIds, getAttachmentColumns} from 'app/common/AttachmentColumns';
+import {AuditEventName} from 'app/common/AuditEvent';
 import {WebhookMessageType} from 'app/common/CommTypes';
 import {
  BulkAddRecord,
@@ -92,6 +93,7 @@ import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
 import {AccessTokenOptions, AccessTokenResult, GristDocAPI, UIRowId} from 'app/plugin/GristAPI';
 import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {AssistanceContext} from 'app/common/AssistancePrompts';
+import {AuditEventProperties} from 'app/server/lib/AuditLogger';
 import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
@@ -115,6 +117,7 @@ import {
  getFullUser,
  getLogMeta,
  getUserId,
+  RequestOrSession,
 } from 'app/server/lib/sessionUtils';
 import {shortDesc} from 'app/server/lib/shortDesc';
 import {TableMetadataLoader} from 'app/server/lib/TableMetadataLoader';
@@ -1451,17 +1454,7 @@ export class ActiveDoc extends EventEmitter {
      }

      await dbManager.forkDoc(userId, doc, forkIds.forkId);
-
-      const isTemplate = doc.type === 'template';
-      this.logTelemetryEvent(docSession, 'documentForked', {
-        limited: {
-          forkIdDigest: forkIds.forkId,
-          forkDocIdDigest: forkIds.docId,
-          trunkIdDigest: doc.trunkId,
-          isTemplate,
-          lastActivity: doc.updatedAt,
-        },
-      });
+      this._logForkDocumentEvents(docSession, {originalDocument: doc, forkIds});
    } finally {
      await permitStore.removePermit(permitKey);
    }
@@ -1865,6 +1858,13 @@ export class ActiveDoc extends EventEmitter {
    });
  }

+  public logAuditEvent<Name extends AuditEventName>(
+    requestOrSession: RequestOrSession,
+    properties: AuditEventProperties<Name>
+  ) {
+    this._docManager.gristServer.getAuditLogger().logEvent(requestOrSession, properties);
+  }
+
  public logTelemetryEvent(
    docSession: OptDocSession | null,
    event: TelemetryEvent,
@@ -2961,6 +2961,38 @@ export class ActiveDoc extends EventEmitter {
    return  this._pyCall('start_timing');
  }

+  private _logForkDocumentEvents(docSession: OptDocSession, options: {
+    originalDocument: Document;
+    forkIds: ForkResult;
+  }) {
+    const {originalDocument, forkIds} = options;
+    this.logAuditEvent(docSession, {
+      event: {
+        name: 'forkDocument',
+        details: {
+          original: {
+            id: originalDocument.id,
+            name: originalDocument.name,
+          },
+          fork: {
+            id: forkIds.forkId,
+            documentId: forkIds.docId,
+            urlId: forkIds.urlId,
+          },
+        },
+        context: {documentId: originalDocument.id},
+      },
+    });
+    this.logTelemetryEvent(docSession, 'documentForked', {
+      limited: {
+        forkIdDigest: forkIds.forkId,
+        forkDocIdDigest: forkIds.docId,
+        trunkIdDigest: originalDocument.trunkId,
+        isTemplate: originalDocument.type === 'template',
+        lastActivity: originalDocument.updatedAt,
+      },
+    });
+  }
 }

 // Helper to initialize a sandbox action bundle with no values.
--- a/app/server/lib/AppEndpoint.ts
+++ b/app/server/lib/AppEndpoint.ts
@@ -153,30 +153,8 @@ export function attachAppEndpoint(options: AttachOptions): void {
      docStatus = workerInfo.docStatus;
      body = await workerInfo.resp.json();
    }
-
-    const isPublic = ((doc as unknown) as APIDocument).public ?? false;
-    const isSnapshot = Boolean(parseUrlId(urlId).snapshotId);
-    const isTemplate = doc.type === 'template';
-    if (isPublic || isTemplate) {
-      gristServer.getTelemetry().logEvent(mreq, 'documentOpened', {
-        limited: {
-          docIdDigest: docId,
-          access: doc.access,
-          isPublic,
-          isSnapshot,
-          isTemplate,
-          lastUpdated: doc.updatedAt,
-        },
-        full: {
-          siteId: doc.workspace.org.id,
-          siteType: doc.workspace.org.billingAccount.product.name,
-          userId: mreq.userId,
-          altSessionId: mreq.altSessionId,
-        },
-      });
-    }
-
-    if (isTemplate) {
+    logOpenDocumentEvents(mreq, {server: gristServer, doc, urlId});
+    if (doc.type === 'template') {
      // Keep track of the last template a user visited in the last hour.
      // If a sign-up occurs within that time period, we'll know which
      // template, if any, was viewed most recently.
@@ -232,3 +210,39 @@ export function attachAppEndpoint(options: AttachOptions): void {
  app.get('/:urlId([^-/]{12,})(/:slug([^/]+):remainder(*))?',
          ...docMiddleware, docHandler);
 }
+
+function logOpenDocumentEvents(req: RequestWithLogin, options: {
+  server: GristServer;
+  doc: Document;
+  urlId: string;
+}) {
+  const {server, doc, urlId} = options;
+  const {forkId, snapshotId} = parseUrlId(urlId);
+  server.getAuditLogger().logEvent(req, {
+    event: {
+      name: 'openDocument',
+      details: {id: doc.id, name: doc.name, urlId, forkId, snapshotId},
+    },
+  });
+
+  const isPublic = ((doc as unknown) as APIDocument).public ?? false;
+  const isTemplate = doc.type === 'template';
+  if (isPublic || isTemplate) {
+    server.getTelemetry().logEvent(req, 'documentOpened', {
+      limited: {
+        docIdDigest: doc.id,
+        access: doc.access,
+        isPublic,
+        isSnapshot: Boolean(snapshotId),
+        isTemplate,
+        lastUpdated: doc.updatedAt,
+      },
+      full: {
+        siteId: doc.workspace.org.id,
+        siteType: doc.workspace.org.billingAccount.product.name,
+        userId: req.userId,
+        altSessionId: req.altSessionId,
+      },
+    });
+  }
+}
--- a/app/server/lib/AuditLogger.ts
+++ b/app/server/lib/AuditLogger.ts
@@ -1,4 +1,4 @@
-import {AuditEvent, AuditEventContext, AuditEventDetails, AuditEventName} from 'app/common/AuditEvent';
+import {AuditEvent, AuditEventContext, AuditEventDetails, AuditEventName, AuditEventUser} from 'app/common/AuditEvent';
 import {RequestOrSession} from 'app/server/lib/sessionUtils';

 export interface IAuditLogger {
@@ -23,20 +23,24 @@ export interface IAuditLogger {
 export interface AuditEventProperties<Name extends AuditEventName> {
  event: {
    /**
-     * The event name.
+     * The name of the event.
     */
    name: Name;
    /**
-     * Additional event details.
+     * Event-specific details (e.g. properties of affected resources).
     */
    details?: AuditEventDetails[Name];
    /**
-     * The context of the event.
+     * The context that the event occurred in (e.g. workspace, document).
     */
    context?: AuditEventContext;
+    /**
+     * The user that triggered the event.
+     */
+    user?: AuditEventUser;
  };
  /**
-   * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occured.
+   * ISO 8601 timestamp (e.g. `2024-09-04T14:54:50Z`) of when the event occurred.
   *
   * Defaults to now.
   */
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -906,8 +906,10 @@ export class DocWorkerApi {
    // Clears all outgoing webhooks in the queue for this document.
    this._app.delete('/api/docs/:docId/webhooks/queue', isOwner,
      withDocTriggersLock(async (activeDoc, req, res) => {
+        const docId = getDocId(req);
        await activeDoc.clearWebhookQueue();
        await activeDoc.sendWebhookNotification();
+        this._logClearAllWebhookQueueEvents(req, {docId});
        res.json({success: true});
      })
    );
@@ -933,7 +935,7 @@ export class DocWorkerApi {
        const webhookId = req.params.webhookId;
        const {fields, url, authorization} = await getWebhookSettings(activeDoc, req, webhookId, req.body);
        if (fields.enabled === false) {
-          await activeDoc.triggers.clearSingleWebhookQueue(webhookId);
+          await activeDoc.clearSingleWebhookQueue(webhookId);
        }

        const triggerRowId = activeDoc.triggers.getWebhookTriggerRecord(webhookId).id;
@@ -960,9 +962,11 @@ export class DocWorkerApi {
    // Clears a single webhook in the queue for this document.
    this._app.delete('/api/docs/:docId/webhooks/queue/:webhookId', isOwner,
      withDocTriggersLock(async (activeDoc, req, res) => {
+        const docId = getDocId(req);
        const webhookId = req.params.webhookId;
        await activeDoc.clearSingleWebhookQueue(webhookId);
        await activeDoc.sendWebhookNotification();
+        this._logClearWebhookQueueEvents(req, {docId, webhookId});
        res.json({success: true});
      })
    );
@@ -978,8 +982,10 @@ export class DocWorkerApi {
    // reopened on use).
    this._app.post('/api/docs/:docId/force-reload', canEdit, async (req, res) => {
      const mreq = req as RequestWithLogin;
+      const docId = getDocId(req);
      const activeDoc = await this._getActiveDoc(mreq);
      await activeDoc.reloadDoc();
+      this._logReloadDocumentEvents(mreq, {docId});
      res.json(null);
    });

@@ -997,16 +1003,16 @@ export class DocWorkerApi {
    // DELETE /api/docs/:docId
    // Delete the specified doc.
    this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => {
-      const {status, data} = await this._removeDoc(req, res, true);
-      if (status === 200) { this._logDeleteDocumentEvents(req, data!); }
+      const {data} = await this._removeDoc(req, res, true);
+      if (data) { this._logDeleteDocumentEvents(req, data); }
    }));

    // POST /api/docs/:docId/remove
    // Soft-delete the specified doc.  If query parameter "permanent" is set,
    // delete permanently.
    this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => {
-      const {status, data} = await this._removeDoc(req, res, isParameterOn(req.query.permanent));
-      if (status === 200) { this._logRemoveDocumentEvents(req, data!); }
+      const {data} = await this._removeDoc(req, res, isParameterOn(req.query.permanent));
+      if (data) { this._logRemoveDocumentEvents(req, data); }
    }));

    this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => {
@@ -1100,6 +1106,7 @@ export class DocWorkerApi {
    // This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it
    // starts with to become muted.
    this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => {
+      const docId = getDocId(req);
      const docSession = docSessionFromRequest(req);
      const activeDoc = await this._getActiveDoc(req);
      const options: DocReplacementOptions = {};
@@ -1160,6 +1167,9 @@ export class DocWorkerApi {
        options.snapshotId = String(req.body.snapshotId);
      }
      await activeDoc.replace(docSession, options);
+      const previous = {id: docId};
+      const current = {id: options.sourceDocId || docId, snapshotId: options.snapshotId};
+      this._logReplaceDocumentEvents(req, {previous, current});
      res.json(null);
    }));

@@ -1169,9 +1179,12 @@ export class DocWorkerApi {
    }));

    this._app.post('/api/docs/:docId/states/remove', isOwner, withDoc(async (activeDoc, req, res) => {
+      const docId = getDocId(req);
      const docSession = docSessionFromRequest(req);
      const keep = integerParam(req.body.keep, 'keep');
-      res.json(await activeDoc.deleteActions(docSession, keep));
+      await activeDoc.deleteActions(docSession, keep);
+      this._logTruncateDocumentHistoryEvents(req, {docId, keep});
+      res.json(null);
    }));

    this._app.get('/api/docs/:docId/compare/:docId2', canView, withDoc(async (activeDoc, req, res) => {
@@ -1675,7 +1688,11 @@ export class DocWorkerApi {
        },
      },
    });
-    this._logDuplicateDocumentEvents(mreq, {id: sourceDocumentId}, {id, name})
+    this._logDuplicateDocumentEvents(mreq, {
+      originalDocument: {id: sourceDocumentId},
+      duplicateDocument: {id, name},
+      asTemplate,
+    })
      .catch(e => log.error('DocApi failed to log duplicate document events', e));
    return id;
  }
@@ -2029,8 +2046,13 @@ export class DocWorkerApi {
    return result;
  }

-  private async _runSql(activeDoc: ActiveDoc, req: RequestWithLogin, res: Response,
-      options: Types.SqlPost) {
+  private async _runSql(
+    activeDoc: ActiveDoc,
+    req: RequestWithLogin,
+    res: Response,
+    options: Types.SqlPost
+  ) {
+    const docId = getDocId(req);
    if (!await activeDoc.canCopyEverything(docSessionFromRequest(req))) {
      throw new ApiError('insufficient document access', 403);
    }
@@ -2071,7 +2093,7 @@ export class DocWorkerApi {
    try {
      const records = await activeDoc.docStorage.all(wrappedStatement,
                                                     ...(options.args || []));
-      this._logRunSQLQueryEvents(req, options);
+      this._logRunSQLQueryEvents(req, {docId, ...options});
      res.status(200).json({
        statement,
        records: records.map(
@@ -2124,13 +2146,6 @@ export class DocWorkerApi {
      },
    });
    this._grist.getTelemetry().logEvent(mreq, 'createdDoc-Empty', {
-      limited: {
-        docIdDigest: id,
-        sourceDocIdDigest: undefined,
-        isImport: false,
-        fileType: undefined,
-        isSaved: workspaceId !== undefined,
-      },
      full: {
        docIdDigest: id,
        userId: mreq.userId,
@@ -2179,17 +2194,64 @@ export class DocWorkerApi {
    });
  }

-  private async _logDuplicateDocumentEvents(
-    req: RequestWithLogin,
-    originalDocument: {id: string},
-    newDocument: {id: string; name: string}
-  ) {
-    const document = await this._dbManager.getRawDocById(originalDocument.id);
-    const isTemplateCopy = document.type === 'template';
+  private _logReplaceDocumentEvents(req: RequestWithLogin, options: {
+    previous: {id: string};
+    current: {id: string; snapshotId?: string};
+  }) {
+    const {previous, current} = options;
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'replaceDocument',
+        details: {
+          previous: {
+            id: previous.id,
+          },
+          current: {
+            id: current.id,
+            snapshotId: current.snapshotId,
+          },
+        },
+        context: {documentId: previous.id},
+      },
+    });
+  }
+
+  private async _logDuplicateDocumentEvents(req: RequestWithLogin, options: {
+    originalDocument: {id: string};
+    duplicateDocument: {id: string; name: string};
+    asTemplate: boolean;
+  }) {
+    const {originalDocument: {id}, duplicateDocument, asTemplate} = options;
+    const originalDocument = await this._dbManager.getRawDocById(id);
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'duplicateDocument',
+        details: {
+          original: {
+            id: originalDocument.id,
+            name: originalDocument.name,
+            workspace: {
+              id: originalDocument.workspace.id,
+              name: originalDocument.workspace.name,
+            },
+          },
+          duplicate: {
+            id: duplicateDocument.id,
+            name: duplicateDocument.name,
+          },
+          asTemplate,
+        },
+        context: {
+          workspaceId: originalDocument.workspace.id,
+          documentId: originalDocument.id,
+        },
+      },
+    });
+    const isTemplateCopy = originalDocument.type === 'template';
    if (isTemplateCopy) {
      this._grist.getTelemetry().logEvent(req, 'copiedTemplate', {
        full: {
-          templateId: parseUrlId(document.urlId || document.id).trunkId,
+          templateId: parseUrlId(originalDocument.urlId || originalDocument.id).trunkId,
          userId: req.userId,
          altSessionId: req.altSessionId,
        },
@@ -2200,7 +2262,7 @@ export class DocWorkerApi {
      `createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`,
      {
        full: {
-          docIdDigest: newDocument.id,
+          docIdDigest: duplicateDocument.id,
          userId: req.userId,
          altSessionId: req.altSessionId,
        },
@@ -2208,15 +2270,60 @@ export class DocWorkerApi {
    );
  }

-  private _logRunSQLQueryEvents(
+  private _logReloadDocumentEvents(req: RequestWithLogin, {docId: documentId}: {docId: string}) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'reloadDocument',
+        context: {documentId},
+      },
+    });
+  }
+
+  private _logTruncateDocumentHistoryEvents(
    req: RequestWithLogin,
-    {sql: query, args, timeout}: Types.SqlPost
+    {docId: documentId, keep}: {docId: string; keep: number}
  ) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'truncateDocumentHistory',
+        details: {keep},
+        context: {documentId},
+      },
+    });
+  }
+
+  private _logClearWebhookQueueEvents(
+    req: RequestWithLogin,
+    {docId: documentId, webhookId: id}: {docId: string; webhookId: string}
+  ) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'clearWebhookQueue',
+        details: {id},
+        context: {documentId},
+      },
+    });
+  }
+
+  private _logClearAllWebhookQueueEvents(
+    req: RequestWithLogin,
+    {docId: documentId}: {docId: string}
+  ) {
+    this._grist.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'clearAllWebhookQueues',
+        context: {documentId},
+      },
+    });
+  }
+
+  private _logRunSQLQueryEvents(req: RequestWithLogin, options: {docId: string} & Types.SqlPost) {
+    const {docId: documentId, sql: query, args, timeout: timeoutMs} = options;
    this._grist.getAuditLogger().logEvent(req, {
      event: {
        name: 'runSQLQuery',
-        details: {query, arguments: args, timeout},
-        context: {documentId: getDocId(req)},
+        details: {query, arguments: args, timeoutMs},
+        context: {documentId},
      },
    });
  }
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -1492,7 +1492,7 @@ export class FlexServer implements GristServer {
        // to other (not public) team sites.
        const doom = await createDoom();
        await doom.deleteUser(userId);
-        this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedAccount');
+        this._logDeleteUserEvents(req as RequestWithLogin);
        return resp.status(200).json(true);
      }));

@@ -1523,16 +1523,10 @@ export class FlexServer implements GristServer {
        }

        // Reuse Doom cli tool for org deletion. Note, this removes everything as a super user.
+        const deletedOrg = structuredClone(org);
        const doom = await createDoom();
        await doom.deleteOrg(org.id);
-
-        this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedSite', {
-          full: {
-            siteId: org.id,
-            userId: mreq.userId,
-          },
-        });
-
+        this._logDeleteSiteEvents(mreq, deletedOrg);
        return resp.status(200).send();
      }));
    }
@@ -2548,6 +2542,30 @@ export class FlexServer implements GristServer {

    return isGristLogHttpEnabled || deprecatedOptionEnablesLog;
  }
+
+  private _logDeleteUserEvents(req: RequestWithLogin) {
+    this.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'deleteUser',
+      },
+    });
+    this.getTelemetry().logEvent(req, 'deletedAccount');
+  }
+
+  private _logDeleteSiteEvents(req: RequestWithLogin, {id, name}: Organization) {
+    this.getAuditLogger().logEvent(req, {
+      event: {
+        name: 'deleteSite',
+        details: {id, name},
+      }
+    });
+    this.getTelemetry().logEvent(req, 'deletedSite', {
+      full: {
+        siteId: id,
+        userId: req.userId,
+      },
+    });
+  }
 }

 /**
--- a/app/server/lib/GoogleExport.ts
+++ b/app/server/lib/GoogleExport.ts
@@ -3,7 +3,7 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {streamXLSX} from 'app/server/lib/ExportXLSX';
 import log from 'app/server/lib/log';
-import {optStringParam} from 'app/server/lib/requestUtils';
+import {getDocId, optStringParam} from 'app/server/lib/requestUtils';
 import {Request, Response} from 'express';
 import {PassThrough, Stream} from 'stream';

@@ -22,6 +22,7 @@ export async function exportToDrive(
    throw new Error("No access token - Can't send file to Google Drive");
  }

+  const docId = getDocId(req);
  const mreq = req as RequestWithLogin;
  const meta = {
    docId: activeDoc.docName,
@@ -39,6 +40,13 @@ export async function exportToDrive(
      streamXLSX(activeDoc, req, stream, {tableId: ''}),
      sendFileToDrive(name, stream, access_token),
    ]);
+    activeDoc.logAuditEvent(mreq, {
+      event: {
+        name: 'sendToGoogleDrive',
+        details: {id: docId},
+        context: {documentId: docId},
+      },
+    });
    log.debug(`Export to drive - File exported, redirecting to Google Spreadsheet ${url}`, meta);
    res.json({ url });
  } catch (err) {
--- a/app/server/lib/Triggers.ts
+++ b/app/server/lib/Triggers.ts
@@ -691,17 +691,16 @@ export class DocTriggers {
      if (this._loopAbort.signal.aborted) {
        continue;
      }
-      let meta: Record<string, any>|undefined;
-
+      let meta: {webhookId: string; host: string, quantity: number} | undefined;
      let success: boolean;
      if (!url) {
        success = true;
      } else {
        await this._stats.logStatus(id, 'sending');
-        meta = {numEvents: batch.length, webhookId: id, host: new URL(url).host};
+        meta = {webhookId: id, host: new URL(url).host, quantity: batch.length};
        this._log("Sending batch of webhook events", meta);
        this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
-          limited: {numEvents: meta.numEvents},
+          limited: {numEvents: meta.quantity},
        });
        success = await this._sendWebhookWithRetries(
          id, url, authorization, body, batch.length, this._loopAbort.signal);
@@ -743,6 +742,14 @@ export class DocTriggers {
        await this._stats.logStatus(id, 'idle');
        if (meta) {
          this._log("Successfully sent batch of webhook events", meta);
+          const {webhookId, host, quantity} = meta;
+          this._activeDoc.logAuditEvent(null, {
+            event: {
+              name: 'deliverWebhookEvents',
+              details: {id: webhookId, host, quantity},
+              user: {type: 'system'},
+            },
+          });
        }
      }