(core) discount indirect changes for access control purposes

Summary:
This diff discounts indirect changes for access control purposes.  A UserAction that updates a cell A, which in turn causes changes in other dependent cells, will be considered a change to cell A for access control purposes.

The `engine.apply_user_actions` method now returns a `direct` array, with a boolean for each `stored` action, set to `true` if the action is attributed to the user or `false` if it is attributed to the engine.  `GranularAccess` ignores actions attributed to the engine when checking for edit rights.

Subtleties:
 * Removal of references to a removed row are considered direct changes.
 * Doesn't play well with undos as yet.  An action that indirectly modifies a cell the user doesn't have rights to may succeed, but it will not be reversible.

Test Plan: added tests, updated tests

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2806

app/common/ActionBundle.ts

@@ -57,6 +57,7 @@ export interface UserActionBundle {
 export interface SandboxActionBundle {
   envelopes: Envelope[];
   stored: Array<EnvContent<DocAction>>;
+  direct: Array<EnvContent<boolean>>;
   calc: Array<EnvContent<DocAction>>;
   undo: Array<EnvContent<DocAction>>;   // Inverse actions for all 'stored' actions.
   retValues: any[];                     // Contains retValue for each of userActions.

app/gen-server/lib/Usage.ts

@@ -14,49 +14,66 @@ const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000;   // log every 1 hour
  */
 export class Usage {
   private _interval: NodeJS.Timeout;
+  private _currentOperation?: Promise<void>;
 
   public constructor(private _dbManager: HomeDBManager) {
-    this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS);
+    this._interval = setInterval(() => this.apply(), USAGE_PERIOD_MS);
     // Log once at beginning, in case we roll over servers faster than
     // the logging period for an extended length of time,
     // and to raise the visibility of this logging step so if it gets
     // slow devs notice.
-    this.apply().catch(log.warn.bind(log));
+    this.apply();
   }
 
-  public close() {
+  /**
+   * Remove any scheduled operation, and wait for the current one to complete
+   * (if one is in progress).
+   */
+  public async close() {
     clearInterval(this._interval);
+    await this._currentOperation;
   }
 
-  public async apply() {
-    const manager = this._dbManager.connection.manager;
-    // raw count of users
-    const userCount = await manager.count(User);
-    // users who have logged in at least once
-    const userWithLoginCount = await manager.createQueryBuilder()
-      .from(User, 'users')
-      .where('first_login_at is not null')
-      .getCount();
-    // raw count of organizations (excluding personal orgs)
-    const orgCount = await manager.createQueryBuilder()
-      .from(Organization, 'orgs')
-      .where('owner_id is null')
-      .getCount();
-    // organizations with subscriptions that are in a non-terminated state
-    const orgInGoodStandingCount = await manager.createQueryBuilder()
-      .from(Organization, 'orgs')
-      .leftJoin('orgs.billingAccount', 'billing_accounts')
-      .where('owner_id is null')
-      .andWhere('billing_accounts.in_good_standing = true')
-      .getCount();
-    // raw count of documents
-    const docCount = await manager.count(Document);
-    log.rawInfo('activity', {
-      docCount,
-      orgCount,
-      orgInGoodStandingCount,
-      userCount,
-      userWithLoginCount,
-    });
+  public apply() {
+    if (!this._currentOperation) {
+      this._currentOperation = this._apply()
+        .finally(() => this._currentOperation = undefined);
+    }
+  }
+
+  private async _apply(): Promise<void> {
+    try {
+      const manager = this._dbManager.connection.manager;
+      // raw count of users
+      const userCount = await manager.count(User);
+      // users who have logged in at least once
+      const userWithLoginCount = await manager.createQueryBuilder()
+        .from(User, 'users')
+        .where('first_login_at is not null')
+        .getCount();
+      // raw count of organizations (excluding personal orgs)
+      const orgCount = await manager.createQueryBuilder()
+        .from(Organization, 'orgs')
+        .where('owner_id is null')
+        .getCount();
+      // organizations with subscriptions that are in a non-terminated state
+      const orgInGoodStandingCount = await manager.createQueryBuilder()
+        .from(Organization, 'orgs')
+        .leftJoin('orgs.billingAccount', 'billing_accounts')
+        .where('owner_id is null')
+        .andWhere('billing_accounts.in_good_standing = true')
+        .getCount();
+      // raw count of documents
+      const docCount = await manager.count(Document);
+      log.rawInfo('activity', {
+        docCount,
+        orgCount,
+        orgInGoodStandingCount,
+        userCount,
+        userWithLoginCount,
+      });
+    } catch (e) {
+      log.warn("Error in Usage._apply", e);
+    }
   }
 }

app/server/lib/ActiveDoc.ts

@@ -1015,6 +1015,7 @@ export class ActiveDoc extends EventEmitter {
           // Note: onDemand stored/undo actions are arbitrarily processed/added after normal actions
           // and do not support access control.
           sandboxActionBundle.stored.push(...stored.map(a => [allIndex, a] as [number, DocAction]));
+          sandboxActionBundle.direct.push(...stored.map(a => [allIndex, true] as [number, boolean]));
           sandboxActionBundle.undo.push(...undo.map(a => [allIndex, a] as [number, DocAction]));
           sandboxActionBundle.retValues.push(retValues);
         }
@@ -1078,8 +1079,8 @@ export class ActiveDoc extends EventEmitter {
    * granular access rules.
    */
   public getGranularAccessForBundle(docSession: OptDocSession, docActions: DocAction[], undo: DocAction[],
-                                    userActions: UserAction[]): GranularAccessForBundle {
-    this._granularAccess.getGranularAccessForBundle(docSession, docActions, undo, userActions);
+                                    userActions: UserAction[], isDirect: boolean[]): GranularAccessForBundle {
+    this._granularAccess.getGranularAccessForBundle(docSession, docActions, undo, userActions, isDirect);
     return this._granularAccess;
   }
 
@@ -1411,6 +1412,7 @@ function createEmptySandboxActionBundle(): SandboxActionBundle {
   return {
     envelopes: [],
     stored: [],
+    direct: [],
     calc: [],
     undo: [],
     retValues: []

app/server/lib/FlexServer.ts

@@ -531,6 +531,7 @@ export class FlexServer implements GristServer {
   }
 
   public async close() {
+    if (this.usage)  { await this.usage.close(); }
     if (this._hosts) { this._hosts.close(); }
     if (this.dbManager) {
       this.dbManager.removeAllListeners();
@@ -538,7 +539,6 @@ export class FlexServer implements GristServer {
     }
     if (this.server)      { this.server.close(); }
     if (this.httpsServer) { this.httpsServer.close(); }
-    if (this.usage)       { this.usage.close(); }
     if (this.housekeeper) { await this.housekeeper.stop(); }
     await this._shutdown();
     // Do this after _shutdown, since DocWorkerMap is used during shutdown.

app/server/lib/GranularAccess.ts

@@ -142,6 +142,7 @@ export class GranularAccess implements GranularAccessForBundle {
     docSession: OptDocSession,
     userActions: UserAction[],
     docActions: DocAction[],
+    isDirect: boolean[],
     undo: DocAction[],
     // Flag tracking whether a set of actions have been applied to the database or not.
     applied: boolean,
@@ -163,10 +164,10 @@ export class GranularAccess implements GranularAccessForBundle {
   }
 
   public getGranularAccessForBundle(docSession: OptDocSession, docActions: DocAction[], undo: DocAction[],
-                                    userActions: UserAction[]): void {
+                                    userActions: UserAction[], isDirect: boolean[]): void {
     if (this._activeBundle) { throw new Error('Cannot start a bundle while one is already in progress'); }
     this._activeBundle = {
-      docSession, docActions, undo, userActions,
+      docSession, docActions, undo, userActions, isDirect,
       applied: false, hasDeliberateRuleChange: false, hasAnyRuleChange: false
     };
     this._activeBundle.hasDeliberateRuleChange =
@@ -211,13 +212,17 @@ export class GranularAccess implements GranularAccessForBundle {
    */
   public async canApplyBundle() {
     if (!this._activeBundle) { throw new Error('no active bundle'); }
-    const {docActions, docSession} = this._activeBundle;
+    const {docActions, docSession, isDirect} = this._activeBundle;
     if (this._activeBundle.hasDeliberateRuleChange && !await this.isOwner(docSession)) {
       throw new ErrorWithCode('ACL_DENY', 'Only owners can modify access rules');
     }
     if (this._ruler.haveRules()) {
       await Promise.all(
-        docActions.map((action, actionIdx) => this._checkIncomingDocAction({docSession, action, actionIdx})));
+        docActions.map((action, actionIdx) => {
+          if (isDirect[actionIdx]) {
+            return this._checkIncomingDocAction({docSession, action, actionIdx});
+          }
+        }));
     }
 
     if (this._recoveryMode) {

app/server/lib/Sharing.ts

@@ -373,9 +373,10 @@ export class Sharing {
     const undo = getEnvContent(sandboxActionBundle.undo);
     const docActions = getEnvContent(sandboxActionBundle.stored).concat(
       getEnvContent(sandboxActionBundle.calc));
+    const isDirect = getEnvContent(sandboxActionBundle.direct);
 
     const accessControl = this._activeDoc.getGranularAccessForBundle(
-      docSession || makeExceptionalDocSession('share'), docActions, undo, userActions
+      docSession || makeExceptionalDocSession('share'), docActions, undo, userActions, isDirect
     );
     try {
       // TODO: see if any of the code paths that have no docSession are relevant outside