(core) move apiserver tests to core, disentangling notifier+billing parts

Summary:
This moves some more tests to core that would be useful for ANCT,
which had been stuck in grist-saas due to some entanglements with
sendgrid and billing. For sendgrid, I've moved around just enough
material to permit the tests to run mostly unchanged. Ideally
the interface to a notification system would be generalized, but
that's a bigger project.

Test Plan:
checked that tests are likely to run as expected
in core using preview laid out by ./buildtools/build_core.sh

Reviewers: georgegevoian

Reviewed By: georgegevoian

Differential Revision: https://phab.getgrist.com/D4149

app/gen-server/lib/NotifierTypes.ts

@@ -0,0 +1,173 @@
+/**
+ *
+ * Grist notifications are currently half-baked.
+ * There is a sendgrid based implementation for Grist Lab's SaaS, but
+ * nothing self-hostable yet.
+ *
+ */
+
+import {FullUser} from 'app/common/LoginSessionAPI';
+import {StringUnion} from 'app/common/StringUnion';
+
+/**
+ * Structure of sendgrid email requests.  Each request references a template
+ * (stored on sendgrid site) and a list of people to send a copy of that template
+ * to, along with the relevant values to use for template variables.
+ */
+export interface SendGridMail {
+  personalizations: SendGridPersonalization[];
+  from: SendGridAddress;
+  reply_to: SendGridAddress;
+  template_id: string;
+  asm?: {  // unsubscribe settings
+    group_id: number;
+  };
+  mail_settings?: {
+    bypass_list_management?: {
+      enable: boolean;
+    }
+  };
+}
+
+export interface SendGridContact {
+  contacts: [{
+    email: string;
+    first_name: string;
+    last_name: string;
+    custom_fields?: Record<string, any>;
+  }],
+  list_ids?: string[];
+}
+
+export interface SendGridAddress {
+  email: string;
+  name: string;
+}
+
+export interface SendGridPersonalization {
+  to: SendGridAddress[];
+  dynamic_template_data: {[key: string]: any};
+}
+
+/**
+ * Structure of sendgrid invite template.  This is entirely under our control, it
+ * is the information we choose to send to an email template for invites.
+ */
+
+export interface SendGridInviteTemplate {
+  user: FullUser;
+  host: FullUser;
+  resource: SendGridInviteResource;
+  access: SendGridInviteAccess;
+}
+
+export interface SendGridInviteResource {
+  kind: SendGridInviteResourceKind;
+  kindUpperFirst: string;
+  name: string;
+  url: string;
+}
+
+export type SendGridInviteResourceKind = 'team site' | 'workspace' | 'document';
+
+export interface SendGridInviteAccess {
+  role: string;
+  canEditAccess?: boolean;
+  canEdit?: boolean;
+  canView?: boolean;
+  canManageBilling?: boolean;
+}
+
+// Common parameters included in emails to active billing managers.
+export interface SendGridBillingTemplate {
+  org: {id: number, name: string};
+  orgUrl: string;
+  billingUrl: string;
+}
+
+export interface SendGridMemberChangeTemplate extends SendGridBillingTemplate {
+  initiatingUser: FullUser;
+  added: FullUser[];
+  removed: FullUser[];
+  org: {id: number, name: string};
+  countBefore: number;
+  countAfter: number;
+  orgUrl: string;
+  billingUrl: string;
+  paidPlan: boolean;
+}
+
+/**
+ * Format of sendgrid responses when looking up a user by email address using
+ * SENDGRID.search
+ */
+export interface SendGridSearchResult {
+  contact_count: number;
+  result: SendGridSearchHit[];
+}
+
+export interface SendGridSearchHit {
+  id: string;
+  email: string;
+  list_ids: string[];
+}
+
+/**
+ * Alternative format of sendgrid responses when looking up a user by email
+ * address using SENDGRID.searchByEmail
+ *   https://docs.sendgrid.com/api-reference/contacts/get-contacts-by-emails
+ */
+export interface SendGridSearchResultVariant {
+  result: Record<string, SendGridSearchPossibleHit>;
+}
+
+/**
+ * Documentation is contradictory on format of results when contacts not found, but if
+ * something is found there should be a contact field.
+ */
+export interface SendGridSearchPossibleHit {
+  contact?: SendGridSearchHit;
+}
+
+export interface SendGridConfig {
+  address: {
+    from: {
+      email: string;
+      name: string;
+    }
+  },
+  template: {
+    invite?: string;
+    billingManagerInvite?: string;
+    memberChange?: string;
+    trialPeriodEndingSoon?: string;
+    twoFactorMethodAdded?: string;
+    twoFactorMethodRemoved?: string;
+    twoFactorPhoneNumberChanged?: string;
+    twoFactorEnabled?: string;
+    twoFactorDisabled?: string;
+  },
+  list: {
+    singleUserOnboarding?: string;
+    appSumoSignUps?: string;
+    trial?: string;
+  },
+  unsubscribeGroup: {
+    invites?: number;
+    billingManagers?: number;
+  },
+  field?: {
+    callScheduled?: string;
+    userRef?: string;
+  },
+}
+
+export const TwoFactorEvents = StringUnion(
+  'twoFactorMethodAdded',
+  'twoFactorMethodRemoved',
+  'twoFactorPhoneNumberChanged',
+  'twoFactorEnabled',
+  'twoFactorDisabled',
+);
+
+export type TwoFactorEvent = typeof TwoFactorEvents.type;

app/server/lib/ICreate.ts

@@ -97,6 +97,7 @@ export function makeSimpleCreator(opts: {
       return notifier?.create(dbManager, gristConfig) ?? {
         get testPending() { return false; },
         async deleteUser()      { /* do nothing */ },
+        testSetSendMessageCallback() { return undefined; },
       };
     },
     ExternalStorage(purpose, extraPrefix) {

app/server/lib/INotifier.ts

@@ -1,6 +1,12 @@
+import {SendGridConfig, SendGridMail} from 'app/gen-server/lib/NotifierTypes';
+
 export interface INotifier {
   // for test purposes, check if any notifications are in progress
   readonly testPending: boolean;
 
   deleteUser(userId: number): Promise<void>;
+
+  // Intercept outgoing messages for test purposes.
+  // Return undefined if no notification system is available.
+  testSetSendMessageCallback(op: (body: SendGridMail, description: string) => Promise<void>): SendGridConfig|undefined;
 }

package.json

@@ -17,7 +17,7 @@
     "test:nbrowser": "TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'",
     "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
     "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
-    "test:server": "TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
+    "test:server": "TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} -g \"${GREP_TESTS}\" -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
     "test:smoke": "mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
     "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",

test/gen-server/ApiServerBenchmark.ts

@@ -0,0 +1,69 @@
+import axios from 'axios';
+import {configForUser} from 'test/gen-server/testUtils';
+import * as testUtils from 'test/server/testUtils';
+
+import {assert} from 'chai';
+
+import {FlexServer} from 'app/server/lib/FlexServer';
+
+import {createBenchmarkServer, removeConnection, setUpDB} from 'test/gen-server/seed';
+
+let home: FlexServer;
+let homeUrl: string;
+
+const chimpy = configForUser('Chimpy');
+
+describe('ApiServerBenchmark', function() {
+
+  testUtils.setTmpLogLevel('error');
+
+  before(async function() {
+    if (!process.env.ENABLE_BENCHMARKS) {
+      this.skip();
+      return;
+    }
+    this.timeout(600000);
+    setUpDB(this);
+    home = await createBenchmarkServer(0);
+    homeUrl = home.getOwnUrl();
+  });
+
+  after(async function() {
+    if (home) {
+      await home.stopListening();
+      await removeConnection();
+    }
+  });
+
+  it('GET /orgs returns in a timely manner', async function() {
+    this.timeout(600000);
+    for (let i = 0; i < 10; i++) {
+      const resp = await axios.get(`${homeUrl}/api/orgs`, chimpy);
+      assert(resp.data.length === 100);
+    }
+  });
+
+  // Note the organization id which is being fetched.
+  it('GET /orgs/{oid} returns in a timely manner', async function() {
+    this.timeout(600000);
+    for (let i = 0; i < 100; i++) {
+      await axios.get(`${homeUrl}/api/orgs/1`, chimpy);
+    }
+  });
+
+  // Note the organization id which is being fetched.
+  it('GET /orgs/{oid}/workspaces returns in a timely manner', async function() {
+    this.timeout(600000);
+    for (let i = 0; i < 100; i++) {
+      await axios.get(`${homeUrl}/api/orgs/1/workspaces`, chimpy);
+    }
+  });
+
+  // Note the workspace ids which are being fetched.
+  it('GET /workspaces/{wid} returns in a timely manner', async function() {
+    this.timeout(600000);
+    for (let wid = 0; wid < 100; wid++) {
+      await axios.get(`${homeUrl}/api/workspaces/${wid}`, chimpy);
+    }
+  });
+});

test/gen-server/ApiServerBugs.ts

@@ -0,0 +1,174 @@
+import axios from 'axios';
+import * as chai from 'chai';
+
+import {configForUser} from 'test/gen-server/testUtils';
+import * as testUtils from 'test/server/testUtils';
+
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+
+import {TestServer} from 'test/gen-server/apiUtils';
+
+const assert = chai.assert;
+
+let server: TestServer;
+let dbManager: HomeDBManager;
+let homeUrl: string;
+
+const charon = configForUser('Charon');
+const chimpy = configForUser('Chimpy');
+const kiwi = configForUser('Kiwi');
+
+const chimpyEmail = 'chimpy@getgrist.com';
+const kiwiEmail = 'kiwi@getgrist.com';
+const charonEmail = 'charon@getgrist.com';
+
+// Tests specific complex scenarios that may have previously resulted in wrong behavior.
+describe('ApiServerBugs', function() {
+
+  testUtils.setTmpLogLevel('error');
+  let userRef: (email: string) => Promise<string>;
+
+  before(async function() {
+    server = new TestServer(this);
+    homeUrl = await server.start();
+    dbManager = server.dbManager;
+    userRef = (email) => server.dbManager.getUserByLogin(email).then((user) => user!.ref);
+  });
+
+  after(async function() {
+    await server.stop();
+  });
+
+  // Re-create a bug scenario in which users being in normal groups and guests groups at the
+  // same time resulted in them being dropped from groups arbitrarily on subsequent patches.
+  it('should properly handle users in multiple groups at once', async function() {
+    // Add Chimpy/Charon/Kiwi to 'Herring' doc and set inheritance to none. They
+    // will become guests in the 'Fish' org along with their owner/viewer roles.
+    const fishOrg = await dbManager.testGetId('Fish');
+    const herringDoc = await dbManager.testGetId('Herring');
+    const delta1 = {
+      maxInheritedRole: null,
+      users: {
+        [kiwiEmail]: 'editors',
+        [charonEmail]: 'viewers'
+      }
+    };
+    let resp = await axios.patch(`${homeUrl}/api/docs/${herringDoc}/access`, {
+      delta: delta1
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    // Ensure that the doc access is as expected.
+    resp = await axios.get(`${homeUrl}/api/docs/${herringDoc}/access`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data, {
+      maxInheritedRole: null,
+      users: [{
+        id: 1,
+        name: 'Chimpy',
+        email: chimpyEmail,
+        ref: await userRef(chimpyEmail),
+        picture: null,
+        parentAccess: "owners",
+        access: "owners",
+        isMember: true,
+      }, {
+        id: 2,
+        name: 'Kiwi',
+        email: kiwiEmail,
+        ref: await userRef(kiwiEmail),
+        picture: null,
+        parentAccess: "editors",
+        access: "editors",
+        isMember: true,
+      }, {
+        id: 3,
+        name: 'Charon',
+        email: charonEmail,
+        ref: await userRef(charonEmail),
+        picture: null,
+        parentAccess: "viewers",
+        access: "viewers",
+        isMember: true,
+      }]
+    });
+
+    // Remove Charon from the 'Fish' org and ensure that Chimpy and Kiwi still have
+    // owner/editor roles on 'Fish'. Charon should no longer have guest access to the org.
+    const delta2 = {
+      users: {
+        [charonEmail]: null
+      }
+    };
+    resp = await axios.patch(`${homeUrl}/api/orgs/${fishOrg}/access`, {
+      delta: delta2
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    resp = await axios.get(`${homeUrl}/api/orgs/${fishOrg}/access`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(resp.data, {
+      users: [{
+        id: 1,
+        name: 'Chimpy',
+        email: chimpyEmail,
+        ref: await userRef(chimpyEmail),
+        picture: null,
+        access: "owners",
+        isMember: true,
+      }, {
+        id: 2,
+        name: 'Kiwi',
+        email: kiwiEmail,
+        ref: await userRef(kiwiEmail),
+        picture: null,
+        access: "editors",
+        isMember: true,
+      }]
+    });
+
+    // Charon should no longer have access to the 'Herring' doc, now that user access
+    // is wiped entirely when removed from org.
+    resp = await axios.get(`${homeUrl}/api/docs/${herringDoc}`, charon);
+    assert.equal(resp.status, 403);
+    resp = await axios.get(`${homeUrl}/api/orgs/${fishOrg}`, charon);
+    assert.equal(resp.status, 403);
+
+    // Remove Kiwi as an editor from the 'Fish' org and ensure that Kiwi no longer has
+    // access to 'Fish' or 'Herring'
+    const delta3 = {
+      users: {
+        [kiwiEmail]: null
+      }
+    };
+    resp = await axios.patch(`${homeUrl}/api/orgs/${fishOrg}/access`, {
+      delta: delta3
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    resp = await axios.get(`${homeUrl}/api/docs/${herringDoc}`, kiwi);
+    assert.equal(resp.status, 403);
+    resp = await axios.get(`${homeUrl}/api/orgs/${fishOrg}`, kiwi);
+    assert.equal(resp.status, 403);
+
+    // Restore initial access.
+    const delta4 = {
+      maxInheritedRole: "owners",
+      users: {
+        [charonEmail]: null,
+        [kiwiEmail]: null
+      }
+    };
+    resp = await axios.patch(`${homeUrl}/api/docs/${herringDoc}/access`, {
+      delta: delta4
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    const delta5 = {
+      users: {
+        [kiwiEmail]: "editors",
+        [charonEmail]: "viewers"
+      }
+    };
+    resp = await axios.patch(`${homeUrl}/api/orgs/${fishOrg}/access`, {
+      delta: delta5
+    }, chimpy);
+    assert.equal(resp.status, 200);
+  });
+});

test/gen-server/ApiSession.ts

@@ -0,0 +1,182 @@
+import {UserProfile} from 'app/common/LoginSessionAPI';
+import {AccessOptionWithRole} from 'app/gen-server/entity/Organization';
+import axios from 'axios';
+import {AxiosRequestConfig} from 'axios';
+import {assert} from 'chai';
+import omit = require('lodash/omit');
+import {TestServer} from 'test/gen-server/apiUtils';
+import * as testUtils from 'test/server/testUtils';
+
+const nobody: AxiosRequestConfig = {
+  responseType: 'json',
+  validateStatus: (status: number) => true
+};
+
+describe('ApiSession', function() {
+
+  let server: TestServer;
+  let serverUrl: string;
+  testUtils.setTmpLogLevel('error');
+
+  const regular = 'chimpy@getgrist.com';
+
+  beforeEach(async function() {
+    this.timeout(5000);
+    server = new TestServer(this);
+    serverUrl = await server.start();
+  });
+
+  afterEach(async function() {
+    await server.stop();
+  });
+
+  it('GET /api/session/access/active returns user and org (with access)', async function() {
+    const cookie = await server.getCookieLogin('nasa', {email: regular, name: 'Chimpy'});
+
+    const resp = await axios.get(`${serverUrl}/o/nasa/api/session/access/active`, cookie);
+    assert.equal(resp.status, 200);
+    assert.sameMembers(['user', 'org'], Object.keys(resp.data));
+    assert.deepEqual(omit(resp.data.user, ['helpScoutSignature', 'ref']), {
+      id: await server.dbManager.testGetId("Chimpy"),
+      email: "chimpy@getgrist.com",
+      name: "Chimpy",
+      picture: null,
+    });
+    assert.deepEqual(omit(resp.data.org, ['billingAccount', 'createdAt', 'updatedAt']), {
+      id: await server.dbManager.testGetId("NASA"),
+      name: "NASA",
+      access: "owners",
+      domain: "nasa",
+      host: null,
+      owner: null
+    });
+  });
+
+  it('GET /api/session/access/active returns org with billing account information', async function() {
+    const cookie = await server.getCookieLogin('nasa', {email: regular, name: 'Chimpy'});
+
+    // Make Chimpy a billing account manager for NASA.
+    await server.addBillingManager('Chimpy', 'nasa');
+
+    const resp = await axios.get(`${serverUrl}/o/nasa/api/session/access/active`, cookie);
+    assert.equal(resp.status, 200);
+    assert.hasAllKeys(resp.data.org, ['id', 'name', 'access', 'domain', 'owner', 'billingAccount',
+                                      'createdAt', 'updatedAt', 'host']);
+    assert.deepEqual(resp.data.org.billingAccount,
+                     { id: 1, individual: false, inGoodStanding: true, status: null,
+                       externalId: null, externalOptions: null,
+                       isManager: true, paid: false,
+                       product: { id: 1, name: 'Free', features: {workspaces: true, vanityDomain: true} } });
+
+    // Check that internally we have access to stripe ids.
+    const userId = await server.dbManager.testGetId('Chimpy') as number;
+    const org2 = await server.dbManager.getOrg({userId}, 'nasa');
+    assert.hasAllKeys(org2.data!.billingAccount,
+                      ['id', 'individual', 'inGoodStanding', 'status', 'stripeCustomerId',
+                       'stripeSubscriptionId', 'stripePlanId', 'product', 'paid', 'isManager',
+                       'externalId', 'externalOptions']);
+  });
+
+  it('GET /api/session/access/active returns orgErr when org is forbidden', async function() {
+    const cookie = await server.getCookieLogin('nasa', {email: 'kiwi@getgrist.com', name: 'Kiwi'});
+
+    const resp = await axios.get(`${serverUrl}/o/nasa/api/session/access/active`, cookie);
+    assert.equal(resp.status, 200);
+    assert.sameMembers(['user', 'org', 'orgError'], Object.keys(resp.data));
+    assert.deepEqual(omit(resp.data.user, ['helpScoutSignature', 'ref']), {
+      id: await server.dbManager.testGetId("Kiwi"),
+      email: "kiwi@getgrist.com",
+      name: "Kiwi",
+      picture: null,
+    });
+    assert.equal(resp.data.org, null);
+    assert.deepEqual(resp.data.orgError, {
+      status: 403,
+      error: 'access denied'
+    });
+  });
+
+  it('GET /api/session/access/active returns orgErr when org is non-existent', async function() {
+    const cookie = await server.getCookieLogin('nasa', {email: 'kiwi@getgrist.com', name: 'Kiwi'});
+
+    const resp = await axios.get(`${serverUrl}/o/boing/api/session/access/active`, cookie);
+    assert.equal(resp.status, 200);
+    assert.sameMembers(['user', 'org', 'orgError'], Object.keys(resp.data));
+    assert.deepEqual(omit(resp.data.user, ['helpScoutSignature', 'ref']), {
+      id: await server.dbManager.testGetId("Kiwi"),
+      email: "kiwi@getgrist.com",
+      name: "Kiwi",
+      picture: null,
+    });
+    assert.equal(resp.data.org, null);
+    assert.deepEqual(resp.data.orgError, {
+      status: 404,
+      error: 'organization not found'
+    });
+  });
+
+  it('POST /api/session/access/active can change user', async function() {
+    // add two profiles
+    const cookie = await server.getCookieLogin('nasa', {email: 'charon@getgrist.com', name: 'Charon'});
+    await server.getCookieLogin('pr', {email: 'kiwi@getgrist.com', name: 'Kiwi'});
+
+    // pick kiwi profile for fish org
+    let resp = await axios.post(`${serverUrl}/o/fish/api/session/access/active`, {
+      email: 'kiwi@getgrist.com'
+    }, cookie);
+    assert.equal(resp.status, 200);
+
+    // check kiwi profile stuck
+    resp = await axios.get(`${serverUrl}/o/fish/api/session/access/active`, cookie);
+    assert.equal(resp.data.user.email, 'kiwi@getgrist.com');
+
+    // ... and that it didn't affect other org
+    resp = await axios.get(`${serverUrl}/o/nasa/api/session/access/active`, cookie);
+    assert.equal(resp.data.user.email, 'charon@getgrist.com');
+
+    // pick charon profile for fish org
+    resp = await axios.post(`${serverUrl}/o/fish/api/session/access/active`, {
+      email: 'charon@getgrist.com'
+    }, cookie);
+    assert.equal(resp.status, 200);
+
+    // check charon profile stuck
+    resp = await axios.get(`${serverUrl}/o/fish/api/session/access/active`, cookie);
+    assert.equal(resp.data.user.email, 'charon@getgrist.com');
+
+    // make sure bogus profile for fish org fails
+    resp = await axios.post(`${serverUrl}/o/fish/api/session/access/active`, {
+      email: 'nonexistent@getgrist.com'
+    }, cookie);
+    assert.equal(resp.status, 403);
+  });
+
+  it('GET /api/session/access/all returns users and orgs', async function() {
+    const cookie = await server.getCookieLogin('nasa', {email: 'charon@getgrist.com', name: 'Charon'});
+    await server.getCookieLogin('pr', {email: 'kiwi@getgrist.com', name: 'Kiwi'});
+    const resp = await axios.get(`${serverUrl}/o/pr/api/session/access/all`, cookie);
+    assert.equal(resp.status, 200);
+    assert.sameMembers(['users', 'orgs'], Object.keys(resp.data));
+    assert.sameMembers(resp.data.users.map((user: UserProfile) => user.name),
+      ['Charon', 'Kiwi']);
+    // In following list, 'Kiwiland' is the the merged personal org, and Chimpyland is not
+    // listed explicitly.
+    assert.sameMembers(resp.data.orgs.map((org: any) => org.name),
+      ['Abyss', 'Fish', 'Flightless', 'Kiwiland', 'NASA', 'Primately']);
+    const fish = resp.data.orgs.find((org: any) => org.name === 'Fish');
+    const accessOptions: AccessOptionWithRole[] = fish.accessOptions;
+    assert.lengthOf(accessOptions, 2);
+    assert.equal('editors', accessOptions.find(opt => opt.name === 'Kiwi')!.access);
+    assert.equal('viewers', accessOptions.find(opt => opt.name === 'Charon')!.access);
+  });
+
+  it('GET /api/session/access/all functions with anonymous access', async function() {
+    const resp = await axios.get(`${serverUrl}/o/pr/api/session/access/all`, nobody);
+    assert.equal(resp.status, 200);
+    // No orgs listed without access
+    assert.lengthOf(resp.data.orgs, 0);
+    // A single anonymous user
+    assert.lengthOf(resp.data.users, 1);
+    assert.equal(resp.data.users[0].anonymous, true);
+  });
+});

test/gen-server/AuthCaching.ts

@@ -0,0 +1,400 @@
+import {delay} from 'app/common/delay';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {FlexServer} from 'app/server/lib/FlexServer';
+import log from 'app/server/lib/log';
+import {main as mergedServerMain} from 'app/server/mergedServerMain';
+import axios from 'axios';
+import {assert} from 'chai';
+import * as fse from 'fs-extra';
+import {tmpdir} from 'os';
+import * as path from 'path';
+import * as sinon from 'sinon';
+import {TestSession} from 'test/gen-server/apiUtils';
+import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
+import {configForUser, getGristConfig} from 'test/gen-server/testUtils';
+import {openClient} from 'test/server/gristClient';
+import * as testUtils from 'test/server/testUtils';
+
+async function createTestDir(ident: string): Promise<string> {
+  // Create a testDir of the form grist_test_{USER}_{SERVER_NAME}, removing any previous one.
+  const username = process.env.USER || "nobody";
+  const testDir = path.join(tmpdir(), `grist_test_${username}_${ident}`);
+  await fse.remove(testDir);
+  return testDir;
+}
+
+const chimpy = configForUser('Chimpy');
+const kiwi = configForUser('Kiwi');
+const charon = configForUser('Charon');
+const chimpyEmail = 'chimpy@getgrist.com';
+const kiwiEmail = 'kiwi@getgrist.com';
+const charonEmail = 'charon@getgrist.com';
+
+
+describe('AuthCaching', function() {
+  this.timeout(10000);
+  testUtils.setTmpLogLevel('error');
+
+  let homeServer: FlexServer, docsServer: FlexServer;
+  let session: TestSession;
+  let homeUrl: string;
+  let helloDocId: string;
+
+  const sandbox = sinon.createSandbox();
+
+  before(async function() {
+    const testDir = process.env.TESTDIR || await createTestDir('authcaching');
+    const testDocDir = path.join(testDir, "data");
+    await fse.mkdirs(testDocDir);
+    log.warn(`Test logs and data are at: ${testDir}/`);
+    setUpDB();
+    await createInitialDb();
+    process.env.GRIST_DATA_DIR = testDocDir;
+    homeServer = await mergedServerMain(0, ['home'],
+      {logToConsole: false, externalStorage: false});
+    homeUrl = homeServer.getOwnUrl();
+    process.env.APP_HOME_URL = homeUrl;
+    docsServer = await mergedServerMain(0, ['docs'],
+      {logToConsole: false, externalStorage: false});
+
+    // Helpers for getting cookie-based logins.
+    session = new TestSession(homeServer);
+
+    // Copy a fixture doc to make it accessible with the given docId.
+    helloDocId = (await homeServer.getHomeDBManager().testGetId('Jupiter')) as string;
+    const srcPath = path.resolve(testUtils.fixturesRoot, 'docs', 'Hello.grist');
+    await fse.copy(srcPath, path.resolve(docsServer.docsRoot, `${helloDocId}.grist`),
+                   { dereference: true });
+
+    // Add Kiwi to 'viewers' for this doc.
+    const resp = await axios.patch(`${homeUrl}/api/docs/${helloDocId}/access`,
+      {delta: {users: {[kiwiEmail]: 'viewers'}}},
+      chimpy);
+    assert.equal(resp.status, 200);
+  });
+
+  after(async function() {
+    delete process.env.GRIST_DATA_DIR;
+    delete process.env.APP_HOME_URL;
+    sandbox.restore();
+    await testUtils.captureLog('warn', async () => {
+      await docsServer.close();
+      await homeServer.close();
+      await removeConnection();
+    });
+  });
+
+  afterEach(async function() {
+    sandbox.restore();
+  });
+
+  function getDocTracker(dbManager: HomeDBManager) {
+    const forced = sandbox.spy(dbManager, "getDoc");
+    const cached = sandbox.spy(dbManager, "getDocAuthCached");
+    const impl = sandbox.spy(dbManager, "getDocImpl");
+    function getCallCounts() {
+      return {
+        forced: forced.callCount,
+        misses: impl.callCount - forced.callCount,
+        hits: cached.callCount - (impl.callCount - forced.callCount),
+      };
+    }
+    function reset() {
+      forced.resetHistory();
+      cached.resetHistory();
+      impl.resetHistory();
+    }
+    function getAndReset() {
+      const res = getCallCounts();
+      reset();
+      return res;
+    }
+    return {getCallCounts, reset, getAndReset};
+  }
+
+  function flushCache() {
+    homeServer.getHomeDBManager().flushDocAuthCache();
+    docsServer.getHomeDBManager().flushDocAuthCache();
+  }
+
+  function getDocCallTracker() {
+    return {
+      home: getDocTracker(homeServer.getHomeDBManager()),
+      docs: getDocTracker(docsServer.getHomeDBManager()),
+    };
+  }
+
+  it('should not cache direct call for doc metadata', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+
+    const resp = await axios.get(`${homeUrl}/api/docs/${helloDocId}`, chimpy);
+    assert.equal(resp.data.name, 'Jupiter');
+
+    // This is a metadata-only call, so only home server is involved.
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 1, misses: 0, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+
+    const resp2 = await axios.get(`${homeUrl}/api/docs/${helloDocId}`, chimpy);
+    assert.deepEqual(resp2.data, resp.data);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 1, misses: 0, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+  });
+
+  it('should cache DocApi + DocApiForwarder calls', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+    const resp = await axios.get(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, chimpy);
+    assert.deepInclude(resp.data, {E: ["HELLO", "", "", ""]});
+
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 1, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    // Try an endpoint requiring editing permissions.
+    const resp2 = await axios.post(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, {A: ['Foo']}, chimpy);
+    assert.equal(resp2.status, 200);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    const resp3 = await axios.get(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, chimpy);
+    assert.deepInclude(resp3.data, {E: ["HELLO", "", "", "", "FOO"]});
+
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+  });
+
+  it('should cache DocAPI + DocApiForwarder no-access calls', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+
+    // Kiwi has view-only access. Check that it's checked, and is cached too.
+    let resp = await axios.post(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, {A: ['Bar']}, kiwi);
+    assert.equal(resp.status, 403);
+    assert.match(resp.data.error, /No write access/);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 1, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    // Second call is cached, but otherwise identical.
+    resp = await axios.post(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, {A: ['Bar']}, kiwi);
+    assert.equal(resp.status, 403);
+    assert.match(resp.data.error, /No write access/);
+    // The read/write distinction isn't checked by DocApiForwarder, so docsServer sees the request.
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // View access works.
+    resp = await axios.get(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, kiwi);
+    assert.deepInclude(resp.data, {E: ["HELLO", "", "", "", "FOO"]});
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // Charon has no access.
+    resp = await axios.get(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, charon);
+    assert.equal(resp.status, 403);
+    assert.match(resp.data.error, /No view access/);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 1, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+
+    // ...or write access (but the check is cached).
+    resp = await axios.post(`${homeUrl}/api/docs/${helloDocId}/tables/Table1/data`, {A: ['Bar']}, charon);
+    assert.equal(resp.status, 403);
+    assert.match(resp.data.error, /No view access/);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    // docsServer never sees the request.
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+  });
+
+  it('should not cache app.html endpoint', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+    const cookie = await session.getCookieLogin('nasa', {email: chimpyEmail, name: 'Chimpy'});
+
+    const resp1 = await axios.get(`${homeUrl}/o/nasa/doc/${helloDocId}`, cookie);
+
+    // gristConfig should include results of the getDoc call.
+    const gristConfig = getGristConfig(resp1.data);
+    assert.hasAnyKeys(gristConfig.getDoc, [helloDocId]);
+    assert.deepInclude(gristConfig.getDoc![helloDocId], {name: 'Jupiter', id: helloDocId});
+
+    // All authentication and getDoc() call are made by homeServer, docsServer not yet in play
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 1, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+
+    // No caching on subsequent call because we force a fresh fetch for this endpoint.
+    const resp2 = await axios.get(`${homeUrl}/o/nasa/doc/${helloDocId}`, cookie);
+    assert.deepEqual(getGristConfig(resp2.data).getDoc, gristConfig.getDoc);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 1, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+  });
+
+  it('should cache openDoc and websocket methods', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+
+    const cli = await openClient(docsServer, chimpyEmail, 'nasa');
+    assert.equal((await cli.readMessage()).type, 'clientConnect');
+    const openDoc = await cli.send("openDoc", helloDocId);
+    assert.equal(openDoc.error, undefined);
+    assert.match(JSON.stringify(openDoc.data), /Table1/);
+
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    // Read access
+    const table = await cli.send("fetchTable", 0, "Table1");
+    assert.includeMembers(table.data.tableData, ['TableData', 'Table1']);
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // Write access
+    const auaResult = await cli.send("applyUserActions", 0,
+      [["UpdateRecord", "Table1", 1, {A: "auth-caching1"}]]);
+    await delay(200); // give a little time for change broadcast.
+    assert.isNumber(auaResult.data.actionNum);
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 2});
+
+    await cli.close();
+  });
+
+  it('should cache openDoc and websocket methods with access failures', async function() {
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+
+    // Repeat with a view-only user (Kiwi)
+    let cli = await openClient(docsServer, kiwiEmail, 'nasa');
+    assert.equal((await cli.readMessage()).type, 'clientConnect');
+    let openDoc = await cli.send("openDoc", helloDocId);
+    assert.equal(openDoc.error, undefined);
+    assert.match(JSON.stringify(openDoc.data), /Table1/);
+
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    // Kiwi has read access
+    const table = await cli.send("fetchTable", 0, "Table1");
+    assert.includeMembers(table.data.tableData, ['TableData', 'Table1']);
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // Kiwi has NO write access.
+    const auaResult = await cli.send("applyUserActions", 0,
+      [["UpdateRecord", "Table1", 1, {A: "auth-caching2"}]]);
+    assert.deepEqual(auaResult.error, 'No write access');
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // Charon has no access at all
+    cli = await openClient(docsServer, charonEmail, 'nasa');
+    assert.equal((await cli.readMessage()).type, 'clientConnect');
+    openDoc = await cli.send("openDoc", helloDocId);
+    assert.equal(openDoc.error, 'No view access');
+
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+    await cli.send("openDoc", helloDocId);
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+
+    // Home server wasn't involved in this test case at all.
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 0});
+  });
+
+  it('should cache across different kinds of calls', async function() {
+    // Fetch the document endpoint and follow with openDoc. Caching should apply.
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+    const cookie = await session.getCookieLogin('nasa', {email: chimpyEmail, name: 'Chimpy'});
+
+    // app.html endpoint warms the cache for the home server.
+    const resp1 = await axios.get(`${homeUrl}/o/nasa/doc/${helloDocId}`, cookie);
+    const gristConfig = getGristConfig(resp1.data);
+    assert.hasAnyKeys(gristConfig.getDoc, [helloDocId]);
+    assert.deepInclude(gristConfig.getDoc![helloDocId], {name: 'Jupiter', id: helloDocId});
+
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 1, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+
+    // openDoc call warms the cache for the doc-worker.
+    const cli = await openClient(docsServer, chimpyEmail, 'nasa');
+    assert.equal((await cli.readMessage()).type, 'clientConnect');
+    const openDoc = await cli.send("openDoc", helloDocId);
+    assert.equal(openDoc.error, undefined);
+    assert.match(JSON.stringify(openDoc.data), /Table1/);
+
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    // the caching applies to API calls for the same doc/user/org combination.
+    const resp = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, chimpy);
+    assert.equal(resp.status, 200);
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 1});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 1});
+  });
+
+  it('should expire the cache after a timeout', async function() {
+    this.timeout(10000);
+
+    // Make an API call; change access; check that after a while, the change is noticed.
+    flushCache();
+    const getDocCalls = getDocCallTracker();
+
+    // Connect up websockets for Kiwi and Charon.
+    const kiwiCli = await openClient(docsServer, kiwiEmail, 'nasa');
+    assert.equal((await kiwiCli.readMessage()).type, 'clientConnect');
+    const charonCli = await openClient(docsServer, charonEmail, 'nasa');
+    assert.equal((await charonCli.readMessage()).type, 'clientConnect');
+
+    // Kiwi has access, Charon doesn't.
+    let resp1 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, kiwi);
+    let resp2 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, charon);
+    assert.equal(resp1.status, 200);
+    assert.equal(resp2.status, 403);
+
+    // home server sees both calls, but only forwards one to the doc-worker.
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 2, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 0});
+
+    assert.equal((await kiwiCli.send("openDoc", helloDocId)).error, undefined);
+    assert.equal((await charonCli.send("openDoc", helloDocId)).error, 'No view access');
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 1, hits: 1});
+
+    // Use Chimpy's access to change access for both.
+    const resp = await axios.patch(`${homeUrl}/o/nasa/api/docs/${helloDocId}/access`,
+      {delta: {users: {[kiwiEmail]: null, [charonEmail]: 'viewers'}}},
+      chimpy);
+    assert.equal(resp.status, 200);
+
+    // Home's UserAPI methods don't call to getDoc() to check doc-level access, so access checks
+    // for Chimpy's patch-access call do not affect our counts.
+    assert.deepEqual(getDocCalls.home.getAndReset(), {forced: 0, misses: 0, hits: 0});
+    assert.deepEqual(getDocCalls.docs.getAndReset(), {forced: 0, misses: 0, hits: 0});
+
+    // The change isn't visible immediately.
+    resp1 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, kiwi);
+    resp2 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, charon);
+    assert.equal(resp1.status, 200);
+    assert.equal(resp2.status, 403);
+
+    // But eventually it is. Should be within 5 seconds, we try up to 10.
+    let passed = false;
+    for (let i = 0; i < 50; i++) {
+      await delay(200);
+      try {
+        // Check if access changes are visible yet.
+        resp1 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, kiwi);
+        resp2 = await axios.get(`${homeUrl}/o/nasa/api/docs/${helloDocId}/tables/Table1/data`, charon);
+        assert.equal(resp1.status, 403);
+        assert.equal(resp2.status, 200);
+        assert.equal((await kiwiCli.send("openDoc", helloDocId)).error, 'No view access');
+        assert.equal((await charonCli.send("openDoc", helloDocId)).error, undefined);
+        passed = true;
+        break;
+      } catch (err) {
+        continue;
+      }
+    }
+    assert.isTrue(passed);
+
+    const homeCalls = getDocCalls.home.getAndReset();
+    const docsCalls = getDocCalls.docs.getAndReset();
+    // There are many cache hits, but one set of misses that discovers the access changes.
+    assert.deepInclude(homeCalls, {forced: 0, misses: 2});
+    assert.deepInclude(docsCalls, {forced: 0, misses: 2});
+    assert.isAbove(homeCalls.hits, 10);
+    assert.isAbove(docsCalls.hits, 10);
+  });
+});

test/gen-server/migrations.ts

@@ -0,0 +1,172 @@
+import {QueryRunner} from "typeorm";
+import * as roles from "app/common/roles";
+import {Organization} from 'app/gen-server/entity/Organization';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {Permissions} from 'app/gen-server/lib/Permissions';
+import {assert} from 'chai';
+import {addSeedData, createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
+
+import {Initial1536634251710 as Initial} from 'app/gen-server/migration/1536634251710-Initial';
+import {Login1539031763952 as Login} from 'app/gen-server/migration/1539031763952-Login';
+import {PinDocs1549313797109 as PinDocs} from 'app/gen-server/migration/1549313797109-PinDocs';
+import {UserPicture1549381727494 as UserPicture} from 'app/gen-server/migration/1549381727494-UserPicture';
+import {LoginDisplayEmail1551805156919 as DisplayEmail} from 'app/gen-server/migration/1551805156919-LoginDisplayEmail';
+import {LoginDisplayEmailNonNull1552416614755
+        as DisplayEmailNonNull} from 'app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull';
+import {Indexes1553016106336 as Indexes} from 'app/gen-server/migration/1553016106336-Indexes';
+import {Billing1556726945436 as Billing} from 'app/gen-server/migration/1556726945436-Billing';
+import {Aliases1561589211752 as Aliases} from 'app/gen-server/migration/1561589211752-Aliases';
+import {TeamMembers1568238234987 as TeamMembers} from 'app/gen-server/migration/1568238234987-TeamMembers';
+import {FirstLogin1569593726320 as FirstLogin} from 'app/gen-server/migration/1569593726320-FirstLogin';
+import {FirstTimeUser1569946508569 as FirstTimeUser} from 'app/gen-server/migration/1569946508569-FirstTimeUser';
+import {CustomerIndex1573569442552 as CustomerIndex} from 'app/gen-server/migration/1573569442552-CustomerIndex';
+import {ExtraIndexes1579559983067 as ExtraIndexes} from 'app/gen-server/migration/1579559983067-ExtraIndexes';
+import {OrgHost1591755411755 as OrgHost} from 'app/gen-server/migration/1591755411755-OrgHost';
+import {DocRemovedAt1592261300044 as DocRemovedAt} from 'app/gen-server/migration/1592261300044-DocRemovedAt';
+import {Prefs1596456522124 as Prefs} from 'app/gen-server/migration/1596456522124-Prefs';
+import {ExternalBilling1623871765992 as ExternalBilling} from 'app/gen-server/migration/1623871765992-ExternalBilling';
+import {DocOptions1626369037484 as DocOptions} from 'app/gen-server/migration/1626369037484-DocOptions';
+import {Secret1631286208009 as Secret} from 'app/gen-server/migration/1631286208009-Secret';
+import {UserOptions1644363380225 as UserOptions} from 'app/gen-server/migration/1644363380225-UserOptions';
+import {GracePeriodStart1647883793388
+        as GracePeriodStart} from 'app/gen-server/migration/1647883793388-GracePeriodStart';
+import {DocumentUsage1651469582887 as DocumentUsage} from 'app/gen-server/migration/1651469582887-DocumentUsage';
+import {Activations1652273656610 as Activations} from 'app/gen-server/migration/1652273656610-Activations';
+import {UserConnectId1652277549983 as UserConnectId} from 'app/gen-server/migration/1652277549983-UserConnectId';
+import {UserUUID1663851423064 as UserUUID} from 'app/gen-server/migration/1663851423064-UserUUID';
+import {UserRefUnique1664528376930 as UserUniqueRefUUID} from 'app/gen-server/migration/1664528376930-UserRefUnique';
+import {Forks1673051005072 as Forks} from 'app/gen-server/migration/1673051005072-Forks';
+import {ForkIndexes1678737195050 as ForkIndexes} from 'app/gen-server/migration/1678737195050-ForkIndexes';
+import {ActivationPrefs1682636695021 as ActivationPrefs} from 'app/gen-server/migration/1682636695021-ActivationPrefs';
+import {AssistantLimit1685343047786 as AssistantLimit} from 'app/gen-server/migration/1685343047786-AssistantLimit';
+
+const home: HomeDBManager = new HomeDBManager();
+
+const migrations = [Initial, Login, PinDocs, UserPicture, DisplayEmail, DisplayEmailNonNull,
+                    Indexes, Billing, Aliases, TeamMembers, FirstLogin, FirstTimeUser,
+                    CustomerIndex, ExtraIndexes, OrgHost, DocRemovedAt, Prefs,
+                    ExternalBilling, DocOptions, Secret, UserOptions, GracePeriodStart,
+                    DocumentUsage, Activations, UserConnectId, UserUUID, UserUniqueRefUUID,
+                    Forks, ForkIndexes, ActivationPrefs, AssistantLimit];
+
+// Assert that the "members" acl rule and group exist (or not).
+function assertMembersGroup(org: Organization, exists: boolean) {
+  const memberAcl = org.aclRules.find(_aclRule => _aclRule.group.name === roles.MEMBER);
+  if (!exists) {
+    assert.isUndefined(memberAcl);
+  } else {
+    assert.isDefined(memberAcl);
+    assert.equal(memberAcl!.permissions, Permissions.VIEW);
+    assert.isDefined(memberAcl!.group);
+    assert.equal(memberAcl!.group.name, roles.MEMBER);
+  }
+}
+
+describe('migrations', function() {
+
+  before(function() {
+    setUpDB(this);
+  });
+
+  beforeEach(async function() {
+    await home.connect();
+    await createInitialDb(home.connection, false);
+  });
+
+  afterEach(async function() {
+    await removeConnection();
+  });
+
+  // a test to exercise the rollback scripts a bit
+  it('can migrate, do full rollback, and migrate again', async function() {
+    this.timeout(60000);
+    const runner = home.connection.createQueryRunner();
+    for (const migration of migrations) {
+      await (new migration()).up(runner);
+    }
+    for (const migration of migrations.slice().reverse()) {
+      await (new migration()).down(runner);
+    }
+    for (const migration of migrations) {
+      await (new migration()).up(runner);
+    }
+    await addSeedData(home.connection);
+    // if we made it this far without an exception, then the rollback scripts must
+    // be doing something.
+  });
+
+  it('can correctly switch display_email column to non-null with data', async function() {
+    this.timeout(60000);
+    const sqlite = home.connection.driver.options.type === 'sqlite';
+    // sqlite migrations need foreign keys turned off temporarily
+    if (sqlite) { await home.connection.query("PRAGMA foreign_keys = OFF;"); }
+    const runner = home.connection.createQueryRunner();
+    for (const migration of migrations) {
+      await (new migration()).up(runner);
+    }
+    await addSeedData(home.connection);
+    // migrate back until just before display_email column added, so we have no
+    // display_emails
+    for (const migration of migrations.slice().reverse()) {
+      await (new migration()).down(runner);
+      if (migration.name === DisplayEmail.name) { break; }
+    }
+    // now check DisplayEmail and DisplayEmailNonNull succeed with data in the db.
+    await (new DisplayEmail()).up(runner);
+    await (new DisplayEmailNonNull()).up(runner);
+    if (sqlite) { await home.connection.query("PRAGMA foreign_keys = ON;"); }
+  });
+
+  // a test to ensure the TeamMember migration works on databases with existing content
+  it('can perform TeamMember migration with seed data set', async function() {
+    this.timeout(30000);
+    const runner = home.connection.createQueryRunner();
+    // Perform full up migration and add the seed data.
+    for (const migration of migrations) {
+      await (new migration()).up(runner);
+    }
+    await addSeedData(home.connection);
+    const initAclCount = await getAclRowCount(runner);
+    const initGroupCount = await getGroupRowCount(runner);
+
+    // Assert that members groups are present to start.
+    for (const org of (await getAllOrgs(runner))) { assertMembersGroup(org, true); }
+
+    // Perform down TeamMembers migration with seed data and assert members groups are removed.
+    await (new TeamMembers()).down(runner);
+    const downMigratedOrgs = await getAllOrgs(runner);
+    for (const org of downMigratedOrgs) { assertMembersGroup(org, false); }
+    // Assert that the correct number of ACLs and groups were removed.
+    assert.equal(await getAclRowCount(runner), initAclCount - downMigratedOrgs.length);
+    assert.equal(await getGroupRowCount(runner), initGroupCount - downMigratedOrgs.length);
+
+    // Perform up TeamMembers migration with seed data and assert members groups are added.
+    await (new TeamMembers()).up(runner);
+    for (const org of (await getAllOrgs(runner))) { assertMembersGroup(org, true); }
+    // Assert that the correct number of ACLs and groups were re-added.
+    assert.equal(await getAclRowCount(runner), initAclCount);
+    assert.equal(await getGroupRowCount(runner), initGroupCount);
+  });
+});
+
+/**
+ * Returns all orgs in the database with aclRules and groups joined.
+ */
+function getAllOrgs(queryRunner: QueryRunner): Promise<Organization[]> {
+  const orgQuery = queryRunner.manager.createQueryBuilder()
+    .select('orgs')
+    .from(Organization, 'orgs')
+    .leftJoinAndSelect('orgs.aclRules', 'org_acl_rules')
+    .leftJoinAndSelect('org_acl_rules.group', 'org_groups');
+  return orgQuery.getMany();
+}
+
+async function getAclRowCount(queryRunner: QueryRunner): Promise<number> {
+  const rows = await queryRunner.query(`SELECT id FROM acl_rules`);
+  return rows.length;
+}
+
+async function getGroupRowCount(queryRunner: QueryRunner): Promise<number> {
+  const rows = await queryRunner.query(`SELECT id FROM groups`);
+  return rows.length;
+}