(core) Persist forks in home db

Summary:
Adds information about forks to the home db. This will be used
later by the UI to list forks of documents.

Test Plan: Browser and server tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3772

test/gen-server/apiUtils.ts

@@ -0,0 +1,304 @@
+import {delay} from 'app/common/delay';
+import {Role} from 'app/common/roles';
+import {UserAPIImpl, UserProfile} from 'app/common/UserAPI';
+import {AclRule, AclRuleDoc, AclRuleOrg, AclRuleWs} from 'app/gen-server/entity/AclRule';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Document} from 'app/gen-server/entity/Document';
+import {Group} from 'app/gen-server/entity/Group';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Resource} from 'app/gen-server/entity/Resource';
+import {User} from 'app/gen-server/entity/User';
+import {Workspace} from 'app/gen-server/entity/Workspace';
+import {SessionUserObj} from 'app/server/lib/BrowserSession';
+import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import * as docUtils from 'app/server/lib/docUtils';
+import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
+import log from 'app/server/lib/log';
+import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
+import axios from 'axios';
+import FormData from 'form-data';
+import fetch from 'node-fetch';
+import * as path from 'path';
+import {createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
+import {setPlan} from 'test/gen-server/testUtils';
+import {fixturesRoot} from 'test/server/testUtils';
+
+export class TestServer {
+  public serverUrl: string;
+  public server: FlexServer;
+  public dbManager: HomeDBManager;
+  public defaultSession: TestSession;
+
+  constructor(context?: Mocha.Context) {
+    setUpDB(context);
+  }
+
+  public async start(servers: ServerType[] = ["home"],
+                     options: FlexServerOptions = {}): Promise<string> {
+    await createInitialDb();
+    this.server = await mergedServerMain(0, servers, {logToConsole: false,
+                                                      externalStorage: false,
+                                                      ...options});
+    this.serverUrl = this.server.getOwnUrl();
+    this.dbManager = this.server.getHomeDBManager();
+    this.defaultSession = new TestSession(this.server);
+    return this.serverUrl;
+  }
+
+  public async stop() {
+    await this.server.close();
+    // Wait a few seconds for any late notifications to finish up.
+    // TypeORM doesn't give us a very clean way to shut down the db connection,
+    // and node-sqlite3 has become fussier about this, and in regular tests
+    // we substitute sqlite for postgres.
+    for (let i = 0; i < 30; i++) {
+      if (!this.server.getNotifier().testPending) { break; }
+      await delay(100);
+    }
+    await removeConnection();
+  }
+
+  // Set up a profile for the given org, and return an axios configuration to
+  // access the api via cookies with that profile.  Leave profile null for anonymous
+  // access.
+  public async getCookieLogin(
+    org: string,
+    profile: UserProfile|null,
+    options: {clearCache?: boolean, sessionProps?: Partial<SessionUserObj>} = {}
+  ) {
+    return this.defaultSession.getCookieLogin(org, profile, options);
+  }
+
+  // add named user as billing manager to org (identified by domain)
+  public async addBillingManager(userName: string, orgDomain: string) {
+    const ents = this.dbManager.connection.createEntityManager();
+    const org = await ents.findOne(Organization, {
+      relations: ['billingAccount'],
+      where: {domain: orgDomain}
+    });
+    const user = await ents.findOne(User, {where: {name: userName}});
+    const manager = new BillingAccountManager();
+    manager.user = user!;
+    manager.billingAccount = org!.billingAccount;
+    await manager.save();
+  }
+
+  // change a user's personal org to a different product (by default, one that allows anything)
+  public async upgradePersonalOrg(userName: string, productName: string = 'Free') {
+    const user = await User.findOne({where: {name: userName}});
+    if (!user) { throw new Error(`Could not find user ${userName}`); }
+    const org = await Organization.findOne({
+      relations: ['billingAccount', 'owner'],
+      where: {owner: {id: user.id}}  // for some reason finding by name generates wrong SQL.
+    });
+    if (!org) { throw new Error(`Could not find personal org of ${userName}`); }
+    await setPlan(this.dbManager, org, productName);
+  }
+
+  // Get an api object for making requests for the named user with the named org.
+  // Careful: all api objects using cookie access will be in the same session.
+  public async createHomeApi(userName: string, orgDomain: string,
+                             useApiKey: boolean = false,
+                             checkAccess: boolean = true): Promise<UserAPIImpl> {
+    return this.defaultSession.createHomeApi(userName, orgDomain, useApiKey, checkAccess);
+  }
+
+  // Get a TestSession representing a distinct session for communicating with the server.
+  public newSession() {
+    return new TestSession(this.server);
+  }
+
+  /**
+   * Lists every resource a user is linked to via direct group
+   * membership.  The same resource can be listed multiple times if
+   * the user is in multiple of its groups (e.g. viewers and guests).
+   * A resource the user has access to will not be listed at all if
+   * access is granted indirectly (e.g. a doc the user is not linked
+   * to via direct group membership won't be listed even if user is in
+   * owners group of workspace containing that doc, and the doc
+   * inherits access from the workspace).
+   */
+  public async listUserMemberships(email: string): Promise<ResourceWithRole[]> {
+    const rules = await this.dbManager.connection.createQueryBuilder()
+      .select('acl_rules')
+      .from(AclRule, 'acl_rules')
+      .leftJoinAndSelect('acl_rules.group', 'groups')
+      .leftJoin('groups.memberUsers', 'users')
+      .leftJoin('users.logins', 'logins')
+      .where('logins.email = :email', {email})
+      .getMany();
+    return Promise.all(rules.map(this._getResourceName.bind(this)));
+  }
+
+  /**
+   * Lists every user with the specified role on the given org.  Only
+   * roles set by direct group membership are listed, nothing indirect
+   * is included.
+   */
+  public async listOrgMembership(domain: string, role: Role|null): Promise<User[]> {
+    return this._listMembers(role)
+      .leftJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id')
+      .andWhere('orgs.domain = :domain', {domain})
+      .getMany();
+  }
+
+  /**
+   * Lists every user with the specified role on the given workspace.  Only
+   * roles set by direct group membership are listed, nothing indirect
+   * is included.
+   */
+  public async listWorkspaceMembership(wsId: number, role: Role|null): Promise<User[]> {
+    return this._listMembers(role)
+      .leftJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id')
+      .andWhere('workspaces.id = :wsId', {wsId})
+      .getMany();
+  }
+
+  // check that the database structure looks sane.
+  public async sanityCheck() {
+    const badGroups = await this.getBadGroupLinks();
+    if (badGroups.length) {
+      throw new Error(`badGroups: ${JSON.stringify(badGroups)}`);
+    }
+  }
+
+  // Find instances of guests and members used in inheritance.
+  public async getBadGroupLinks(): Promise<Group[]> {
+    // guests and members should never be in other groups, or have groups.
+    return this.dbManager.connection.createQueryBuilder()
+      .select('groups')
+      .from(Group, 'groups')
+      .innerJoinAndSelect('groups.memberGroups', 'memberGroups')
+      .where(`memberGroups.name IN ('guests', 'members')`)
+      .orWhere(`groups.name IN ('guests', 'members')`)
+      .getMany();
+  }
+
+  /**
+   * Copy a fixture doc (e.g. "Hello.grist", no path needed) and make
+   * it accessible with the given docId (no ".grist" extension or path).
+   */
+  public async copyFixtureDoc(srcName: string, docId: string) {
+    const docsRoot = this.server.docsRoot;
+    const srcPath = path.resolve(fixturesRoot, 'docs', srcName);
+    await docUtils.copyFile(srcPath, path.resolve(docsRoot, `${docId}.grist`));
+  }
+
+  public getWorkStore() {
+    return getDocWorkerMap();
+  }
+
+  /**
+   * Looks up the resource related to an aclRule.
+   * TODO: rework AclRule to automate this kind of step.
+   */
+  private async _getResourceName(aclRule: AclRule): Promise<ResourceWithRole> {
+    const con = this.dbManager.connection.manager;
+    let res: Document|Workspace|Organization|null;
+    if (aclRule instanceof AclRuleDoc) {
+      res = await con.findOne(Document, {where: {id: aclRule.docId}});
+    } else if (aclRule instanceof AclRuleWs) {
+      res = await con.findOne(Workspace, {where: {id: aclRule.workspaceId}});
+    } else if (aclRule instanceof AclRuleOrg) {
+      res = await con.findOne(Organization, {where: {id: aclRule.orgId}});
+    } else {
+      throw new Error('unknown type');
+    }
+    if (!res) { throw new Error('could not find resource'); }
+    return {res, role: aclRule.group.name};
+  }
+
+  /**
+   * Lists users and the groups/aclRules they are members of.
+   * Filters for groups of the specified name.
+   */
+  private _listMembers(role: Role|null) {
+    let q = this.dbManager.connection.createQueryBuilder()
+      .select('users')
+      .from(User, 'users')
+      .leftJoin('users.groups', 'groups')
+      .leftJoin('groups.aclRule', 'acl_rules')
+      .leftJoinAndSelect('users.logins', 'logins');
+    if (role) {
+      q = q.andWhere('groups.name = :role', {role});
+    }
+    return q;
+  }
+}
+
+/**
+ * A distinct session.  Any api objects created with this that use cookies will share
+ * the same session as each other, and be in a distinct session to other TestSessions.
+ *
+ * Calling createHomeApi on the server object directly results in api objects that are
+ * all within the same session, which is not always desirable.  Api key access can be
+ * used to work around this, but that can also be awkward.
+ */
+export class TestSession {
+  public headers: {[key: string]: string};
+
+  constructor(public home: FlexServer) {
+    this.headers = {};
+  }
+
+  // Set up a profile for the given org, and return an axios configuration to
+  // access the api via cookies with that profile.  Leave profile null for anonymous
+  // access.
+  public async getCookieLogin(
+    org: string,
+    profile: UserProfile|null,
+    {clearCache, sessionProps}: {clearCache?: boolean, sessionProps?: Partial<SessionUserObj>} = {}
+  ) {
+    const resp = await axios.get(`${this.home.getOwnUrl()}/test/session`,
+                                 {validateStatus: (s => s < 400), headers: this.headers});
+    const cookie = this.headers.Cookie || resp.headers['set-cookie'][0];
+    const cid = decodeURIComponent(cookie.split('=')[1].split(';')[0]);
+    const sessionId = this.home.getSessions().getSessionIdFromCookie(cid);
+    const scopedSession = this.home.getSessions().getOrCreateSession(sessionId as string, org, '');
+    await scopedSession.updateUserProfile({} as any, profile);
+    if (sessionProps) { await scopedSession.updateUser({} as any, sessionProps); }
+    if (clearCache) { this.home.getSessions().clearCacheIfNeeded(); }
+    this.headers.Cookie = cookie;
+    return {
+      validateStatus: (status: number) => true,
+      headers: {
+        'Cookie': cookie,
+        'X-Requested-With': 'XMLHttpRequest',
+      }
+    };
+  }
+
+  // get an api object for making requests for the named user with the named org.
+  public async createHomeApi(userName: string, orgDomain: string,
+                             useApiKey: boolean = false,
+                             checkAccess: boolean = true): Promise<UserAPIImpl> {
+    const headers: {[key: string]: string} = {};
+    if (useApiKey) {
+      headers.Authorization = 'Bearer api_key_for_' + userName.toLowerCase();
+    } else {
+      const cookie = await this.getCookieLogin(orgDomain, {
+        email: `${userName.toLowerCase()}@getgrist.com`,
+        name: userName
+      });
+      headers.Cookie = cookie.headers.Cookie;
+    }
+    const api = new UserAPIImpl(`${this.home.getOwnUrl()}/o/${orgDomain}`, {
+      fetch: fetch as any,
+      headers,
+      newFormData: () => new FormData() as any,
+      logger: log,
+    });
+    // Make sure api is functioning, and create user if this is their first time to hit API.
+    if (checkAccess) { await api.getOrg('current'); }
+    return api;
+  }
+}
+
+/**
+ * A resource and the name of the group associated with it that the user is in.
+ */
+export interface ResourceWithRole {
+  res: Resource;
+  role: string;
+}

test/nbrowser/Fork.ts

@@ -26,6 +26,39 @@ describe("Fork", function() {
     doc = await team.tempDoc(cleanup, 'Hello.grist', {load: false});
   }
 
+  async function getForks(api: UserAPI) {
+    const wss = await api.getOrgWorkspaces('current');
+    const forks = wss
+      .find(ws => ws.name === team.workspace)
+      ?.docs
+      .find(d => d.id === doc.id)
+      ?.forks;
+    return forks ?? [];
+  }
+
+  async function testForksOfForks(isLoggedIn: boolean = true) {
+    const session = isLoggedIn ? await team.login() : await team.anon.login();
+    await session.loadDoc(`/doc/${doc.id}/m/fork`);
+    await gu.getCell({rowNum: 1, col: 0}).click();
+    await gu.enterCell('1');
+    await gu.waitForServer();
+    assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '1');
+    const forkUrl = await driver.getCurrentUrl();
+    assert.match(forkUrl, isLoggedIn ? /^[^~]*~[^~]+~\d+$/ : /^[^~]*~[^~]*$/);
+    await session.loadDoc((new URL(forkUrl)).pathname + '/m/fork');
+    await gu.getCell({rowNum: 1, col: 0}).click();
+    await gu.enterCell('2');
+    await gu.waitForServer();
+    await driver.findContentWait(
+      '.test-notifier-toast-wrapper',
+      /Cannot fork a document that's already a fork.*Report a problem/s,
+      2000
+    );
+    assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '1');
+    assert.equal(await driver.getCurrentUrl(), `${forkUrl}/m/fork`);
+    await gu.wipeToasts();
+  }
+
   // Run tests with both regular docId and a custom urlId in URL, to make sure
   // ids are kept straight during forking.
 
@@ -182,27 +215,7 @@ describe("Fork", function() {
         assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '2');
       });
 
-      it('allows an anonymous fork to be forked', async function() {
-        const anonSession = await team.anon.login();
-        await anonSession.loadDoc(`/doc/${doc.id}/m/fork`);
-        await gu.getCell({rowNum: 1, col: 0}).click();
-        await gu.enterCell('1');
-        await gu.waitForServer();
-        assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '1');
-        const fork1 = await driver.getCurrentUrl();
-        assert.match(fork1, /^[^~]*~[^~]*$/);  // just one ~
-        await anonSession.loadDoc((new URL(fork1)).pathname + '/m/fork');
-        await gu.getCell({rowNum: 1, col: 0}).click();
-        await gu.enterCell('2');
-        await gu.waitForServer();
-        assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '2');
-        const fork2 = await driver.getCurrentUrl();
-        assert.match(fork2, /^[^~]*~[^~]*$/);  // just one ~
-        await anonSession.loadDoc((new URL(fork1)).pathname);
-        assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '1');
-        await anonSession.loadDoc((new URL(fork2)).pathname);
-        assert.equal(await gu.getCell({rowNum: 1, col: 0}).getText(), '2');
-      });
+      it('does not allow an anonymous fork to be forked', () => testForksOfForks(false));
 
       it('shows the right page item after forking', async function() {
         const anonSession = await team.anon.login();
@@ -236,6 +249,14 @@ describe("Fork", function() {
           const forkUrl = await driver.getCurrentUrl();
           assert.match(forkUrl, /~/);
 
+          // Check that the fork was saved to the home db
+          const api = userSession.createHomeApi();
+          const forks = await getForks(api);
+          const forkId = forkUrl.match(/\/[\w-]+~(\w+)(?:~\w)?/)?.[1];
+          const fork = forks.find(f => f.id === forkId);
+          assert.isDefined(fork);
+          assert.equal(fork!.trunkId, doc.id);
+
           // Open the original url and make sure the change we made is not there
           await userSession.loadDoc(`/doc/${doc.id}`);
           assert.notEqual(await gu.getCell({rowNum: 1, col: 0}).value(), '123');
@@ -619,6 +640,8 @@ describe("Fork", function() {
           await api.updateDocPermissions(doc.id, {users: {[altSession.email]: null}});
         }
       });
+
+      it('does not allow a fork to be forked', testForksOfForks);
     });
   }
 });

test/server/lib/DocApi2.ts

@@ -0,0 +1,74 @@
+import {UserAPI} from 'app/common/UserAPI';
+import axios from 'axios';
+import {assert} from 'chai';
+import * as fse from 'fs-extra';
+import {TestServer} from 'test/gen-server/apiUtils';
+import {configForUser} from 'test/gen-server/testUtils';
+import {createTmpDir} from 'test/server/docTools';
+import {openClient} from 'test/server/gristClient';
+import * as testUtils from 'test/server/testUtils';
+
+const chimpy = configForUser('Chimpy');
+
+describe('DocApi2', function() {
+  this.timeout(40000);
+  let server: TestServer;
+  let serverUrl: string;
+  let owner: UserAPI;
+  let wsId: number;
+  testUtils.setTmpLogLevel('error');
+  const oldEnv = new testUtils.EnvironmentSnapshot();
+
+  before(async function() {
+    const tmpDir = await createTmpDir();
+    process.env.GRIST_DATA_DIR = tmpDir;
+    process.env.STRIPE_ENDPOINT_SECRET = 'TEST_WITHOUT_ENDPOINT_SECRET';
+    // Use the TEST_REDIS_URL as the global redis url, if supplied.
+    if (process.env.TEST_REDIS_URL && !process.env.REDIS_URL) {
+      process.env.REDIS_URL = process.env.TEST_REDIS_URL;
+    }
+
+    server = new TestServer(this);
+    serverUrl = await server.start(['home', 'docs']);
+    const api = await server.createHomeApi('chimpy', 'docs', true);
+    await api.newOrg({name: 'testy', domain: 'testy'});
+    owner = await server.createHomeApi('chimpy', 'testy', true);
+    wsId = await owner.newWorkspace({name: 'ws'}, 'current');
+  });
+
+  after(async function() {
+    const api = await server.createHomeApi('chimpy', 'docs');
+    await api.deleteOrg('testy');
+    await server.stop();
+    oldEnv.restore();
+  });
+
+  describe('DELETE /docs/{did}', async () => {
+    it('permanently deletes a document and all of its forks', async function() {
+      // Create a new document and fork it twice.
+      const docId = await owner.newDoc({name: 'doc'}, wsId);
+      const session = await owner.getSessionActive();
+      const client = await openClient(server.server, session.user.email, session.org?.domain || 'docs');
+      await client.openDocOnConnect(docId);
+      const forkDocResponse1 = await client.send('fork', 0);
+      const forkDocResponse2 = await client.send('fork', 0);
+
+      // Check that files were created for the trunk and forks.
+      const docPath = server.server.getStorageManager().getPath(docId);
+      const forkPath1 = server.server.getStorageManager().getPath(forkDocResponse1.data.docId);
+      const forkPath2 = server.server.getStorageManager().getPath(forkDocResponse2.data.docId);
+      assert.equal(await fse.pathExists(docPath), true);
+      assert.equal(await fse.pathExists(forkPath1), true);
+      assert.equal(await fse.pathExists(forkPath2), true);
+
+      // Delete the trunk via API.
+      const deleteDocResponse = await axios.delete(`${serverUrl}/api/docs/${docId}`, chimpy);
+      assert.equal(deleteDocResponse.status, 200);
+
+      // Check that files for the trunk and forks were deleted.
+      assert.equal(await fse.pathExists(docPath), false);
+      assert.equal(await fse.pathExists(forkPath1), false);
+      assert.equal(await fse.pathExists(forkPath2), false);
+    });
+  });
+});