(core) updates from grist-core

test/gen-server/ApiServer.ts

@@ -49,9 +49,9 @@ describe('ApiServer', function() {
     homeUrl = await server.start(['home', 'docs']);
     dbManager = server.dbManager;
 
-    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user!.ref);
-    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user!.ref);
-    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user!.ref);
+    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user.ref);
+    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user.ref);
+    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user.ref);
 
     // Listen to user count updates and add them to an array.
     dbManager.on('userChange', ({org, countBefore, countAfter}: UserChange) => {
@@ -2070,8 +2070,7 @@ describe('ApiServer', function() {
     // create a new user
     const profile = {email: 'meep@getgrist.com', name: 'Meep'};
     const user = await dbManager.getUserByLogin('meep@getgrist.com', {profile});
-    assert(user);
-    const userId = user!.id;
+    const userId = user.id;
     // set up an api key
     await dbManager.connection.query("update users set api_key = 'api_key_for_meep' where id = $1", [userId]);
 
@@ -2111,11 +2110,10 @@ describe('ApiServer', function() {
     const userBlank = await dbManager.getUserByLogin('blank@getgrist.com',
                                                      {profile: {email: 'blank@getgrist.com',
                                                       name: ''}});
-    assert(userBlank);
-    await dbManager.connection.query("update users set api_key = 'api_key_for_blank' where id = $1", [userBlank!.id]);
+    await dbManager.connection.query("update users set api_key = 'api_key_for_blank' where id = $1", [userBlank.id]);
 
     // check that user can delete themselves
-    resp = await axios.delete(`${homeUrl}/api/users/${userBlank!.id}`,
+    resp = await axios.delete(`${homeUrl}/api/users/${userBlank.id}`,
                               {data: {name: ""}, ...configForUser("blank")});
     assert.equal(resp.status, 200);
 

test/gen-server/ApiServerAccess.ts

@@ -61,9 +61,9 @@ describe('ApiServerAccess', function() {
       }
     );
     dbManager = server.dbManager;
-    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user!.ref);
-    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user!.ref);
-    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user!.ref);
+    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user.ref);
+    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user.ref);
+    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user.ref);
     // Listen to user count updates and add them to an array.
     dbManager.on('userChange', ({org, countBefore, countAfter}: UserChange) => {
       if (countBefore === countAfter) { return; }

test/gen-server/ApiServerBugs.ts

@@ -32,7 +32,7 @@ describe('ApiServerBugs', function() {
     server = new TestServer(this);
     homeUrl = await server.start();
     dbManager = server.dbManager;
-    userRef = (email) => server.dbManager.getUserByLogin(email).then((user) => user!.ref);
+    userRef = (email) => server.dbManager.getUserByLogin(email).then((user) => user.ref);
   });
 
   after(async function() {

test/gen-server/lib/HomeDBManager.ts

@@ -33,14 +33,14 @@ describe('HomeDBManager', function() {
 
   it('can find existing user by email', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    assert.equal(user!.name, 'Chimpy');
+    assert.equal(user.name, 'Chimpy');
   });
 
   it('can create new user by email, with personal org', async function() {
     const profile = {email: 'unseen@getgrist.com', name: 'Unseen'};
     const user = await home.getUserByLogin('unseen@getgrist.com', {profile});
-    assert.equal(user!.name, 'Unseen');
-    const orgs = await home.getOrgs(user!.id, null);
+    assert.equal(user.name, 'Unseen');
+    const orgs = await home.getOrgs(user.id, null);
     assert.isAtLeast(orgs.data!.length, 1);
     assert.equal(orgs.data![0].name, 'Personal');
     assert.equal(orgs.data![0].owner.name, 'Unseen');
@@ -65,37 +65,37 @@ describe('HomeDBManager', function() {
     // log in without a name
     let user = await home.getUserByLogin('unseen2@getgrist.com');
     // name is blank
-    assert.equal(user!.name, '');
+    assert.equal(user.name, '');
     // log in with a name
     const profile: UserProfile = {email: 'unseen2@getgrist.com', name: 'Unseen2'};
     user = await home.getUserByLogin('unseen2@getgrist.com', {profile});
     // name is now set
-    assert.equal(user!.name, 'Unseen2');
+    assert.equal(user.name, 'Unseen2');
     // log in without a name
     user = await home.getUserByLogin('unseen2@getgrist.com');
     // name is still set
-    assert.equal(user!.name, 'Unseen2');
+    assert.equal(user.name, 'Unseen2');
     // no picture yet
-    assert.equal(user!.picture, null);
+    assert.equal(user.picture, null);
     // log in with picture link
     profile.picture = 'http://picture.pic';
     user = await home.getUserByLogin('unseen2@getgrist.com', {profile});
     // now should have a picture link
-    assert.equal(user!.picture, 'http://picture.pic');
+    assert.equal(user.picture, 'http://picture.pic');
     // log in without picture
     user = await home.getUserByLogin('unseen2@getgrist.com');
     // should still have picture link
-    assert.equal(user!.picture, 'http://picture.pic');
+    assert.equal(user.picture, 'http://picture.pic');
   });
 
   it('can add an org', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    const orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, teamOptions)).data!;
-    const org = await home.getOrg({userId: user!.id}, orgId);
+    const orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, teamOptions)).data!;
+    const org = await home.getOrg({userId: user.id}, orgId);
     assert.equal(org.data!.name, 'NewOrg');
     assert.equal(org.data!.domain, 'novel-org');
     assert.equal(org.data!.billingAccount.product.name, TEAM_PLAN);
-    await home.deleteOrg({userId: user!.id}, orgId);
+    await home.deleteOrg({userId: user.id}, orgId);
   });
 
   it('creates default plan if defined', async function() {
@@ -104,28 +104,28 @@ describe('HomeDBManager', function() {
     try {
       // Set the default product to be the free plan.
       process.env.GRIST_DEFAULT_PRODUCT = FREE_PLAN;
-      let orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, {
+      let orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, {
         setUserAsOwner: false,
         useNewPlan: true,
         // omit plan, to use a default one (teamInitial)
         // it will either be 'stub' or anything set in GRIST_DEFAULT_PRODUCT
       })).data!;
-      let org = await home.getOrg({userId: user!.id}, orgId);
+      let org = await home.getOrg({userId: user.id}, orgId);
       assert.equal(org.data!.name, 'NewOrg');
       assert.equal(org.data!.domain, 'novel-org');
       assert.equal(org.data!.billingAccount.product.name, FREE_PLAN);
-      await home.deleteOrg({userId: user!.id}, orgId);
+      await home.deleteOrg({userId: user.id}, orgId);
 
       // Now remove the default product, and check that the default plan is used.
       delete process.env.GRIST_DEFAULT_PRODUCT;
-      orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, {
+      orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, {
         setUserAsOwner: false,
         useNewPlan: true,
       })).data!;
 
-      org = await home.getOrg({userId: user!.id}, orgId);
+      org = await home.getOrg({userId: user.id}, orgId);
       assert.equal(org.data!.billingAccount.product.name, STUB_PLAN);
-      await home.deleteOrg({userId: user!.id}, orgId);
+      await home.deleteOrg({userId: user.id}, orgId);
     } finally {
       oldEnv.restore();
     }
@@ -134,17 +134,17 @@ describe('HomeDBManager', function() {
   it('cannot duplicate a domain', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
     const domain = 'repeated-domain';
-    const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+    const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
     const orgId = result.data!;
     assert.equal(result.status, 200);
-    await assert.isRejected(home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions),
+    await assert.isRejected(home.addOrg(user, {name: `${domain}!`, domain}, teamOptions),
                             /Domain already in use/);
-    await home.deleteOrg({userId: user!.id}, orgId);
+    await home.deleteOrg({userId: user.id}, orgId);
   });
 
   it('cannot add an org with a (blacklisted) dodgy domain', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    const userId = user!.id;
+    const userId = user.id;
     const misses = [
       'thing!', ' thing', 'ww', 'docs-999', 'o-99', '_domainkey', 'www', 'api',
       'thissubdomainiswaytoolongmyfriendyoushouldrethinkitoratleastsummarizeit',
@@ -154,13 +154,13 @@ describe('HomeDBManager', function() {
       'thing', 'jpl', 'xyz', 'appel', '123', '1google'
     ];
     for (const domain of misses) {
-      const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+      const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
       assert.equal(result.status, 400);
       const org = await home.getOrg({userId}, domain);
       assert.equal(org.status, 404);
     }
     for (const domain of hits) {
-      const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+      const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
       assert.equal(result.status, 200);
       const org = await home.getOrg({userId}, domain);
       assert.equal(org.status, 200);
@@ -189,7 +189,7 @@ describe('HomeDBManager', function() {
 
     // Fetch the doc and check that the updatedAt value is as expected.
     const kiwi = await home.getUserByLogin('kiwi@getgrist.com');
-    const resp1 = await home.getOrgWorkspaces({userId: kiwi!.id}, primatelyOrgId);
+    const resp1 = await home.getOrgWorkspaces({userId: kiwi.id}, primatelyOrgId);
     assert.equal(resp1.status, 200);
 
     // Check that the apples metadata is as expected. updatedAt should have been set
@@ -209,7 +209,7 @@ describe('HomeDBManager', function() {
 
     // Check that the shark metadata is as expected. updatedAt should have been set
     // to 2004. usage should be set.
-    const resp2 = await home.getOrgWorkspaces({userId: kiwi!.id}, fishOrgId);
+    const resp2 = await home.getOrgWorkspaces({userId: kiwi.id}, fishOrgId);
     assert.equal(resp2.status, 200);
     const shark = resp2.data![0].docs.find((doc: any) => doc.name === 'Shark');
     assert.equal(shark!.updatedAt.toISOString(), setDateISO2);
@@ -340,7 +340,7 @@ describe('HomeDBManager', function() {
 
   it('can fork docs', async function() {
     const user1 = await home.getUserByLogin('kiwi@getgrist.com');
-    const user1Id = user1!.id;
+    const user1Id = user1.id;
     const orgId = await home.testGetId('Fish') as number;
     const doc1Id = await home.testGetId('Shark') as string;
     const scope = {userId: user1Id, urlId: doc1Id};
@@ -393,7 +393,7 @@ describe('HomeDBManager', function() {
 
     // Now fork "Shark" as Chimpy, and check that Kiwi's forks aren't listed.
     const user2 = await home.getUserByLogin('chimpy@getgrist.com');
-    const user2Id = user2!.id;
+    const user2Id = user2.id;
     const resp4 = await home.getOrgWorkspaces({userId: user2Id}, orgId);
     const resp4Doc = resp4.data![0].docs.find((d: any) => d.name === 'Shark');
     assert.deepEqual(resp4Doc!.forks, []);

test/gen-server/lib/emails.ts

@@ -19,7 +19,7 @@ describe('emails', function() {
   beforeEach(async function() {
     this.timeout(5000);
     server = new TestServer(this);
-    ref = (email: string) => server.dbManager.getUserByLogin(email).then((user) => user!.ref);
+    ref = (email: string) => server.dbManager.getUserByLogin(email).then((user) => user.ref);
     serverUrl = await server.start();
   });
 

test/gen-server/lib/removedAt.ts

@@ -258,7 +258,7 @@ describe('removedAt', function() {
           'test3@getgrist.com': 'editors',
         }
       });
-      const userRef = (email: string) => home.dbManager.getUserByLogin(email).then((user) => user!.ref);
+      const userRef = (email: string) => home.dbManager.getUserByLogin(email).then((user) => user.ref);
       const idTest1 = (await home.dbManager.getUserByLogin("test1@getgrist.com"))!.id;
       const idTest2 = (await home.dbManager.getUserByLogin("test2@getgrist.com"))!.id;
       const idTest3 = (await home.dbManager.getUserByLogin("test3@getgrist.com"))!.id;

test/gen-server/seed.ts

@@ -42,7 +42,9 @@ import {User} from "app/gen-server/entity/User";
 import {Workspace} from "app/gen-server/entity/Workspace";
 import {EXAMPLE_WORKSPACE_NAME} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {Permissions} from 'app/gen-server/lib/Permissions';
-import {getOrCreateConnection, runMigrations, undoLastMigration, updateDb} from 'app/server/lib/dbUtils';
+import {
+  getConnectionName, getOrCreateConnection, runMigrations, undoLastMigration, updateDb
+} from 'app/server/lib/dbUtils';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import * as fse from 'fs-extra';
 
@@ -527,16 +529,27 @@ class Seed {
 // When running mocha on several test files at once, we need to reset our database connection
 // if it exists.  This is a little ugly since it is stored globally.
 export async function removeConnection() {
-  if (getConnectionManager().connections.length > 0) {
-    if (getConnectionManager().connections.length > 1) {
+  const connections = getConnectionManager().connections;
+  if (connections.length > 0) {
+    if (connections.length > 1) {
       throw new Error("unexpected number of connections");
     }
-    await getConnectionManager().connections[0].close();
-    // There is still no official way to delete connections that I've found.
-    (getConnectionManager() as any).connectionMap = new Map();
+    await connections[0].destroy();
+    dereferenceConnection(getConnectionName());
   }
 }
 
+export function dereferenceConnection(name: string) {
+  // There seem to be no official way to delete connections.
+  // Also we should probably get rid of the use of connectionManager, which is deprecated
+  const connectionMgr = getConnectionManager();
+  const connectionMap = (connectionMgr as any).connectionMap as Map<string, Connection>;
+  if (!connectionMap.has(name)) {
+    throw new Error('connection with this name not found: ' + name);
+  }
+  connectionMap.delete(name);
+}
+
 export async function createInitialDb(connection?: Connection, migrateAndSeedData: boolean = true) {
   // In jenkins tests, we may want to reset the database to a clean
   // state.  If so, TEST_CLEAN_DATABASE will have been set.  How to

test/gen-server/testUtils.ts

@@ -46,7 +46,6 @@ export async function createUser(dbManager: HomeDBManager, name: string): Promis
   const username = name.toLowerCase();
   const email = `${username}@getgrist.com`;
   const user = await dbManager.getUserByLogin(email, {profile: {email, name}});
-  if (!user) { throw new Error('failed to create user'); }
   user.apiKey = `api_key_for_${username}`;
   await user.save();
   const userHome = (await dbManager.getOrg({userId: user.id}, null)).data;

test/nbrowser/homeUtil.ts

@@ -420,7 +420,7 @@ export class HomeUtil {
     if (this.server.isExternalServer()) { throw new Error('not supported'); }
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (user) { await dbManager.deleteUser({userId: user.id}, user.id, user.name); }
+    await dbManager.deleteUser({userId: user.id}, user.id, user.name);
   }
 
   // Set whether this is the user's first time logging in.  Requires access to the database.
@@ -428,10 +428,8 @@ export class HomeUtil {
     if (this.server.isExternalServer()) { throw new Error('not supported'); }
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (user) {
-      user.isFirstTimeUser = isFirstLogin;
-      await user.save();
-    }
+    user.isFirstTimeUser = isFirstLogin;
+    await user.save();
   }
 
   private async _initShowGristTour(email: string, showGristTour: boolean) {
@@ -463,7 +461,6 @@ export class HomeUtil {
 
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (!user) { return; }
 
     if (user.personalOrg) {
       const org = await dbManager.getOrg({userId: user.id}, user.personalOrg.id);

test/server/fixSiteProducts.ts

@@ -122,7 +122,7 @@ describe('fixSiteProducts', function() {
     assert.equal(getDefaultProductNames().teamInitial, 'stub');
 
     const db = server.dbManager;
-    const user = await db.getUserByLogin(email, {profile}) as any;
+    const user = await db.getUserByLogin(email, {profile});
     const orgId = db.unwrapQueryResult(await db.addOrg(user, {
       name: 'sanity-check-org',
       domain: 'sanity-check-org',

test/server/lib/GranularAccess.ts

@@ -3279,7 +3279,7 @@ describe('GranularAccess', function() {
     cliOwner.flush();
     let perm: PermissionDataWithExtraUsers = (await cliOwner.send("getUsersForViewAs", 0)).data;
     const getId = (name: string) => home.dbManager.testGetId(name) as Promise<number>;
-    const getRef = (email: string) => home.dbManager.getUserByLogin(email).then(user => user!.ref);
+    const getRef = (email: string) => home.dbManager.getUserByLogin(email).then(user => user.ref);
     assert.deepEqual(perm.users, [
       { id: await getId('Chimpy'), email: 'chimpy@getgrist.com', name: 'Chimpy',
         ref: await getRef('chimpy@getgrist.com'),

test/server/lib/OIDCConfig.ts

@@ -5,7 +5,7 @@ import {Sessions} from "app/server/lib/Sessions";
 import log from "app/server/lib/log";
 import {assert} from "chai";
 import Sinon from "sinon";
-import {Client, generators, errors as OIDCError} from "openid-client";
+import {Client, custom, generators, errors as OIDCError} from "openid-client";
 import express from "express";
 import _ from "lodash";
 import {RequestWithLogin} from "app/server/lib/Authorizer";
@@ -192,6 +192,55 @@ describe('OIDCConfig', () => {
         });
       });
     });
+
+    describe('GRIST_OIDC_SP_HTTP_TIMEOUT', function () {
+      [
+        {
+          itMsg: 'when omitted should not override openid-client default value',
+          expectedUserDefinedHttpOptions: {}
+        },
+        {
+          itMsg: 'should reject when the provided value is not a number',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '__NOT_A_NUMBER__',
+          },
+          expectedErrorMsg: /__NOT_A_NUMBER__ does not look like a number/,
+        },
+        {
+          itMsg: 'should override openid-client timeout accordingly to the provided value',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '10000',
+          },
+          shouldSetTimeout: true,
+          expectedUserDefinedHttpOptions: {
+            timeout: 10000
+          }
+        },
+        {
+          itMsg: 'should allow disabling the timeout by having its value set to 0',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '0',
+          },
+          expectedUserDefinedHttpOptions: {
+            timeout: 0
+          }
+        }
+      ].forEach(ctx => {
+        it(ctx.itMsg, async () => {
+          const setHttpOptionsDefaultsStub = sandbox.stub(custom, 'setHttpOptionsDefaults');
+          setEnvVars();
+          Object.assign(process.env, ctx.env);
+          const promise = OIDCConfigStubbed.buildWithStub();
+          if (ctx.expectedErrorMsg) {
+            await assert.isRejected(promise, ctx.expectedErrorMsg);
+          } else {
+            await assert.isFulfilled(promise, 'initOIDC should have been fulfilled');
+            assert.isTrue(setHttpOptionsDefaultsStub.calledOnce, 'Should have called custom.setHttpOptionsDefaults');
+            assert.deepEqual(setHttpOptionsDefaultsStub.firstCall.args[0], ctx.expectedUserDefinedHttpOptions);
+          }
+        });
+      });
+    });
   });
 
   describe('GRIST_OIDC_IDP_ENABLED_PROTECTIONS', () => {

test/server/lib/helpers/PrepareDatabase.ts

@@ -2,13 +2,13 @@ import path from "path";
 import * as testUtils from "test/server/testUtils";
 import {execFileSync} from "child_process";
 
-export async function prepareDatabase(tempDirectory: string) {
+export async function prepareDatabase(tempDirectory: string, filename: string = 'landing.db') {
   // Let's create a sqlite db that we can share with servers that run in other processes, hence
   // not an in-memory db. Running seed.ts directly might not take in account the most recent value
   // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
   // configuration (in-memory for instance). Spawning a process is one way to make sure that the
   // latest value prevail.
-  process.env.TYPEORM_DATABASE = path.join(tempDirectory, 'landing.db');
+  process.env.TYPEORM_DATABASE = path.join(tempDirectory, filename);
   const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
   execFileSync('node', [seed, 'init'], {
     env: process.env,

test/test_env.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+export GRIST_SESSION_COOKIE="grist_test_cookie"
+export LANGUAGE="en_US"
+export SE_BROWSER="chrome"
+export SE_BROWSER_VERSION="127"
+export SE_DRIVER="chrome-driver"
+export SE_DRIVER_VERSION="127.0.6533.119"
+export TEST_CLEAN_DATABASE="true"
+export TEST_SUPPORT_API_KEY="api_key_for_support"
+
+exec "$@"

test/test_under_docker.sh

@@ -14,6 +14,8 @@ trap 'cleanup' EXIT
 trap 'echo "Exiting on SIGINT"; exit 1' INT
 trap 'echo "Exiting on SIGTERM"; exit 1' TERM
 
+source $(dirname $0)/test_env.sh
+
 PORT=8585
 DOCKER_CONTAINER=grist-core-test
 DOCKER_PID=""
@@ -65,8 +67,6 @@ fi
 
 TEST_ADD_SAMPLES=1 TEST_ACCOUNT_PASSWORD=not-needed \
   HOME_URL=http://localhost:8585 \
-  GRIST_SESSION_COOKIE=grist_test_cookie \
   GRIST_TEST_LOGIN=1 \
   NODE_PATH=_build:_build/stubs \
-  LANGUAGE=en_US \
   $MOCHA _build/test/deployment/*.js --slow 6000 -g "${GREP_TESTS:-}" "$@"