(core) updates from grist-core

.github/workflows/docker_latest.yml

@@ -46,6 +46,7 @@ jobs:
   push_to_registry:
     name: Push latest Docker image to Docker Hub
     runs-on: ubuntu-latest
+    if: ${{ vars.RUN_DAILY_BUILD }}
     strategy:
       matrix:
         python-version: [3.11]
@@ -123,6 +124,9 @@ jobs:
         if: ${{ !inputs.disable_tests }}
         run: yarn run build:prod
 
+      - name: Install Google Chrome for Testing
+        run: ./test/test_env.sh node_modules/selenium-webdriver/bin/linux/selenium-manager
+
       - name: Run tests
         if: ${{ !inputs.disable_tests }}
         run: TEST_IMAGE=${{ env.DOCKER_HUB_OWNER }}/${{ matrix.image.name }}:${{ env.TAG }} VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker

.github/workflows/main.yml

@@ -69,9 +69,9 @@ jobs:
       - name: Build Node.js code
         run: yarn run build:prod
 
-      - name: Install chromedriver
+      - name: Install Google Chrome for Testing
         if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:') || contains(matrix.tests, ':stubs:')
-        run: ./node_modules/selenium-webdriver/bin/linux/selenium-manager --driver chromedriver
+        run: ./test/test_env.sh ./node_modules/selenium-webdriver/bin/linux/selenium-manager
 
       - name: Run smoke test
         if: contains(matrix.tests, ':smoke:')

app/client/ui/PageWidgetPicker.ts

@@ -1,5 +1,6 @@
 import {BehavioralPromptsManager} from 'app/client/components/BehavioralPromptsManager';
 import {GristDoc} from 'app/client/components/GristDoc';
+import {FocusLayer} from 'app/client/lib/FocusLayer';
 import {makeT} from 'app/client/lib/localization';
 import {reportError} from 'app/client/models/AppModel';
 import {ColumnRec, TableRec, ViewSectionRec} from 'app/client/models/DocModel';
@@ -260,8 +261,7 @@ export function buildPageWidgetPicker(
     dom.create(PageWidgetSelect,
       value, tables, columns, onSaveCB, behavioralPromptsManager, options),
 
-    // gives focus and binds keydown events
+    elem => { FocusLayer.create(ctl, {defaultFocusElem: elem, pauseMousetrap: true}); },
-    (elem: any) => { setTimeout(() => elem.focus(), 0); },
     onKeyDown({
       Escape: () => ctl.close(),
       Enter: () => isValid() && onSaveCB()

app/client/widgets/FormulaAssistant.ts

@@ -35,7 +35,7 @@ import {Marked} from 'marked';
 import {markedHighlight} from 'marked-highlight';
 import {v4 as uuidv4} from 'uuid';
 
-const t = makeT('FormulaEditor');
+const t = makeT('FormulaAssistant');
 const testId = makeTestId('test-formula-editor-');
 
 const LOW_CREDITS_WARNING_BANNER_THRESHOLD = 10;

app/gen-server/ApiServer.ts

@@ -429,7 +429,7 @@ export class ApiServer {
         throw new ApiError('Name expected in the body', 400);
       }
       const name = req.body.name;
-      await this._dbManager.updateUserName(userId, name);
+      await this._dbManager.updateUser(userId, { name });
       res.sendStatus(200);
     }));
 

app/gen-server/lib/homedb/HomeDBManager.ts

@@ -56,7 +56,7 @@ import {
   readJson
 } from 'app/gen-server/sqlUtils';
 import {appSettings} from 'app/server/lib/AppSettings';
-import {getOrCreateConnection} from 'app/server/lib/dbUtils';
+import {createNewConnection, getOrCreateConnection} from 'app/server/lib/dbUtils';
 import {makeId} from 'app/server/lib/idUtils';
 import log from 'app/server/lib/log';
 import {Permit} from 'app/server/lib/Permit';
@@ -70,6 +70,7 @@ import {
   Brackets,
   Connection,
   DatabaseType,
+  DataSourceOptions,
   EntityManager,
   ObjectLiteral,
   SelectQueryBuilder,
@@ -248,7 +249,6 @@ export type BillingOptions = Partial<Pick<BillingAccount,
 export class HomeDBManager extends EventEmitter {
   private _usersManager = new UsersManager(this, this._runInTransaction.bind(this));
   private _connection: Connection;
-  private _dbType: DatabaseType;
   private _exampleWorkspaceId: number;
   private _exampleOrgId: number;
   private _idPrefix: string = "";  // Place this before ids in subdomains, used in routing to
@@ -258,6 +258,10 @@ export class HomeDBManager extends EventEmitter {
   // In restricted mode, documents should be read-only.
   private _restrictedMode: boolean = false;
 
+  private get _dbType(): DatabaseType {
+    return this._connection.driver.options.type;
+  }
+
   /**
    * Five aclRules, each with one group (with the names 'owners', 'editors', 'viewers',
    * 'guests', and 'members') are created by default on every new entity (Organization,
@@ -348,7 +352,10 @@ export class HomeDBManager extends EventEmitter {
 
   public async connect(): Promise<void> {
     this._connection = await getOrCreateConnection();
-    this._dbType = this._connection.driver.options.type;
+  }
+
+  public async createNewConnection(overrideConf?: Partial<DataSourceOptions>): Promise<void> {
+    this._connection = await createNewConnection(overrideConf);
   }
 
   // make sure special users and workspaces are available
@@ -461,10 +468,6 @@ export class HomeDBManager extends EventEmitter {
     }
   }
 
-  public async updateUserName(userId: number, name: string) {
-    return this._usersManager.updateUserName(userId, name);
-  }
-
   public async updateUserOptions(userId: number, props: Partial<UserOptions>) {
     return this._usersManager.updateUserOptions(userId, props);
   }
@@ -472,14 +475,14 @@ export class HomeDBManager extends EventEmitter {
   /**
    * @see UsersManager.prototype.getUserByLoginWithRetry
    */
-  public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
+  public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User> {
     return this._usersManager.getUserByLoginWithRetry(email, options);
   }
 
   /**
    * @see UsersManager.prototype.getUserByLogin
    */
-  public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
+  public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User> {
     return this._usersManager.getUserByLogin(email, options);
   }
 
@@ -4362,7 +4365,6 @@ export class HomeDBManager extends EventEmitter {
     });
     return verifyEntity(orgQuery);
   }
-
 }
 
 // Return a QueryResult reflecting the output of a query builder.

app/gen-server/lib/homedb/Interfaces.ts

@@ -34,7 +34,7 @@ export type NonGuestGroup = Group & { name: roles.NonGuestRole };
 
 export type Resource = Organization|Workspace|Document;
 
-export type RunInTransaction = (
+export type RunInTransaction = <T>(
   transaction: EntityManager|undefined,
-  op: ((manager: EntityManager) => Promise<any>)
+  op: ((manager: EntityManager) => Promise<T>)
-) => Promise<any>;
+) => Promise<T>;

app/gen-server/lib/homedb/UsersManager.ts

@@ -97,7 +97,7 @@ export class UsersManager {
   public async testClearUserPrefs(emails: string[]) {
     return await this._connection.transaction(async manager => {
       for (const email of emails) {
-        const user = await this.getUserByLogin(email, {manager});
+        const user = await this.getExistingUserByLogin(email, manager);
         if (user) {
           await manager.delete(Pref, {userId: user.id});
         }
@@ -116,7 +116,7 @@ export class UsersManager {
    */
   public getAnonymousUserId(): number {
     const id = this._specialUserIds[ANONYMOUS_USER_EMAIL];
-    if (!id) { throw new Error("Anonymous user not available"); }
+    if (!id) { throw new Error("'Anonymous' user not available"); }
     return id;
   }
 
@@ -125,7 +125,7 @@ export class UsersManager {
    */
   public getPreviewerUserId(): number {
     const id = this._specialUserIds[PREVIEWER_EMAIL];
-    if (!id) { throw new Error("Previewer user not available"); }
+    if (!id) { throw new Error("'Previewer' user not available"); }
     return id;
   }
 
@@ -134,7 +134,7 @@ export class UsersManager {
    */
   public getEveryoneUserId(): number {
     const id = this._specialUserIds[EVERYONE_EMAIL];
-    if (!id) { throw new Error("'everyone' user not available"); }
+    if (!id) { throw new Error("'Everyone' user not available"); }
     return id;
   }
 
@@ -143,7 +143,7 @@ export class UsersManager {
    */
   public getSupportUserId(): number {
     const id = this._specialUserIds[SUPPORT_EMAIL];
-    if (!id) { throw new Error("'support' user not available"); }
+    if (!id) { throw new Error("'Support' user not available"); }
     return id;
   }
 
@@ -221,9 +221,6 @@ export class UsersManager {
           profile,
           manager
         });
-        if (!newUser) {
-          throw new ApiError("Unable to create user", 500);
-        }
         // No need to survey this user.
         newUser.isFirstTimeUser = false;
         await newUser.save();
@@ -286,13 +283,7 @@ export class UsersManager {
     return { user, isWelcomed };
   }
 
-  public async updateUserName(userId: number, name: string) {
+  // TODO: rather use the updateUser() method, if that makes sense?
-    const user = await User.findOne({where: {id: userId}});
-    if (!user) { throw new ApiError("unable to find user", 400); }
-    user.name = name;
-    await user.save();
-  }
-
   public async updateUserOptions(userId: number, props: Partial<UserOptions>) {
     const user = await User.findOne({where: {id: userId}});
     if (!user) { throw new ApiError("unable to find user", 400); }
@@ -321,7 +312,7 @@ export class UsersManager {
   // for an email key conflict failure. This is in case our transaction conflicts with a peer
   // doing the same thing. This is quite likely if the first page visited by a previously
   // unseen user fires off multiple api calls.
-  public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
+  public async getUserByLoginWithRetry(email: string, options: GetUserOptions = {}): Promise<User> {
     try {
       return await this.getUserByLogin(email, options);
     } catch (e) {
@@ -361,10 +352,10 @@ export class UsersManager {
    * unset/outdated fields of an existing record.
    *
    */
-  public async getUserByLogin(email: string, options: GetUserOptions = {}): Promise<User|undefined> {
+  public async getUserByLogin(email: string, options: GetUserOptions = {}) {
     const {manager: transaction, profile, userOptions} = options;
     const normalizedEmail = normalizeEmail(email);
-    const userByLogin = await this._runInTransaction(transaction, async manager => {
+    return await this._runInTransaction(transaction, async manager => {
       let needUpdate = false;
       const userQuery = manager.createQueryBuilder()
         .select('user')
@@ -473,9 +464,8 @@ export class UsersManager {
         // In principle this could be optimized, but this is simpler to maintain.
         user = await userQuery.getOne();
       }
-      return user;
+      return user!;
     });
-    return userByLogin;
   }
 
   /**
@@ -520,6 +510,63 @@ export class UsersManager {
     };
   }
 
+  public async initializeSpecialIds(): Promise<void> {
+    await this._maybeCreateSpecialUserId({
+      email: ANONYMOUS_USER_EMAIL,
+      name: "Anonymous"
+    });
+    await this._maybeCreateSpecialUserId({
+      email: PREVIEWER_EMAIL,
+      name: "Preview"
+    });
+    await this._maybeCreateSpecialUserId({
+      email: EVERYONE_EMAIL,
+      name: "Everyone"
+    });
+    await this._maybeCreateSpecialUserId({
+      email: SUPPORT_EMAIL,
+      name: "Support"
+    });
+  }
+
+  /**
+   *
+   * Take a list of user profiles coming from the client's session, correlate
+   * them with Users and Logins in the database, and construct full profiles
+   * with user ids, standardized display emails, pictures, and anonymous flags.
+   *
+   */
+  public async completeProfiles(profiles: UserProfile[]): Promise<FullUser[]> {
+    if (profiles.length === 0) { return []; }
+    const qb = this._connection.createQueryBuilder()
+      .select('logins')
+      .from(Login, 'logins')
+      .leftJoinAndSelect('logins.user', 'user')
+      .where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))});
+    const completedProfiles: {[email: string]: FullUser} = {};
+    for (const login of await qb.getMany()) {
+      completedProfiles[login.email] = {
+        id: login.user.id,
+        email: login.displayEmail,
+        name: login.user.name,
+        picture: login.user.picture,
+        anonymous: login.user.id === this.getAnonymousUserId(),
+        locale: login.user.options?.locale
+      };
+    }
+    return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)])
+      .filter(fullProfile => fullProfile);
+  }
+
+  /**
+   * ==================================
+   *
+   * Below methods are public but not exposed by HomeDBManager
+   *
+   * They are meant to be used internally (i.e. by homedb/ modules)
+   *
+   */
+
   // Looks up the emails in the permission delta and adds them to the users map in
   // the delta object.
   // Returns a QueryResult based on the validity of the passed in PermissionDelta object.
@@ -589,25 +636,6 @@ export class UsersManager {
     };
   }
 
-  public async initializeSpecialIds(): Promise<void> {
-    await this._maybeCreateSpecialUserId({
-      email: ANONYMOUS_USER_EMAIL,
-      name: "Anonymous"
-    });
-    await this._maybeCreateSpecialUserId({
-      email: PREVIEWER_EMAIL,
-      name: "Preview"
-    });
-    await this._maybeCreateSpecialUserId({
-      email: EVERYONE_EMAIL,
-      name: "Everyone"
-    });
-    await this._maybeCreateSpecialUserId({
-      email: SUPPORT_EMAIL,
-      name: "Support"
-    });
-  }
-
   /**
    * Check for anonymous user, either encoded directly as an id, or as a singular
    * profile (this case arises during processing of the session/access/all endpoint
@@ -684,34 +712,6 @@ export class UsersManager {
     return members;
   }
 
-  /**
-   *
-   * Take a list of user profiles coming from the client's session, correlate
-   * them with Users and Logins in the database, and construct full profiles
-   * with user ids, standardized display emails, pictures, and anonymous flags.
-   *
-   */
-  public async completeProfiles(profiles: UserProfile[]): Promise<FullUser[]> {
-    if (profiles.length === 0) { return []; }
-    const qb = this._connection.createQueryBuilder()
-      .select('logins')
-      .from(Login, 'logins')
-      .leftJoinAndSelect('logins.user', 'user')
-      .where('logins.email in (:...emails)', {emails: profiles.map(profile => normalizeEmail(profile.email))});
-    const completedProfiles: {[email: string]: FullUser} = {};
-    for (const login of await qb.getMany()) {
-      completedProfiles[login.email] = {
-        id: login.user.id,
-        email: login.displayEmail,
-        name: login.user.name,
-        picture: login.user.picture,
-        anonymous: login.user.id === this.getAnonymousUserId(),
-        locale: login.user.options?.locale
-      };
-    }
-    return profiles.map(profile => completedProfiles[normalizeEmail(profile.email)])
-      .filter(profile => profile);
-  }
 
   // For the moment only the support user can add both everyone@ and anon@ to a
   // resource, since that allows spam. TODO: enhance or remove.
@@ -735,7 +735,7 @@ export class UsersManager {
       // user if a bunch of servers start simultaneously and the user doesn't exist
       // yet.
       const user = await this.getUserByLoginWithRetry(profile.email, {profile});
-      if (user) { id = this._specialUserIds[profile.email] = user.id; }
+      id = this._specialUserIds[profile.email] = user.id;
     }
     if (!id) { throw new Error(`Could not find or create user ${profile.email}`); }
     return id;

app/server/companion.ts

@@ -166,10 +166,6 @@ export function addSiteCommand(program: commander.Command,
       const profile = {email, name: email};
       const db = await getHomeDBManager();
       const user = await db.getUserByLogin(email, {profile});
-      if (!user) {
-        // This should not happen.
-        throw new Error('failed to create user');
-      }
       db.unwrapQueryResult(await db.addOrg(user, {
         name: domain,
         domain,

app/server/lib/DocApi.ts

@@ -972,11 +972,12 @@ export class DocWorkerApi {
 
     // Reload a document forcibly (in fact this closes the doc, it will be automatically
     // reopened on use).
-    this._app.post('/api/docs/:docId/force-reload', canEdit, throttled(async (req, res) => {
+    this._app.post('/api/docs/:docId/force-reload', canEdit, async (req, res) => {
-      const activeDoc = await this._getActiveDoc(req);
+      const mreq = req as RequestWithLogin;
+      const activeDoc = await this._getActiveDoc(mreq);
       await activeDoc.reloadDoc();
       res.json(null);
-    }));
+    });
 
     this._app.post('/api/docs/:docId/recover', canEdit, throttled(async (req, res) => {
       const recoveryModeRaw = req.body.recoveryMode;

app/server/lib/OIDCConfig.ts

@@ -47,7 +47,9 @@
  *        A JSON object with extra client metadata to pass to openid-client. Optional.
  *        Be aware that setting this object may override any other values passed to the openid client.
  *        More info: https://github.com/panva/node-openid-client/tree/main/docs#new-clientmetadata-jwks-options
- *
+ *    env GRIST_OIDC_SP_HTTP_TIMEOUT
+ *        The timeout in milliseconds for HTTP requests to the IdP. The default value is set to 3500 by the
+ *        openid-client library. See: https://github.com/panva/node-openid-client/blob/main/docs/README.md#customizing-http-requests
  *
  * This version of OIDCConfig has been tested with Keycloak OIDC IdP following the instructions
  * at:
@@ -66,7 +68,7 @@
 import * as express from 'express';
 import { GristLoginSystem, GristServer } from './GristServer';
 import {
-  Client, ClientMetadata, Issuer, errors as OIDCError, TokenSet, UserinfoResponse
+  Client, ClientMetadata, custom, Issuer, errors as OIDCError, TokenSet, UserinfoResponse
 } from 'openid-client';
 import { Sessions } from './Sessions';
 import log from 'app/server/lib/log';
@@ -137,6 +139,9 @@ export class OIDCConfig {
       envVar: 'GRIST_OIDC_IDP_CLIENT_SECRET',
       censor: true,
     });
+    const httpTimeout = section.flag('httpTimeout').readInt({
+      envVar: 'GRIST_OIDC_SP_HTTP_TIMEOUT',
+    });
     this._namePropertyKey = section.flag('namePropertyKey').readString({
       envVar: 'GRIST_OIDC_SP_PROFILE_NAME_ATTR',
     });
@@ -173,6 +178,9 @@ export class OIDCConfig {
     this._protectionManager = new ProtectionsManager(enabledProtections);
 
     this._redirectUrl = new URL(CALLBACK_URL, spHost).href;
+    custom.setHttpOptionsDefaults({
+      ...(httpTimeout !== undefined ? {timeout: httpTimeout} : {}),
+    });
     await this._initClient({ issuerUrl, clientId, clientSecret, extraMetadata });
 
     if (this._client.issuer.metadata.end_session_endpoint === undefined &&

app/server/lib/TestLogin.ts

@@ -25,10 +25,8 @@ export async function getTestLoginSystem(): Promise<GristLoginSystem> {
           if (process.env.TEST_SUPPORT_API_KEY) {
             const dbManager = gristServer.getHomeDBManager();
             const user = await dbManager.getUserByLogin(SUPPORT_EMAIL);
-            if (user) {
+            user.apiKey = process.env.TEST_SUPPORT_API_KEY;
-              user.apiKey = process.env.TEST_SUPPORT_API_KEY;
+            await user.save();
-              await user.save();
-            }
           }
           return "test-login";
         },

app/server/lib/dbUtils.ts

@@ -45,38 +45,53 @@ export async function updateDb(connection?: Connection) {
   await synchronizeProducts(connection, true);
 }
 
+export function getConnectionName() {
+  return process.env.TYPEORM_NAME || 'default';
+}
+
 /**
  * Get a connection to db if one exists, or create one. Serialized to
  * avoid duplication.
  */
 const connectionMutex = new Mutex();
+
+async function buildConnection(overrideConf?: Partial<DataSourceOptions>) {
+  const settings = getTypeORMSettings(overrideConf);
+  const connection = await createConnection(settings);
+  // When using Sqlite, set a busy timeout of 3s to tolerate a little
+  // interference from connections made by tests. Logging doesn't show
+  // any particularly slow queries, but bad luck is possible.
+  // This doesn't affect when Postgres is in use. It also doesn't have
+  // any impact when there is a single connection to the db, as is the
+  // case when Grist is run as a single process.
+  if (connection.driver.options.type === 'sqlite') {
+    await connection.query('PRAGMA busy_timeout = 3000');
+  }
+  return connection;
+}
+
 export async function getOrCreateConnection(): Promise<Connection> {
-  return connectionMutex.runExclusive(async() => {
+  return connectionMutex.runExclusive(async () => {
     try {
       // If multiple servers are started within the same process, we
       // share the database connection.  This saves locking trouble
       // with Sqlite.
-      const connection = getConnection();
+      return getConnection(getConnectionName());
-      return connection;
     } catch (e) {
       if (!String(e).match(/ConnectionNotFoundError/)) {
         throw e;
       }
-      const connection = await createConnection(getTypeORMSettings());
+      return buildConnection();
-      // When using Sqlite, set a busy timeout of 3s to tolerate a little
-      // interference from connections made by tests. Logging doesn't show
-      // any particularly slow queries, but bad luck is possible.
-      // This doesn't affect when Postgres is in use. It also doesn't have
-      // any impact when there is a single connection to the db, as is the
-      // case when Grist is run as a single process.
-      if (connection.driver.options.type === 'sqlite') {
-        await connection.query('PRAGMA busy_timeout = 3000');
-      }
-      return connection;
     }
   });
 }
 
+export async function createNewConnection(overrideConf?: Partial<DataSourceOptions>): Promise<Connection> {
+  return connectionMutex.runExclusive(async () => {
+    return buildConnection(overrideConf);
+  });
+}
+
 export async function runMigrations(connection: Connection) {
   // on SQLite, migrations fail if we don't temporarily disable foreign key
   // constraint checking.  This is because for sqlite typeorm copies each
@@ -103,7 +118,7 @@ export async function undoLastMigration(connection: Connection) {
 // Replace the old janky ormconfig.js file, which was always a source of
 // pain to use since it wasn't properly integrated into the typescript
 // project.
-export function getTypeORMSettings(): DataSourceOptions {
+export function getTypeORMSettings(overrideConf?: Partial<DataSourceOptions>): DataSourceOptions {
   // If we have a redis server available, tell typeorm.  Then any queries built with
   // .cache() called on them will be cached via redis.
   // We use a separate environment variable for the moment so that we don't have to
@@ -120,7 +135,7 @@ export function getTypeORMSettings(): DataSourceOptions {
   } : undefined;
 
   return {
-    "name": process.env.TYPEORM_NAME || "default",
+    "name": getConnectionName(),
     "type": (process.env.TYPEORM_TYPE as any) || "sqlite",  // officially, TYPEORM_CONNECTION -
                                                    // but if we use that, this file will never
                                                    // be read, and we can't configure
@@ -144,5 +159,6 @@ export function getTypeORMSettings(): DataSourceOptions {
     ],
     ...JSON.parse(process.env.TYPEORM_EXTRA || "{}"),
     ...cache,
+    ...overrideConf,
   };
 }

buildtools/.grist-ee-version

@@ -1 +1 @@
-0.9.8
+0.9.9

package.json

@@ -1,6 +1,6 @@
 {
   "name": "grist-core",
-  "version": "1.1.17",
+  "version": "1.1.18",
   "license": "Apache-2.0",
   "description": "Grist is the evolution of spreadsheets",
   "homepage": "https://github.com/gristlabs/grist-core",
@@ -14,13 +14,13 @@
     "install:python3": "buildtools/prepare_python3.sh",
     "build:prod": "buildtools/build.sh",
     "start:prod": "sandbox/run.sh",
-    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
+    "test": "GRIST_TEST_LOGIN=1 ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} --slow 8000 $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/nbrowser_with_stubs/**/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
-    "test:nbrowser": "TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'",
+    "test:nbrowser": "GRIST_TEST_LOGIN=1 TEST_SUITE=nbrowser TEST_SUITE_FOR_TIMINGS=nbrowser TIMINGS_FILE=test/timings/nbrowser.txt ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser/**/*.js'",
-    "test:stubs": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true LANGUAGE=en_US mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser_with_stubs/**/*.js'",
+    "test:stubs": "GRIST_TEST_LOGIN=1 ./test/test_env.sh mocha ${DEBUG:+-b --no-exit} $([ -z $DEBUG ] && echo --forbid-only) -g \"${GREP_TESTS}\" --slow 8000 -R test/xunit-file '_build/test/nbrowser_with_stubs/**/*.js'",
-    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
+    "test:client": "./test/test_env.sh mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
-    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
+    "test:common": "./test/test_env.sh mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
-    "test:server": "TEST_CLEAN_DATABASE=true TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt GRIST_SESSION_COOKIE=grist_test_cookie mocha ${DEBUG:+'-b'} -g \"${GREP_TESTS}\" -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
+    "test:server": "TEST_SUITE=server TEST_SUITE_FOR_TIMINGS=server TIMINGS_FILE=test/timings/server.txt ./test/test_env.sh mocha ${DEBUG:+'-b'} -g \"${GREP_TESTS}\" -R test/xunit-file '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
-    "test:smoke": "LANGUAGE=en_US mocha _build/test/nbrowser/Smoke.js",
+    "test:smoke": "./test/test_env.sh mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
     "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
     "cli": "NODE_PATH=_build:_build/stubs:_build/ext node _build/app/server/companion.js",

static/locales/fr.client.json

@@ -958,7 +958,8 @@
         "Don't show again.": "Ne plus montrer.",
         "Got it": "J'ai compris",
         "Are you sure you want to delete these records?": "Êtes-vous sûr de vouloir supprimer ces enregistrements ?",
-        "Are you sure you want to delete this record?": "Êtes-vous sûr de vouloir supprimer cet enregistrement ?"
+        "Are you sure you want to delete this record?": "Êtes-vous sûr de vouloir supprimer cet enregistrement ?",
+        "TIP": "CONSEIL"
     },
     "pages": {
         "Rename": "Renommer",
@@ -1242,7 +1243,8 @@
         "Table": "Table",
         "Enabled": "Activé",
         "Name": "Nom",
-        "Sorry, not all fields can be edited.": "Désolé, tous les champs ne peuvent pas être modifiés."
+        "Sorry, not all fields can be edited.": "Désolé, tous les champs ne peuvent pas être modifiés.",
+        "Header Authorization": "Entête de sécurité"
     },
     "FormulaAssistant": {
         "Grist's AI Formula Assistance. ": "Assistance des formules de l'IA de Grist ",
@@ -1558,7 +1560,9 @@
         "Sandboxing": "Bac à sable",
         "Self Checks": "Auto contrôles",
         "Session Secret": "Secret de session",
-        "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Ou, comme plan B., vous pouvez renseigner : {{bootKey}} dans l'environnement et visiter : {{url}}"
+        "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Ou, comme plan B., vous pouvez renseigner : {{bootKey}} dans l'environnement et visiter : {{url}}",
+        "Enable Grist Enterprise": "Activer Grist Entreprise",
+        "Enterprise": "Entreprise"
     },
     "Field": {
         "No choices configured": "Aucun choix configuré",
@@ -1665,5 +1669,17 @@
         "Tell us who you are": "Dites-nous qui vous êtes",
         "What brings you to Grist (you can select multiple)?": "Qu'est-ce qui vous amène à Grist (vous pouvez en sélectionner plusieurs) ?",
         "What is your role?": "Quel est votre rôle ?"
+    },
+    "ToggleEnterpriseWidget": {
+        "Disable Grist Enterprise": "Désactiver Grist Entreprise",
+        "Enable Grist Enterprise": "Activer Grist Entreprise",
+        "Grist Enterprise is **enabled**.": "Grist Entreprise est **activité**."
+    },
+    "ViewLayout": {
+        "Delete": "Supprimer",
+        "Delete data and this widget.": "Supprimer les données et le composant.",
+        "Keep data and delete widget. Table will remain available in {{rawDataLink}}": "Garder les données et supprimer le composant. La table restera disponible dans {{rawDataLink}}",
+        "Table {{tableName}} will no longer be visible": "La table {{tableName}} ne sera plus visible",
+        "raw data page": "page de données brutes"
     }
 }

stubs/app/server/server.ts

@@ -79,10 +79,6 @@ async function setupDb() {
       }
       const profile = {email, name: email};
       const user = await db.getUserByLogin(email, {profile});
-      if (!user) {
-        // This should not happen.
-        throw new Error('failed to create GRIST_DEFAULT_EMAIL user');
-      }
       db.unwrapQueryResult(await db.addOrg(user, {
         name: org,
         domain: org,

test/gen-server/ApiServer.ts

@@ -49,9 +49,9 @@ describe('ApiServer', function() {
     homeUrl = await server.start(['home', 'docs']);
     dbManager = server.dbManager;
 
-    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user!.ref);
+    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user.ref);
-    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user!.ref);
+    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user.ref);
-    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user!.ref);
+    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user.ref);
 
     // Listen to user count updates and add them to an array.
     dbManager.on('userChange', ({org, countBefore, countAfter}: UserChange) => {
@@ -2070,8 +2070,7 @@ describe('ApiServer', function() {
     // create a new user
     const profile = {email: 'meep@getgrist.com', name: 'Meep'};
     const user = await dbManager.getUserByLogin('meep@getgrist.com', {profile});
-    assert(user);
+    const userId = user.id;
-    const userId = user!.id;
     // set up an api key
     await dbManager.connection.query("update users set api_key = 'api_key_for_meep' where id = $1", [userId]);
 
@@ -2111,11 +2110,10 @@ describe('ApiServer', function() {
     const userBlank = await dbManager.getUserByLogin('blank@getgrist.com',
                                                      {profile: {email: 'blank@getgrist.com',
                                                       name: ''}});
-    assert(userBlank);
+    await dbManager.connection.query("update users set api_key = 'api_key_for_blank' where id = $1", [userBlank.id]);
-    await dbManager.connection.query("update users set api_key = 'api_key_for_blank' where id = $1", [userBlank!.id]);
 
     // check that user can delete themselves
-    resp = await axios.delete(`${homeUrl}/api/users/${userBlank!.id}`,
+    resp = await axios.delete(`${homeUrl}/api/users/${userBlank.id}`,
                               {data: {name: ""}, ...configForUser("blank")});
     assert.equal(resp.status, 200);
 

test/gen-server/ApiServerAccess.ts

@@ -61,9 +61,9 @@ describe('ApiServerAccess', function() {
       }
     );
     dbManager = server.dbManager;
-    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user!.ref);
+    chimpyRef = await dbManager.getUserByLogin(chimpyEmail).then((user) => user.ref);
-    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user!.ref);
+    kiwiRef = await dbManager.getUserByLogin(kiwiEmail).then((user) => user.ref);
-    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user!.ref);
+    charonRef = await dbManager.getUserByLogin(charonEmail).then((user) => user.ref);
     // Listen to user count updates and add them to an array.
     dbManager.on('userChange', ({org, countBefore, countAfter}: UserChange) => {
       if (countBefore === countAfter) { return; }

test/gen-server/ApiServerBugs.ts

@@ -32,7 +32,7 @@ describe('ApiServerBugs', function() {
     server = new TestServer(this);
     homeUrl = await server.start();
     dbManager = server.dbManager;
-    userRef = (email) => server.dbManager.getUserByLogin(email).then((user) => user!.ref);
+    userRef = (email) => server.dbManager.getUserByLogin(email).then((user) => user.ref);
   });
 
   after(async function() {

test/gen-server/lib/HomeDBManager.ts

@@ -33,14 +33,14 @@ describe('HomeDBManager', function() {
 
   it('can find existing user by email', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    assert.equal(user!.name, 'Chimpy');
+    assert.equal(user.name, 'Chimpy');
   });
 
   it('can create new user by email, with personal org', async function() {
     const profile = {email: 'unseen@getgrist.com', name: 'Unseen'};
     const user = await home.getUserByLogin('unseen@getgrist.com', {profile});
-    assert.equal(user!.name, 'Unseen');
+    assert.equal(user.name, 'Unseen');
-    const orgs = await home.getOrgs(user!.id, null);
+    const orgs = await home.getOrgs(user.id, null);
     assert.isAtLeast(orgs.data!.length, 1);
     assert.equal(orgs.data![0].name, 'Personal');
     assert.equal(orgs.data![0].owner.name, 'Unseen');
@@ -65,37 +65,37 @@ describe('HomeDBManager', function() {
     // log in without a name
     let user = await home.getUserByLogin('unseen2@getgrist.com');
     // name is blank
-    assert.equal(user!.name, '');
+    assert.equal(user.name, '');
     // log in with a name
     const profile: UserProfile = {email: 'unseen2@getgrist.com', name: 'Unseen2'};
     user = await home.getUserByLogin('unseen2@getgrist.com', {profile});
     // name is now set
-    assert.equal(user!.name, 'Unseen2');
+    assert.equal(user.name, 'Unseen2');
     // log in without a name
     user = await home.getUserByLogin('unseen2@getgrist.com');
     // name is still set
-    assert.equal(user!.name, 'Unseen2');
+    assert.equal(user.name, 'Unseen2');
     // no picture yet
-    assert.equal(user!.picture, null);
+    assert.equal(user.picture, null);
     // log in with picture link
     profile.picture = 'http://picture.pic';
     user = await home.getUserByLogin('unseen2@getgrist.com', {profile});
     // now should have a picture link
-    assert.equal(user!.picture, 'http://picture.pic');
+    assert.equal(user.picture, 'http://picture.pic');
     // log in without picture
     user = await home.getUserByLogin('unseen2@getgrist.com');
     // should still have picture link
-    assert.equal(user!.picture, 'http://picture.pic');
+    assert.equal(user.picture, 'http://picture.pic');
   });
 
   it('can add an org', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    const orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, teamOptions)).data!;
+    const orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, teamOptions)).data!;
-    const org = await home.getOrg({userId: user!.id}, orgId);
+    const org = await home.getOrg({userId: user.id}, orgId);
     assert.equal(org.data!.name, 'NewOrg');
     assert.equal(org.data!.domain, 'novel-org');
     assert.equal(org.data!.billingAccount.product.name, TEAM_PLAN);
-    await home.deleteOrg({userId: user!.id}, orgId);
+    await home.deleteOrg({userId: user.id}, orgId);
   });
 
   it('creates default plan if defined', async function() {
@@ -104,28 +104,28 @@ describe('HomeDBManager', function() {
     try {
       // Set the default product to be the free plan.
       process.env.GRIST_DEFAULT_PRODUCT = FREE_PLAN;
-      let orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, {
+      let orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, {
         setUserAsOwner: false,
         useNewPlan: true,
         // omit plan, to use a default one (teamInitial)
         // it will either be 'stub' or anything set in GRIST_DEFAULT_PRODUCT
       })).data!;
-      let org = await home.getOrg({userId: user!.id}, orgId);
+      let org = await home.getOrg({userId: user.id}, orgId);
       assert.equal(org.data!.name, 'NewOrg');
       assert.equal(org.data!.domain, 'novel-org');
       assert.equal(org.data!.billingAccount.product.name, FREE_PLAN);
-      await home.deleteOrg({userId: user!.id}, orgId);
+      await home.deleteOrg({userId: user.id}, orgId);
 
       // Now remove the default product, and check that the default plan is used.
       delete process.env.GRIST_DEFAULT_PRODUCT;
-      orgId = (await home.addOrg(user!, {name: 'NewOrg', domain: 'novel-org'}, {
+      orgId = (await home.addOrg(user, {name: 'NewOrg', domain: 'novel-org'}, {
         setUserAsOwner: false,
         useNewPlan: true,
       })).data!;
 
-      org = await home.getOrg({userId: user!.id}, orgId);
+      org = await home.getOrg({userId: user.id}, orgId);
       assert.equal(org.data!.billingAccount.product.name, STUB_PLAN);
-      await home.deleteOrg({userId: user!.id}, orgId);
+      await home.deleteOrg({userId: user.id}, orgId);
     } finally {
       oldEnv.restore();
     }
@@ -134,17 +134,17 @@ describe('HomeDBManager', function() {
   it('cannot duplicate a domain', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
     const domain = 'repeated-domain';
-    const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+    const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
     const orgId = result.data!;
     assert.equal(result.status, 200);
-    await assert.isRejected(home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions),
+    await assert.isRejected(home.addOrg(user, {name: `${domain}!`, domain}, teamOptions),
                             /Domain already in use/);
-    await home.deleteOrg({userId: user!.id}, orgId);
+    await home.deleteOrg({userId: user.id}, orgId);
   });
 
   it('cannot add an org with a (blacklisted) dodgy domain', async function() {
     const user = await home.getUserByLogin('chimpy@getgrist.com');
-    const userId = user!.id;
+    const userId = user.id;
     const misses = [
       'thing!', ' thing', 'ww', 'docs-999', 'o-99', '_domainkey', 'www', 'api',
       'thissubdomainiswaytoolongmyfriendyoushouldrethinkitoratleastsummarizeit',
@@ -154,13 +154,13 @@ describe('HomeDBManager', function() {
       'thing', 'jpl', 'xyz', 'appel', '123', '1google'
     ];
     for (const domain of misses) {
-      const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+      const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
       assert.equal(result.status, 400);
       const org = await home.getOrg({userId}, domain);
       assert.equal(org.status, 404);
     }
     for (const domain of hits) {
-      const result = await home.addOrg(user!, {name: `${domain}!`, domain}, teamOptions);
+      const result = await home.addOrg(user, {name: `${domain}!`, domain}, teamOptions);
       assert.equal(result.status, 200);
       const org = await home.getOrg({userId}, domain);
       assert.equal(org.status, 200);
@@ -189,7 +189,7 @@ describe('HomeDBManager', function() {
 
     // Fetch the doc and check that the updatedAt value is as expected.
     const kiwi = await home.getUserByLogin('kiwi@getgrist.com');
-    const resp1 = await home.getOrgWorkspaces({userId: kiwi!.id}, primatelyOrgId);
+    const resp1 = await home.getOrgWorkspaces({userId: kiwi.id}, primatelyOrgId);
     assert.equal(resp1.status, 200);
 
     // Check that the apples metadata is as expected. updatedAt should have been set
@@ -209,7 +209,7 @@ describe('HomeDBManager', function() {
 
     // Check that the shark metadata is as expected. updatedAt should have been set
     // to 2004. usage should be set.
-    const resp2 = await home.getOrgWorkspaces({userId: kiwi!.id}, fishOrgId);
+    const resp2 = await home.getOrgWorkspaces({userId: kiwi.id}, fishOrgId);
     assert.equal(resp2.status, 200);
     const shark = resp2.data![0].docs.find((doc: any) => doc.name === 'Shark');
     assert.equal(shark!.updatedAt.toISOString(), setDateISO2);
@@ -340,7 +340,7 @@ describe('HomeDBManager', function() {
 
   it('can fork docs', async function() {
     const user1 = await home.getUserByLogin('kiwi@getgrist.com');
-    const user1Id = user1!.id;
+    const user1Id = user1.id;
     const orgId = await home.testGetId('Fish') as number;
     const doc1Id = await home.testGetId('Shark') as string;
     const scope = {userId: user1Id, urlId: doc1Id};
@@ -393,7 +393,7 @@ describe('HomeDBManager', function() {
 
     // Now fork "Shark" as Chimpy, and check that Kiwi's forks aren't listed.
     const user2 = await home.getUserByLogin('chimpy@getgrist.com');
-    const user2Id = user2!.id;
+    const user2Id = user2.id;
     const resp4 = await home.getOrgWorkspaces({userId: user2Id}, orgId);
     const resp4Doc = resp4.data![0].docs.find((d: any) => d.name === 'Shark');
     assert.deepEqual(resp4Doc!.forks, []);

test/gen-server/lib/emails.ts

@@ -19,7 +19,7 @@ describe('emails', function() {
   beforeEach(async function() {
     this.timeout(5000);
     server = new TestServer(this);
-    ref = (email: string) => server.dbManager.getUserByLogin(email).then((user) => user!.ref);
+    ref = (email: string) => server.dbManager.getUserByLogin(email).then((user) => user.ref);
     serverUrl = await server.start();
   });
 

test/gen-server/lib/removedAt.ts

@@ -258,7 +258,7 @@ describe('removedAt', function() {
           'test3@getgrist.com': 'editors',
         }
       });
-      const userRef = (email: string) => home.dbManager.getUserByLogin(email).then((user) => user!.ref);
+      const userRef = (email: string) => home.dbManager.getUserByLogin(email).then((user) => user.ref);
       const idTest1 = (await home.dbManager.getUserByLogin("test1@getgrist.com"))!.id;
       const idTest2 = (await home.dbManager.getUserByLogin("test2@getgrist.com"))!.id;
       const idTest3 = (await home.dbManager.getUserByLogin("test3@getgrist.com"))!.id;

test/gen-server/seed.ts

@@ -42,7 +42,9 @@ import {User} from "app/gen-server/entity/User";
 import {Workspace} from "app/gen-server/entity/Workspace";
 import {EXAMPLE_WORKSPACE_NAME} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {Permissions} from 'app/gen-server/lib/Permissions';
-import {getOrCreateConnection, runMigrations, undoLastMigration, updateDb} from 'app/server/lib/dbUtils';
+import {
+  getConnectionName, getOrCreateConnection, runMigrations, undoLastMigration, updateDb
+} from 'app/server/lib/dbUtils';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import * as fse from 'fs-extra';
 
@@ -527,16 +529,27 @@ class Seed {
 // When running mocha on several test files at once, we need to reset our database connection
 // if it exists.  This is a little ugly since it is stored globally.
 export async function removeConnection() {
-  if (getConnectionManager().connections.length > 0) {
+  const connections = getConnectionManager().connections;
-    if (getConnectionManager().connections.length > 1) {
+  if (connections.length > 0) {
+    if (connections.length > 1) {
       throw new Error("unexpected number of connections");
     }
-    await getConnectionManager().connections[0].close();
+    await connections[0].destroy();
-    // There is still no official way to delete connections that I've found.
+    dereferenceConnection(getConnectionName());
-    (getConnectionManager() as any).connectionMap = new Map();
   }
 }
 
+export function dereferenceConnection(name: string) {
+  // There seem to be no official way to delete connections.
+  // Also we should probably get rid of the use of connectionManager, which is deprecated
+  const connectionMgr = getConnectionManager();
+  const connectionMap = (connectionMgr as any).connectionMap as Map<string, Connection>;
+  if (!connectionMap.has(name)) {
+    throw new Error('connection with this name not found: ' + name);
+  }
+  connectionMap.delete(name);
+}
+
 export async function createInitialDb(connection?: Connection, migrateAndSeedData: boolean = true) {
   // In jenkins tests, we may want to reset the database to a clean
   // state.  If so, TEST_CLEAN_DATABASE will have been set.  How to

test/gen-server/testUtils.ts

@@ -46,7 +46,6 @@ export async function createUser(dbManager: HomeDBManager, name: string): Promis
   const username = name.toLowerCase();
   const email = `${username}@getgrist.com`;
   const user = await dbManager.getUserByLogin(email, {profile: {email, name}});
-  if (!user) { throw new Error('failed to create user'); }
   user.apiKey = `api_key_for_${username}`;
   await user.save();
   const userHome = (await dbManager.getOrg({userId: user.id}, null)).data;

test/nbrowser/homeUtil.ts

@@ -420,7 +420,7 @@ export class HomeUtil {
     if (this.server.isExternalServer()) { throw new Error('not supported'); }
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (user) { await dbManager.deleteUser({userId: user.id}, user.id, user.name); }
+    await dbManager.deleteUser({userId: user.id}, user.id, user.name);
   }
 
   // Set whether this is the user's first time logging in.  Requires access to the database.
@@ -428,10 +428,8 @@ export class HomeUtil {
     if (this.server.isExternalServer()) { throw new Error('not supported'); }
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (user) {
+    user.isFirstTimeUser = isFirstLogin;
-      user.isFirstTimeUser = isFirstLogin;
+    await user.save();
-      await user.save();
-    }
   }
 
   private async _initShowGristTour(email: string, showGristTour: boolean) {
@@ -463,7 +461,6 @@ export class HomeUtil {
 
     const dbManager = await this.server.getDatabase();
     const user = await dbManager.getUserByLogin(email);
-    if (!user) { return; }
 
     if (user.personalOrg) {
       const org = await dbManager.getOrg({userId: user.id}, user.personalOrg.id);

test/server/fixSiteProducts.ts

@@ -122,7 +122,7 @@ describe('fixSiteProducts', function() {
     assert.equal(getDefaultProductNames().teamInitial, 'stub');
 
     const db = server.dbManager;
-    const user = await db.getUserByLogin(email, {profile}) as any;
+    const user = await db.getUserByLogin(email, {profile});
     const orgId = db.unwrapQueryResult(await db.addOrg(user, {
       name: 'sanity-check-org',
       domain: 'sanity-check-org',

test/server/lib/GranularAccess.ts

@@ -3279,7 +3279,7 @@ describe('GranularAccess', function() {
     cliOwner.flush();
     let perm: PermissionDataWithExtraUsers = (await cliOwner.send("getUsersForViewAs", 0)).data;
     const getId = (name: string) => home.dbManager.testGetId(name) as Promise<number>;
-    const getRef = (email: string) => home.dbManager.getUserByLogin(email).then(user => user!.ref);
+    const getRef = (email: string) => home.dbManager.getUserByLogin(email).then(user => user.ref);
     assert.deepEqual(perm.users, [
       { id: await getId('Chimpy'), email: 'chimpy@getgrist.com', name: 'Chimpy',
         ref: await getRef('chimpy@getgrist.com'),

test/server/lib/OIDCConfig.ts

@@ -5,7 +5,7 @@ import {Sessions} from "app/server/lib/Sessions";
 import log from "app/server/lib/log";
 import {assert} from "chai";
 import Sinon from "sinon";
-import {Client, generators, errors as OIDCError} from "openid-client";
+import {Client, custom, generators, errors as OIDCError} from "openid-client";
 import express from "express";
 import _ from "lodash";
 import {RequestWithLogin} from "app/server/lib/Authorizer";
@@ -192,6 +192,55 @@ describe('OIDCConfig', () => {
         });
       });
     });
+
+    describe('GRIST_OIDC_SP_HTTP_TIMEOUT', function () {
+      [
+        {
+          itMsg: 'when omitted should not override openid-client default value',
+          expectedUserDefinedHttpOptions: {}
+        },
+        {
+          itMsg: 'should reject when the provided value is not a number',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '__NOT_A_NUMBER__',
+          },
+          expectedErrorMsg: /__NOT_A_NUMBER__ does not look like a number/,
+        },
+        {
+          itMsg: 'should override openid-client timeout accordingly to the provided value',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '10000',
+          },
+          shouldSetTimeout: true,
+          expectedUserDefinedHttpOptions: {
+            timeout: 10000
+          }
+        },
+        {
+          itMsg: 'should allow disabling the timeout by having its value set to 0',
+          env: {
+            GRIST_OIDC_SP_HTTP_TIMEOUT: '0',
+          },
+          expectedUserDefinedHttpOptions: {
+            timeout: 0
+          }
+        }
+      ].forEach(ctx => {
+        it(ctx.itMsg, async () => {
+          const setHttpOptionsDefaultsStub = sandbox.stub(custom, 'setHttpOptionsDefaults');
+          setEnvVars();
+          Object.assign(process.env, ctx.env);
+          const promise = OIDCConfigStubbed.buildWithStub();
+          if (ctx.expectedErrorMsg) {
+            await assert.isRejected(promise, ctx.expectedErrorMsg);
+          } else {
+            await assert.isFulfilled(promise, 'initOIDC should have been fulfilled');
+            assert.isTrue(setHttpOptionsDefaultsStub.calledOnce, 'Should have called custom.setHttpOptionsDefaults');
+            assert.deepEqual(setHttpOptionsDefaultsStub.firstCall.args[0], ctx.expectedUserDefinedHttpOptions);
+          }
+        });
+      });
+    });
   });
 
   describe('GRIST_OIDC_IDP_ENABLED_PROTECTIONS', () => {

test/server/lib/helpers/PrepareDatabase.ts

@@ -2,13 +2,13 @@ import path from "path";
 import * as testUtils from "test/server/testUtils";
 import {execFileSync} from "child_process";
 
-export async function prepareDatabase(tempDirectory: string) {
+export async function prepareDatabase(tempDirectory: string, filename: string = 'landing.db') {
   // Let's create a sqlite db that we can share with servers that run in other processes, hence
   // not an in-memory db. Running seed.ts directly might not take in account the most recent value
   // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
   // configuration (in-memory for instance). Spawning a process is one way to make sure that the
   // latest value prevail.
-  process.env.TYPEORM_DATABASE = path.join(tempDirectory, 'landing.db');
+  process.env.TYPEORM_DATABASE = path.join(tempDirectory, filename);
   const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
   execFileSync('node', [seed, 'init'], {
     env: process.env,

test/test_env.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+export GRIST_SESSION_COOKIE="grist_test_cookie"
+export LANGUAGE="en_US"
+export SE_BROWSER="chrome"
+export SE_BROWSER_VERSION="127"
+export SE_DRIVER="chrome-driver"
+export SE_DRIVER_VERSION="127.0.6533.119"
+export TEST_CLEAN_DATABASE="true"
+export TEST_SUPPORT_API_KEY="api_key_for_support"
+
+exec "$@"

test/test_under_docker.sh

@@ -14,6 +14,8 @@ trap 'cleanup' EXIT
 trap 'echo "Exiting on SIGINT"; exit 1' INT
 trap 'echo "Exiting on SIGTERM"; exit 1' TERM
 
+source $(dirname $0)/test_env.sh
+
 PORT=8585
 DOCKER_CONTAINER=grist-core-test
 DOCKER_PID=""
@@ -65,8 +67,6 @@ fi
 
 TEST_ADD_SAMPLES=1 TEST_ACCOUNT_PASSWORD=not-needed \
   HOME_URL=http://localhost:8585 \
-  GRIST_SESSION_COOKIE=grist_test_cookie \
   GRIST_TEST_LOGIN=1 \
   NODE_PATH=_build:_build/stubs \
-  LANGUAGE=en_US \
   $MOCHA _build/test/deployment/*.js --slow 6000 -g "${GREP_TESTS:-}" "$@"