(core) Update dependencies

Summary:
Changes the minimum version of Node to 18, and updates the Docker images and GitHub workflows to build Grist with Node 18.

Also updates various dependencies and scripts to support building running tests with arm64 builds of Node.

Test Plan: Existing tests.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3968

.github/workflows/docker_latest.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         python-version: [3.9]
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [3.9]
-        node-version: [14.x]
+        node-version: [18.x]
         tests:
           - ':lint:python:client:common:smoke:'
           - ':server-1-of-2:'
@@ -30,10 +30,10 @@ jobs:
           - ':nbrowser-^[^A-S]:'
         include:
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 14.x
+            node-version: 18.x
             python-version: '3.10'
           - tests: ':lint:python:client:common:smoke:'
-            node-version: 14.x
+            node-version: 18.x
             python-version: '3.11'
     steps:
       - uses: actions/checkout@v3
@@ -72,6 +72,10 @@ jobs:
       - name: Build Node.js code
         run: yarn run build:prod
 
+      - name: Install chromedriver
+        if: contains(matrix.tests, ':nbrowser-') || contains(matrix.tests, ':smoke:')
+        run: ./node_modules/selenium-webdriver/bin/linux/selenium-manager --driver chromedriver
+
       - name: Run smoke test
         if: contains(matrix.tests, ':smoke:')
         run: VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:smoke

.gitignore

@@ -77,5 +77,6 @@ jspm_packages/
 # Test
 timings.txt
 xunit.xml
+.clipboard.lock
 
 **/_build

.nvmrc

@@ -1 +1 @@
-14.20.1
+18.17.1

Dockerfile

@@ -10,7 +10,7 @@ FROM scratch as ext
 ## Javascript build stage
 ################################################################################
 
-FROM node:14-buster as builder
+FROM node:18-buster as builder
 
 # Install all node dependencies.
 WORKDIR /grist
@@ -67,7 +67,7 @@ FROM gristlabs/gvisor-unprivileged:buster as sandbox
 ################################################################################
 
 # Now, start preparing final image.
-FROM node:14-buster-slim
+FROM node:18-buster-slim
 
 # Install libexpat1, libsqlite3-0 for python3 library binary dependencies.
 # Install pgrep for managing gvisor processes.

app/client/ui/CustomSectionConfig.ts

@@ -628,6 +628,8 @@ export class CustomSectionConfig extends Disposable {
   protected async _getWidgets() {
     const api = this._gristDoc.app.topAppModel.api;
     const widgets = await api.getWidgets();
+    if (this.isDisposed()) { return; }
+
     // Request for rest of the widgets.
     if (this._canSelect) {
       // From the start we will provide single widget definition

app/plugin/grist-plugin-api.ts

@@ -243,6 +243,15 @@ async function getMappingsIfChanged(data: any): Promise<WidgetColumnMap|null> {
   return _mappingsCache ? JSON.parse(JSON.stringify(_mappingsCache)) : null;
 }
 
+/**
+ * Used by tests to wait for all pending requests to settle.
+ *
+ * TODO: currently only waits for requests for mappings.
+ */
+export async function testWaitForPendingRequests() {
+  return await _activeRefreshReq;
+}
+
 /**
  * Renames columns in the result using columns mapping configuration passed in ready method.
  * Returns null if not all required columns were mapped or not widget doesn't support

app/server/lib/FlexServer.ts

@@ -68,7 +68,6 @@ import {addUploadRoute} from 'app/server/lib/uploads';
 import {buildWidgetRepository, IWidgetRepository} from 'app/server/lib/WidgetRepository';
 import {setupLocale} from 'app/server/localization';
 import axios from 'axios';
-import * as bodyParser from 'body-parser';
 import * as cookie from 'cookie';
 import express from 'express';
 import * as fse from 'fs-extra';
@@ -789,7 +788,7 @@ export class FlexServer implements GristServer {
 
   public addJsonSupport() {
     if (this._check('json')) { return; }
-    this.app.use(bodyParser.json({limit: '1mb'}));  // Increase from the default 100kb
+    this.app.use(express.json({limit: '1mb'}));  // Increase from the default 100kb
   }
 
   public addSessions() {

app/server/lib/NSandbox.ts

@@ -737,6 +737,10 @@ function gvisor(options: ISandboxOptions): SandboxProcess {
     wrapperArgs.push(process.env.GRIST_CHECKPOINT!);
   }
   const child = spawn(command, [...wrapperArgs.get(), `python${pythonVersion}`, '--', ...pythonArgs]);
+  if (!child.pid) {
+    throw new Error(`failed to spawn python${pythonVersion}`);
+  }
+
   // For gvisor under ptrace, main work is done by a traced process identifiable as
   // being labeled "exe" and having a parent also labeled "exe".
   const recognizeTracedProcess = (p: ProcessInfo) => {

app/server/lib/SamlConfig.ts

@@ -52,7 +52,6 @@
  *
  */
 
-import * as bodyParser from 'body-parser';
 import * as express from 'express';
 import * as fse from 'fs-extra';
 import * as saml2 from 'saml2-js';
@@ -161,7 +160,7 @@ export class SamlConfig {
     }));
 
     // Assert endpoint for when the login completes as POST.
-    app.post("/saml/assert", bodyParser.urlencoded({extended: true}), expressWrap(async (req, res, next) => {
+    app.post("/saml/assert", express.urlencoded({extended: true}), expressWrap(async (req, res, next) => {
       const relayState: string = req.body.RelayState;
       if (!relayState) { throw new Error('Login or logout failed to complete'); }
       const permitStore = this._gristServer.getExternalPermitStore();

app/server/lib/SandboxControl.ts

@@ -30,12 +30,16 @@ export interface ISandboxControl {
  * Control a single process directly. A thin wrapper around the Throttle class.
  */
 export class DirectProcessControl implements ISandboxControl {
+  private _pid: number;
   private _throttle?: Throttle;
 
   constructor(private _process: childProcess.ChildProcess, logMeta?: log.ILogMeta) {
+    if (!_process.pid) { throw new Error(`process identifier (PID) is undefined`); }
+
+    this._pid = _process.pid;
     if (process.env.GRIST_THROTTLE_CPU) {
       this._throttle = new Throttle({
-        pid: _process.pid,
+        pid: this._pid,
         logMeta: {...logMeta, pid: _process.pid},
       });
     }
@@ -55,7 +59,7 @@ export class DirectProcessControl implements ISandboxControl {
   }
 
   public async getUsage() {
-    const memory = (await pidusage(this._process.pid)).memory;
+    const memory = (await pidusage(this._pid)).memory;
     return { memory };
   }
 }

app/server/lib/expressWrap.ts

@@ -2,10 +2,16 @@ import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import log from 'app/server/lib/log';
 import * as express from 'express';
 
+export type AsyncRequestHandler = (
+  req: express.Request,
+  res: express.Response,
+  next: express.NextFunction
+) => any | Promise<any>;
+
 /**
  * Wrapper for async express endpoints to catch errors and forward them to the error handler.
  */
-export function expressWrap(callback: express.RequestHandler): express.RequestHandler {
+export function expressWrap(callback: AsyncRequestHandler): express.RequestHandler {
   return async (req, res, next) => {
     try {
       await callback(req, res, next);

package.json

@@ -49,7 +49,7 @@
     "@types/diff-match-patch": "1.0.32",
     "@types/dompurify": "2.4.0",
     "@types/double-ended-queue": "2.1.0",
-    "@types/express": "4.16.0",
+    "@types/express": "4.17.17",
     "@types/form-data": "2.2.1",
     "@types/fs-extra": "5.0.4",
     "@types/http-proxy": "1.17.9",
@@ -64,17 +64,18 @@
     "@types/marked": "4.0.8",
     "@types/mime-types": "2.1.0",
     "@types/minio": "7.0.15",
-    "@types/mocha": "5.2.5",
+    "@types/mocha": "10.0.1",
     "@types/moment-timezone": "0.5.9",
     "@types/mousetrap": "1.6.2",
-    "@types/node": "^14",
+    "@types/node": "18.11.9",
     "@types/node-fetch": "2.6.2",
     "@types/pidusage": "2.0.1",
     "@types/plotly.js": "2.12.1",
+    "@types/proper-lockfile": "4.1.2",
     "@types/qrcode": "1.4.2",
     "@types/redlock": "3.0.2",
     "@types/saml2-js": "2.0.1",
-    "@types/selenium-webdriver": "4.0.0",
+    "@types/selenium-webdriver": "4.1.15",
     "@types/sinon": "5.0.5",
     "@types/sqlite3": "3.1.6",
     "@types/tmp": "0.0.33",
@@ -94,10 +95,11 @@
     "i18next-scanner": "4.1.0",
     "jsdom": "16.5.0",
     "mocha": "10.2.0",
-    "mocha-webdriver": "0.2.13",
+    "mocha-webdriver": "0.3.1",
     "moment-locales-webpack-plugin": "^1.2.0",
     "nodemon": "^2.0.4",
     "otplib": "12.0.1",
+    "proper-lockfile": "4.1.2",
     "sinon": "7.1.1",
     "source-map-loader": "^0.2.4",
     "tmp-promise": "1.0.5",
@@ -137,7 +139,7 @@
     "dompurify": "3.0.0",
     "double-ended-queue": "2.1.0-0",
     "exceljs": "4.2.1",
-    "express": "4.16.4",
+    "express": "4.18.2",
     "file-type": "16.5.4",
     "fs-extra": "7.0.0",
     "grain-rpc": "0.1.7",
@@ -148,7 +150,7 @@
     "https-proxy-agent": "5.0.1",
     "i18n-iso-countries": "6.1.0",
     "i18next": "21.9.1",
-    "i18next-http-middleware": "3.2.1",
+    "i18next-http-middleware": "3.3.2",
     "image-size": "0.6.3",
     "jquery": "3.5.0",
     "js-yaml": "3.14.1",

static/test.html

@@ -24,8 +24,8 @@
             }
 
             mocha.checkLeaks();
-            // fxdriver_id is set by selenium, execWebdriverJQuery by webdriverjq.js.
+            // fxdriver_id and ret_nodes are set by selenium, execWebdriverJQuery by webdriverjq.js.
-            mocha.globals(['cmd', 'fxdriver_id', 'execWebdriverJQuery']);
+            mocha.globals(['cmd', 'fxdriver_id', 'ret_nodes', 'execWebdriverJQuery']);
             var runner = mocha.run();
             mocha.failedTests = [];
             runner.on('fail', function(test, err) {

test/declarations.d.ts

@@ -15,31 +15,3 @@ declare namespace Chai {
     notIncludeMembers<T>(superset: T[], subset: T[], message?: string): void;
   }
 }
-
-declare module "selenium-webdriver" {
-  interface WebDriver {
-    withActions(cb: (actions: WebActions) => void): Promise<void>;
-  }
-
-  // This is not a complete definition of available methods, but only those that we use for now.
-  // TODO: find documentation for this interface or update selenium-webdriver.
-  interface WebActions {
-    contextClick(el?: WebElement): WebActions;
-    click(el?: WebElement): WebActions;
-    press(): WebActions;
-    move(params: {origin?: WebElement|string, x?: number, y?: number}): WebActions;
-    keyDown(key: string): WebActions;
-    keyUp(key: string): WebActions;
-    dragAndDrop(element: WebElement, target: WebElement): WebActions;
-    release(): WebActions;
-    doubleClick(element: WebElement): WebActions;
-    pause(ms: number): WebActions;
-  }
-}
-
-import "mocha-webdriver";
-declare module "mocha-webdriver" {
-  // It looks like this hack makes tsc see our definition as primary, adding
-  // the typed version override (of the withActions method) as the default one.
-  export declare let driver: import("selenium-webdriver").WebDriver;
-}

test/gen-server/seed.ts

@@ -23,7 +23,7 @@
  */
 
 import {addPath} from 'app-module-path';
-import {IHookCallbackContext} from 'mocha';
+import {Context} from 'mocha';
 import * as path from 'path';
 import {Connection, getConnectionManager, Repository} from 'typeorm';
 
@@ -643,7 +643,7 @@ function _generateData(numOrgs: number, numWorkspaces: number, numDocs: number)
  * To set up TYPEORM_* environment variables for testing, call this in a before() call of a test
  * suite, using setUpDB(this);
  */
-export function setUpDB(context?: IHookCallbackContext) {
+export function setUpDB(context?: Context) {
   if (!process.env.TYPEORM_DATABASE) {
     process.env.TYPEORM_DATABASE = ":memory:";
   } else {

test/init-mocha-webdriver.js

@@ -1,6 +1,6 @@
 /**
  * Settings that affect tests using mocha-webdriver. This module is imported by any run of mocha,
- * by being listed in test/mocha.opts. (Keep in mind that it's imported by non-browser tests, such
+ * by being listed in package.json. (Keep in mind that it's imported by non-browser tests, such
  * as test/common, as well.)
  */
 

test/mocha.opts

@@ -1,5 +0,0 @@
---require source-map-support/register
-test/report-why-tests-hang
-test/init-mocha-webdriver
-test/split-tests
-test/chai-as-promised

test/nbrowser/ChartView1.ts

@@ -0,0 +1,804 @@
+import {UserAPI} from 'app/common/UserAPI';
+import {assert, driver, Key} from 'mocha-webdriver';
+import {addYAxis, checkAxisConfig, checkAxisRange, findYAxis, getAxisTitle, getChartData,
+        removeYAxis, selectChartType, selectXAxis,
+        setSplitSeries} from 'test/nbrowser/chartViewTestUtils';
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('ChartView1', function() {
+  this.timeout(20000);
+  const cleanup = setupTestSuite();
+  let api: UserAPI;
+  let doc: any;
+
+  before(async function() {
+    const session = await gu.session().teamSite.login();
+    doc = await session.tempDoc(cleanup, 'ChartData.grist');
+    api = session.createHomeApi();
+  });
+
+  gu.bigScreen();
+  afterEach(() => gu.checkForErrors());
+
+  it('should allow adding and removing chart viewsections', async function() {
+    // Starting out with one section
+    assert.lengthOf(await driver.findAll('.test-gristdoc .view_leaf'), 1);
+
+    // Add a new chart section
+    await gu.addNewSection(/Chart/, /ChartData/);
+
+    // Check that there are now two sections
+    assert.lengthOf(await driver.findAll('.test-gristdoc .view_leaf'), 2);
+
+    // Delete the newly added one
+    await gu.openSectionMenu('viewLayout', 'CHARTDATA Chart');
+    await driver.find('.test-section-delete').click();
+    await gu.waitForServer();
+
+    // Check that there is now only one section
+    assert.lengthOf(await driver.findAll('.test-gristdoc .view_leaf'), 1);
+  });
+
+  it('should display a bar chart by default', async function() {
+    // Add a new chart section, and make sure it has focus
+    await gu.addNewSection(/Chart/, /ChartData/);
+    const section = await gu.getSection('CHARTDATA Chart');
+    assert.equal(await section.matches('.active_section'), true);
+
+    const chartDom = await section.find('.test-chart-container');
+    assert.equal(await chartDom.isDisplayed(), true);
+
+    const data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 6, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 1, 2, 3, 4, 5, 6 ]);
+  });
+
+  it('should allow viewing raw data underlying chart', async function() {
+    // No raw data overlay at first
+    assert.isFalse(await driver.find('.test-raw-data-overlay').isPresent());
+
+    // Show raw data overlay
+    await gu.openSectionMenu('viewLayout');
+    await driver.find('.test-show-raw-data').click();
+
+    // Test that overlay is showed.
+    assert.isTrue(await driver.findWait('.test-raw-data-overlay', 100).isDisplayed());
+
+    // Test that the widget menu doesn't have the raw data option any more
+    await gu.openSectionMenu('viewLayout');
+    assert.isTrue(await driver.findContentWait('.grist-floating-menu li', 'Print widget', 100).isDisplayed());
+    assert.isFalse(await driver.findContent('.grist-floating-menu li', 'Show raw data').isPresent());
+
+    // Go back and confirm that the overlay is gone again
+    await driver.find('.test-raw-data-close-button').click();
+    assert.isFalse(await driver.find('.test-raw-data-overlay').isPresent());
+
+    // Open once again and close by escaping.
+    await gu.openSectionMenu('viewLayout');
+    await driver.find('.test-show-raw-data').click();
+    assert.isTrue(await driver.findWait('.test-raw-data-overlay', 100).isDisplayed());
+    await gu.sendKeys(Key.ESCAPE);
+    assert.isFalse(await driver.find('.test-raw-data-overlay').isPresent());
+  });
+
+  it('should update as the underlying data changes', async function() {
+    await gu.getCell({section: 'ChartData', col: 0, rowNum: 1}).click();
+    await driver.sendKeys(Key.ENTER, '1', Key.ENTER);   // Change from 6 to 61
+    await gu.waitForServer();
+
+    const chartDom = await driver.find('.test-chart-container');
+    let data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 1, 2, 3, 4, 5, 6 ]);
+
+    await gu.getCell({section: 'ChartData', col: 1, rowNum: 1}).click();
+    await driver.sendKeys(Key.ENTER, '6', Key.ENTER);              // Change from 1 to 16
+    await gu.waitForServer();
+
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 16, 2, 3, 4, 5, 6 ]);
+  });
+
+  it('should skip empty points', async function() {
+    const chartDom = await driver.find('.test-chart-container');
+    let data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 16, 2, 3, 4, 5, 6 ]);
+
+    // Enter some blank values and a zero. The zero should be included in the plot, but blanks
+    // should not.
+    await gu.getCell({col: 1, rowNum: 1}).click();
+    await driver.sendKeys(Key.DELETE);
+    await gu.getCell({col: 1, rowNum: 4}).click();
+    await driver.sendKeys(Key.DELETE);
+    await gu.getCell({col: 1, rowNum: 6}).click();
+    await driver.sendKeys('0', Key.ENTER);
+    await gu.waitForServer();
+
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].x, [ 5, 4, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 2, 3, 5, 0 ]);
+
+    // Undo and verify that the range is restored.
+    await gu.undo(3);
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 16, 2, 3, 4, 5, 6 ]);
+  });
+
+  it('should update chart when new columns are included', async function() {
+    const chartDom = await driver.find('.test-chart-container');
+    // Check to make sure intial values are correct.
+    let data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 16, 2, 3, 4, 5, 6 ]);
+
+    // Check that the intial scales are correct for the dataset.
+    checkAxisRange(await getChartData(chartDom), 0.5, 61.5, 0, 16.5);
+
+    // Open the view config pane for the Chart section.
+    await gu.getSection('ChartData chart').find('.viewsection_title').click();
+    await gu.toggleSidePanel('right', 'open');
+    await driver.find('.test-right-tab-pagewidget').click();
+    await driver.find('.test-config-widget').click();
+
+    // Check intial visible fields.
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['value']
+    });
+
+    // Adds 'largeValue'
+    await driver.find('.test-chart-add-y-axis').click();
+    await driver.findContent('.grist-floating-menu li', 'largeValue').click();
+    await gu.waitForServer();
+
+    // Check axis are correct
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['value', 'largeValue']
+    });
+
+    // Move 'largeValue' above 'value'. Scroll it into view first, since dragging is a bit messed
+    // up when it causes the pane to scroll.
+    await gu.scrollIntoView(findYAxis('largeValue'));
+    await driver.withActions((actions) => actions.dragAndDrop(findYAxis('largeValue'), findYAxis('value')));
+    await gu.waitForServer();
+
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value']
+    });
+
+    // Make sure only y axis updates to the new column of data
+    await driver.sleep(50);
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[0].y, [ 22, 33, 11, 44, 22, 55 ]);
+    assert.deepEqual(data[1].type, 'bar');
+    assert.deepEqual(data[1].x, [ 61, 5, 4, 3, 2, 1 ]);
+    assert.deepEqual(data[1].y, [ 16, 2, 3, 4, 5, 6 ]);
+
+    // Check that the scales are correct for the new y values.
+    checkAxisRange(await getChartData(chartDom), 0.5, 61.5, 0, 57);
+
+    // select 'largeValue' as x axis
+    await selectXAxis('largeValue');
+
+    // check x-axis is correct
+    await checkAxisConfig({
+      xaxis: 'largeValue',
+      yaxis: ['value'] // note: 'largeValue' was correctly removed from y-axis
+    });
+
+    // adds 'label' as y axis
+    await addYAxis('label');
+
+    // check axis are correct
+    await checkAxisConfig({
+      xaxis: 'largeValue',
+      yaxis: ['value', 'label']
+    });
+
+    // Reverse the order of the columns and make sure the data updates to reflect that.
+    await driver.sleep(50);
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].x, [ 22, 33, 11, 44, 55 ]);
+    assert.deepEqual(data[0].y, [ 16, 2, 3, 4, 6 ]);
+    assert.deepEqual(data[1].type, 'bar');
+    assert.deepEqual(data[1].x, [ 22, 33, 11, 44, 55 ]);
+    assert.deepEqual(data[1].y, [ 61, 5, 4, 3, 1 ]);
+
+    // Check that the scales are correct for the new values.
+    checkAxisRange(await getChartData(chartDom), 5.5, 60.5, 0, 61);
+
+    // select 'label' as x axis
+    await selectXAxis('label');
+
+    // adds 'largeValue' as y axis
+    await addYAxis('largeValue');
+
+    // moves 'largeValue' above 'value'
+    await driver.withActions((actions) => actions.dragAndDrop(findYAxis('largeValue'), findYAxis('value')));
+    await gu.waitForServer();
+
+    // check axis correctness
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value']
+    });
+  });
+
+  it('should be able to render different types of charts', async function() {
+    const chartDom = await driver.find('.test-chart-container');
+
+    await selectChartType('Pie Chart');
+    let data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'pie');
+    assert.equal(await driver.find('.test-chart-first-field-label').getText(), 'LABEL');
+    await selectChartType('Line Chart');
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'scatter');
+    // Make sure we are not grouping (which would produce names like "1 · value")
+    assert.equal(data[0].name, 'largeValue');
+    assert.equal(data[1].name, 'value');
+    assert.equal(await driver.find('.test-chart-first-field-label').getText(), 'X-AXIS');
+
+    await selectChartType('Area Chart');
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data[0].line!.shape, 'spline');
+    assert.deepEqual(data[0].fill, 'tozeroy');
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Area Chart');
+
+    // Make sure first field of scatter plot is marked label, not x-axis.
+    await selectChartType('Scatter Plot');
+    assert.equal(await driver.find('.test-chart-first-field-label').getText(), 'LABEL');
+
+    // Make sure first field of Kaplan-Meier plot is marked label, not x-axis.
+    await selectChartType('Kaplan-Meier Plot');
+    assert.equal(await driver.find('.test-chart-first-field-label').getText(), 'LABEL');
+
+    // Return to Area Chart.
+    await selectChartType('Area Chart');
+  });
+
+  it('should render pie charts with a single series, or counts', async function() {
+    await selectChartType('Pie Chart');
+
+    // select 'person' for x axis
+    await selectXAxis('person');
+
+    // adds 'label' and move to be first y axis
+    await addYAxis('label');
+    await driver.withActions((actions) => actions.dragAndDrop(findYAxis('label'), findYAxis('largeValue')));
+    await gu.waitForServer();
+
+    // check axis
+    await checkAxisConfig({
+      xaxis: 'person',
+      yaxis: ['label', 'largeValue', 'value']
+    });
+
+    const chartDom = await driver.find('.test-chart-container');
+    let data = (await getChartData(chartDom)).data;
+    // Only the first series of values is included.
+    assert.deepEqual(data[0].values, [ 61, 4, 2, 5, 3, 1 ]);
+    assert.lengthOf(data, 1);
+
+    // When no series is included, just counts are used.
+    await removeYAxis('largeValue');
+    await removeYAxis('label');
+    await removeYAxis('value');
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].values, [1, 1, 1, 1, 1, 1]);
+    assert.lengthOf(data, 1);
+
+    await gu.undo(7);
+
+    // check axis
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value']
+    });
+
+    // check chart type
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Area Chart');
+  });
+
+  it('should support Y-axis options', async function() {
+    const chartDom = await driver.find('.test-chart-container');
+    await selectChartType('Bar Chart');
+    checkAxisRange(await getChartData(chartDom), 0.5, 61.5, 0, 57);
+
+    await driver.findContent('label', /Invert Y-axis/).find('input').click();
+    await gu.waitForServer();
+    checkAxisRange(await getChartData(chartDom), 0.5, 61.5, 57, 0);
+
+    await driver.findContent('label', /Invert Y-axis/).find('input').click();
+    await driver.findContent('label', /Log scale Y-axis/).find('input').click();
+    await gu.waitForServer();
+    checkAxisRange(await getChartData(chartDom), 0.5, 61.5, 0.22, 1.82);
+
+    await gu.undo(4);
+    // check axis
+    await checkAxisConfig({
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value']
+    });
+    // check chart type
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Area Chart');
+  });
+
+  it('should be able to render multiseries line charts', async function() {
+    const chartDom = await driver.find('.test-chart-container');
+
+    // switch type to line chart
+    await selectChartType('Line Chart');
+
+    // pick 'largeValue' as the x axis
+    await selectXAxis('largeValue');
+
+    // set 'label' as the groupby column
+    await setSplitSeries('label');
+
+    let {data, layout} = await getChartData(chartDom);
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data.map(d => d.name), ['1', '2', '3', '4', '5', '61']);
+    assert.equal(getAxisTitle(layout.xaxis), 'largeValue');
+    assert.equal(getAxisTitle(layout.yaxis), 'value');
+
+    // Select person for grouping by column
+    await setSplitSeries('person');
+
+    await checkAxisConfig({
+      groupingByColumn: 'person',
+      xaxis: 'largeValue',
+      yaxis: ['value'],
+    });
+
+    ({data, layout} = await getChartData(chartDom));
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data.map(d => d.name), ['Alice', 'Bob']);
+    assert.equal(getAxisTitle(layout.xaxis), 'largeValue');
+    assert.equal(getAxisTitle(layout.yaxis), 'value');
+
+    // Add a second series. If we have more than one, its name should be included into the series
+    // names rather than in the yaxis.title.
+    await addYAxis('label');
+
+    await checkAxisConfig({
+      groupingByColumn: 'person',
+      xaxis: 'largeValue',
+      yaxis: ['value', 'label'],
+    });
+
+    ({data, layout} = await getChartData(chartDom));
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data.map(d => d.name), ['Alice • value', 'Alice • label', 'Bob • value', 'Bob • label']);
+    assert.equal(getAxisTitle(layout.xaxis), 'largeValue');
+    assert.equal(getAxisTitle(layout.yaxis), undefined);
+
+    await gu.undo(5);
+    await checkAxisConfig({
+      groupingByColumn: false,
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value'],
+    });
+    // check chart type
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Area Chart');
+  });
+
+  it('should get options for SPLIT SERIES and X AXIS in sync when table changes', async function() {
+
+    // click change widget
+    await driver.findContent('button', 'Change Widget').click();
+
+    // click sum symbol
+    await driver.findContent('.test-wselect-table', 'People').click();
+
+    // click save
+    await driver.find('.test-wselect-addBtn').click();
+    await gu.waitForServer();
+
+    // click Split series
+    await driver.findContent('label', 'Split series').click();
+
+    // open split series options
+    await driver.find('.test-chart-group-by-column').click();
+
+    // check group-data options
+    assert.deepEqual(
+      await driver.findAll('.test-select-menu li', e => e.getText()),
+      ['Pick a column', 'Name', 'B']
+    );
+
+    // send ESCAPE to close menu
+    await driver.sendKeys(Key.ESCAPE);
+
+    // open x axis options
+    await driver.find('.test-chart-x-axis').click();
+
+    // check x axis options
+    assert.deepEqual(
+      await driver.findAll('.test-select-menu li', e => e.getText()),
+      ['Name', 'B']
+    );
+
+    // send ESCAPE to close menu
+    await driver.sendKeys(Key.ESCAPE);
+
+    // undo
+    await gu.undo(1);
+  });
+
+  it('should get series name right when grouped column has \'\' values', async function() {
+    // remove series 'value'
+    await removeYAxis('value');
+
+    // add a row with person left as blank
+    const {retValues} = await api.applyUserActions(doc.id, [
+      ['AddRecord', 'ChartData', 7, {largeValue: 44}]
+    ]);
+    await setSplitSeries('person');
+
+    // check that series name is correct
+    const data = (await getChartData()).data;
+    assert.deepEqual(data.map(d => d.name), ['[Blank]', 'Alice', 'Bob']);
+
+    // remove row
+    await api.applyUserActions(doc.id, [
+      ['RemoveRecord', 'ChartData', retValues[0]]
+    ]);
+
+    // undo
+    await gu.undo(2);
+  });
+
+  it('should disabled split series option for pie charts', async function() {
+
+    // start with line chart type
+    await selectChartType('Line Chart');
+
+    // check the split series option is present
+    assert.equal(await driver.findContent('label', /Split series/).isPresent(), true);
+    assert.equal(await driver.find('.test-chart-group-by-column').isPresent(), true);
+
+    // select 'person' as the split series column
+    await setSplitSeries('person');
+
+    // check split series option
+    assert.equal(await driver.findContent('label', /Split series/).isPresent(), true);
+    assert.equal(await driver.find('.test-chart-group-by-column').isPresent(), true);
+
+    // check axis
+    await checkAxisConfig({
+      groupingByColumn: 'person',
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value'],
+    });
+
+    // select pie chart type
+    await selectChartType('Pie Chart');
+
+    // check that the split series option is not present
+    assert.equal(await driver.findContent('label', /Split series/).isPresent(), false);
+    assert.equal(await driver.find('.test-chart-group-by-column').isPresent(), false);
+
+    // check axis
+    await checkAxisConfig({
+      groupingByColumn: false,
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value'],
+    });
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Pie Chart');
+
+    // undo
+    await gu.undo(2);
+    await checkAxisConfig({
+      groupingByColumn: false,
+      xaxis: 'label',
+      yaxis: ['largeValue', 'value'],
+    });
+    assert.equal(await driver.find('.test-chart-type').getText(), 'Line Chart');
+  });
+
+  it('should render dates properly on X-axis', async function() {
+    await gu.getSection('ChartData').find('.viewsection_title').click();
+
+    // Add a new first column.
+    await gu.getCell({col: 0, rowNum: 1}).click();
+    // driver.sendKeys() doesn't support key combinations, but elem.sendKeys() does.
+    await driver.find('body').sendKeys(Key.chord(Key.ALT, Key.SHIFT, '='));
+    await gu.waitForServer();
+    await driver.find('.test-column-title-label').sendKeys('MyDate', Key.ENTER);
+    await gu.waitForServer();
+
+    // Convert it to Date
+    await gu.toggleSidePanel('right', 'open');
+    await driver.find('.test-right-tab-field').click();
+    await gu.setType(/Date/);
+    await gu.waitForServer();
+
+    // Enter some values.
+    await gu.enterGridRows({col: 0, rowNum: 1}, [
+      ["2018-01-15"], ["2018-01-31"], ["2018-02-14"], ["2018-03-04"], ["2018-03-14"], ["2018-03-26"]
+    ]);
+
+    // Open the view config pane for the Chart section.
+    await gu.getSection('ChartData chart').find('.viewsection_title').click();
+    await driver.find('.test-right-tab-pagewidget').click();
+
+    // select MyDate for x axis
+    await selectXAxis('MyDate');
+
+    const chartDom = await driver.find('.test-chart-container');
+    const {data, layout} = await getChartData(chartDom);
+    // This check helps understand Plotly's actual interpretation of the dates. E.g. if the range
+    // endpoints are like '2018-03-25 20:00', plotly is misinterpreting the timezone.
+    assert.deepEqual(layout.xaxis.range, ['2018-01-15', '2018-03-26']);
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data[0].name, 'largeValue');
+    assert.deepEqual(data[0].x, [
+      "2018-01-15T00:00:00.000Z", "2018-01-31T00:00:00.000Z", "2018-02-14T00:00:00.000Z",
+      "2018-03-04T00:00:00.000Z", "2018-03-14T00:00:00.000Z", "2018-03-26T00:00:00.000Z"
+    ]);
+    assert.deepEqual(data[0].y, [22, 33, 11, 44, 22, 55]);
+    assert.deepEqual(data[1].type, 'scatter');
+    assert.deepEqual(data[1].name, 'value');
+    assert.deepEqual(data[0].x, [
+      "2018-01-15T00:00:00.000Z", "2018-01-31T00:00:00.000Z", "2018-02-14T00:00:00.000Z",
+      "2018-03-04T00:00:00.000Z", "2018-03-14T00:00:00.000Z", "2018-03-26T00:00:00.000Z"
+    ]);
+    assert.deepEqual(data[1].y, [16, 2, 3, 4, 5, 6]);
+  });
+
+  it('should support error bars', async function() {
+    // We start with a line chart with MyDate on X-axis, and two series: largeValue and value.
+    await selectChartType('Line Chart');
+    await checkAxisConfig({xaxis: 'MyDate', yaxis: ['largeValue', 'value']});
+
+    // Symmetric error bars should leave only the largeValue series, with 'value' for error bars.
+    await driver.find('.test-chart-error-bars .test-select-open').click();
+    await driver.findContent('.test-select-menu li', /Symmetric/).click();
+    await gu.waitForServer();
+
+    const chartDom = await driver.find('.test-chart-container');
+    let data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'scatter');
+    assert.deepEqual(data[0].name, 'largeValue');
+    assert.deepEqual(data[0].y, [22, 33, 11, 44, 22, 55]);
+    assert.deepEqual((data[0].error_y as any).array, [16, 2, 3, 4, 5, 6]);
+    assert.deepEqual(data[0].error_y!.symmetric, true);
+    assert.lengthOf(data, 1);
+
+    // Using separate error bars for above+below will leave just the "above" error bars.
+    await driver.find('.test-chart-error-bars .test-select-open').click();
+    await driver.findContent('.test-select-menu li', /Above.*Below/).click();
+    await gu.waitForServer();
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].y, [22, 33, 11, 44, 22, 55]);
+    assert.deepEqual((data[0].error_y as any).array, [16, 2, 3, 4, 5, 6]);
+    assert.deepEqual((data[0].error_y as any).arrayminus, null);
+    assert.deepEqual(data[0].error_y!.symmetric, false);
+    assert.lengthOf(data, 1);
+
+    // If we add another line, it'll be used for "below" error bars.
+    await addYAxis('label');
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].y, [22, 33, 11, 44, 22, 55]);
+    assert.deepEqual((data[0].error_y as any).array, [16, 2, 3, 4, 5, 6]);
+    assert.deepEqual((data[0].error_y as any).arrayminus, [61, 5, 4, 3, 2, 1]);
+    assert.deepEqual(data[0].error_y!.symmetric, false);
+    assert.lengthOf(data, 1);
+
+    // Should work also for bar charts
+    await selectChartType('Bar Chart');
+    data = (await getChartData(chartDom)).data;
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].y, [22, 33, 11, 44, 22, 55]);
+    assert.deepEqual((data[0].error_y as any).array, [16, 2, 3, 4, 5, 6]);
+    assert.deepEqual((data[0].error_y as any).arrayminus, [61, 5, 4, 3, 2, 1]);
+    assert.deepEqual(data[0].error_y!.symmetric, false);
+    assert.lengthOf(data, 1);
+    await gu.undo(1);
+
+
+    await gu.undo(3);
+  });
+
+  it('should fetch data for tables not yet loaded', async function() {
+    // Create a Page that only has a Chart, no other sections.
+    await gu.addNewPage(/Chart/, /ChartData/);
+
+    let chartDom = await driver.findWait('.test-chart-container', 1000);
+    assert.equal(await chartDom.isDisplayed(), true);
+    let data = (await getChartData(chartDom)).data;
+    assert.lengthOf(data, 1);
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].y, [ 61, 5, 4, 3, 2, 1 ]);
+
+    // Reload the page and test that the chart loaded.
+    await driver.navigate().refresh();
+    await gu.waitForDocToLoad();
+
+    await driver.sleep(1000);
+    chartDom = await driver.findWait('.test-chart-container', 1000);
+    assert.equal(await chartDom.isDisplayed(), true);
+    data = (await getChartData(chartDom)).data;
+    assert.lengthOf(data, 1);
+    assert.deepEqual(data[0].type, 'bar');
+    assert.deepEqual(data[0].y, [ 61, 5, 4, 3, 2, 1 ]);
+  });
+
+  it('should resize chart when side panels open or close', async function() {
+    // Open a document with some chart data.
+    const session = await gu.session().teamSite.login();
+    doc = await session.tempDoc(cleanup, 'ChartData.grist');
+    await gu.toggleSidePanel('right', 'close');
+
+    // Add a chart section.
+    await gu.addNewSection(/Chart/, /ChartData/);
+    const chart = await driver.findWait('.viewsection_content .svg-container', 1000);
+    const initialRect = await chart.getRect();
+    // We expect the left panel open initially.
+    assert.equal(await gu.isSidePanelOpen('left'), true);
+
+    // Open the RightPanel, check that chart's width was reduced.
+    await gu.toggleSidePanel('right', 'open');
+    await driver.wait(async () => (await chart.getRect()).width < initialRect.width, 1000);
+
+    // Close the panel and check the chart went back to initial size.
+    await gu.toggleSidePanel('right', 'close');
+    await driver.wait(async () => (await chart.getRect()).width === initialRect.width, 1000);
+    assert.deepEqual(await chart.getRect(), initialRect);
+
+    // Close the left panel, and check that chart width was increased.
+    await gu.toggleSidePanel('left', 'close');
+    await driver.wait(async () => (await chart.getRect()).width > initialRect.width, 1000);
+
+    // Reopen the left panel and check the chart went back to initial size.
+    await gu.toggleSidePanel('left', 'open');
+    await driver.wait(async () => (await chart.getRect()).width === initialRect.width, 1000);
+    assert.deepEqual(await chart.getRect(), initialRect);
+  });
+
+  // Tests a bug where js errors would be thrown when fewer than 2 series were visible
+  // and any chart settings were changed.
+  it('should not throw errors when no y-axis are set', async function() {
+    // Open the RightPanel and hide both series.
+    await gu.toggleSidePanel('right', 'open');
+    await removeYAxis('value');
+
+    // Invert the y-axis. (This is meant to trigger js errors if the bug is present)
+    await driver.findContent('label', /Invert Y-axis/).find('input').click();
+    await gu.waitForServer();
+
+    // Group by the first column. (This is meant to trigger js errors if the bug is present)
+    await setSplitSeries('value');
+
+    // Disable groupby column
+    await setSplitSeries(false);
+
+    // Revert changes.
+    await gu.undo(3);
+  });
+
+  // Tests a bug where hitting enter would try to edit a non-existent cell for summary charts.
+  it('should not throw errors when pressing enter on summary charts', async function() {
+    // Click the section and press 'Enter'.
+    await gu.getSection('ChartData chart').click();
+    await driver.sendKeys(Key.ENTER);
+    await gu.checkForErrors();
+  });
+
+  it('should not throw errors when switching to a chart page', async function() {
+    await gu.getPageItem('People').click();
+    await gu.waitForServer();
+    await gu.getPageItem('ChartData').click();
+    await gu.waitForServer();
+    const chartDom = await gu.getSection('ChartData chart').find('.test-chart-container');
+    assert.equal(await chartDom.isDisplayed(), true);
+    await gu.checkForErrors();
+  });
+
+  it('should not throw errors when summarizing or un-summarizing underlying table', async function() {
+    // activate the chart widget
+    await gu.getSection('ChartData chart').click();
+
+    // open widget option
+    await gu.openSectionMenu('viewLayout');
+    await driver.findContent('.grist-floating-menu li', 'Widget options').click();
+
+    // open the page widget picker
+    await driver.findContent('.test-right-panel button', 'Change Widget').click();
+
+    // click the summarize button
+    await driver.findContent('.test-wselect-table', 'ChartData').find('.test-wselect-pivot').click();
+
+    // click save
+    await driver.find('.test-wselect-addBtn').click();
+
+    // wait for server
+    await gu.waitForServer();
+
+    // wait for chart to be changed
+    await gu.waitToPass(async () => {
+      assert.equal(
+        await gu.getActiveSectionTitle(),
+        'CHARTDATA [Totals] Chart'
+      );
+    });
+
+    // check for error
+    await gu.checkForErrors();
+
+    // undo 1
+    await gu.undo(1);
+  });
+
+  it('should sort x-axis values', async function() {
+    // Import a small table of numbers to test this.
+    await gu.importFileDialog('uploads/ChartData-Sort_Test.csv');
+
+    await driver.find('.test-modal-confirm').click();
+    await gu.waitForServer();
+
+    // Add a chart of this data, and configure it first to just show X and Y1, Y2 series.
+    await gu.addNewSection(/Chart/, /ChartData-Sort_Test/);
+    await gu.toggleSidePanel('right', 'open');
+    await selectChartType('Line Chart');
+
+    // Show series X, Y1, Y2, grouped by Group.
+    await selectXAxis('X');
+    await setSplitSeries('Group');
+    await addYAxis('Y1');
+    await addYAxis('Y2');
+
+    const chartDom = await driver.findWait('.test-chart-container', 1000);
+    let {data} = await getChartData(chartDom);
+    assert.lengthOf(data, 4);
+    assert.deepInclude(data[0], {type: 'scatter', name: 'Bar • Y1'});
+    assert.deepInclude(data[1], {type: 'scatter', name: 'Bar • Y2'});
+    assert.deepInclude(data[2], {type: 'scatter', name: 'Foo • Y1'});
+    assert.deepInclude(data[3], {type: 'scatter', name: 'Foo • Y2'});
+    assert.deepEqual(data[0].x, [ 1.5, 2.5, 3.5, 4.5, 5.5 ]);
+    assert.deepEqual(data[0].y, [ 1.5, 1, 3.5, 2.5, 4 ]);
+    assert.deepEqual(data[1].x, [ 1.5, 2.5, 3.5, 4.5, 5.5 ]);
+    assert.deepEqual(data[1].y, [ 6.9, 6, 4.9, 5, 7 ]);
+    assert.deepEqual(data[2].x, [ 1, 2, 3, 4, 5 ]);
+    assert.deepEqual(data[2].y, [ 1.5, 1, 3.5, 2.5, 4 ]);
+    assert.deepEqual(data[3].x, [ 1, 2, 3, 4, 5 ]);
+    assert.deepEqual(data[3].y, [ 6.9, 6, 4.9, 5, 7 ]);
+
+    // Now show series ungrouped.
+    await setSplitSeries(false);
+
+    ({data} = await getChartData(chartDom));
+    assert.lengthOf(data, 2);
+    assert.deepInclude(data[0], {type: 'scatter', name: 'Y1'});
+    assert.deepInclude(data[1], {type: 'scatter', name: 'Y2'});
+    assert.deepEqual(data[0].x, [ 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 ]);
+    assert.deepEqual(data[0].y, [ 1.5, 1.5, 1, 1, 3.5, 3.5, 2.5, 2.5, 4, 4 ]);
+    assert.deepEqual(data[1].x, [ 1, 1.5, 2, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 ]);
+    assert.deepEqual(data[1].y, [ 6.9, 6.9, 6, 6, 4.9, 4.9, 5, 5, 7, 7 ]);
+  });
+
+  it('should not throw when picking the grouping by column for the x-axis', async function() {
+    await checkAxisConfig({xaxis: 'X', yaxis: ['Y1', 'Y2']});
+    await setSplitSeries('Group');
+    await checkAxisConfig({xaxis: 'X', yaxis: ['Y1', 'Y2'], groupingByColumn: 'Group'});
+    await selectXAxis('Group');
+    await checkAxisConfig({xaxis: 'Group', yaxis: ['Y1', 'Y2']});
+    await gu.checkForErrors();
+    await gu.undo(2);
+  });
+});

test/nbrowser/ChoiceList.ts

@@ -137,6 +137,7 @@ async function saveChoiceEntries() {
 describe('ChoiceList', function() {
   this.timeout(20000);
   const cleanup = setupTestSuite();
+  const clipboard = gu.getLockableClipboard();
 
   const WHITE_FILL = 'rgba(255, 255, 255, 1)';
   const UNSET_FILL = WHITE_FILL;
@@ -748,12 +749,17 @@ describe('ChoiceList', function() {
     await gu.sendKeys('Choice 1', Key.ENTER, 'Choice 2', Key.ENTER, 'Choice 3', Key.ENTER);
 
     // Copy all the choices
-    await gu.sendKeys(Key.chord(modKey, 'a'), await gu.copyKey());
+    await gu.sendKeys(await gu.selectAllKey());
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
 
-    // Delete all the choices, then paste them back and verify no data was lost.
+      // Delete all the choices, then paste them back
-    await driver.sendKeys(Key.BACK_SPACE);
+      await driver.sendKeys(Key.BACK_SPACE);
-    assert.deepEqual(await getEditModeChoiceLabels(), []);
+      assert.deepEqual(await getEditModeChoiceLabels(), []);
-    await gu.sendKeys(await gu.pasteKey());
+      await cb.paste();
+    });
+
+    // Verify no data was lost
     assert.deepEqual(await getEditModeChoiceLabels(), ['Choice 1', 'Choice 2', 'Choice 3']);
 
     // In Jenkins, clipboard contents are pasted from the system clipboard, which only copies
@@ -846,14 +852,18 @@ describe('ChoiceList', function() {
 
       // Make sure we can copy and paste without adding new item
       await clickEntry('foo');
-      await gu.sendKeys(await gu.cutKey());
+      await clipboard.lockAndPerform(async (cb) => {
-      await gu.sendKeys(await gu.pasteKey());
+        await cb.cut();
-      await gu.sendKeys(await gu.pasteKey());
+        await cb.paste();
+        await cb.paste();
+      });
       await gu.sendKeys(Key.ENTER);
       await clickEntry('two');
-      await gu.sendKeys(await gu.copyKey());
+      await clipboard.lockAndPerform(async (cb) => {
-      await gu.sendKeys(Key.ARROW_RIGHT);
+        await cb.copy();
-      await gu.sendKeys(await gu.pasteKey());
+        await gu.sendKeys(Key.ARROW_RIGHT);
+        await cb.paste();
+      });
       await gu.sendKeys(Key.ENTER);
       assert.deepEqual(await getEditModeChoiceLabels(), ["foofoo", "three", "twotwo"]);
       await saveChoiceEntries();

test/nbrowser/CopyPaste.ts

@@ -0,0 +1,676 @@
+/**
+ * Test for copy-pasting Grist data.
+ *
+ * TODO Most of the testing for copy-pasting lives in test/nbrowser/CopyPaste.ntest.js.
+ * This file just has some more recent additions to these test.
+ */
+import {arrayRepeat} from 'app/common/gutil';
+import * as _ from 'lodash';
+import {assert, driver, Key, WebElement} from 'mocha-webdriver';
+import * as path from 'path';
+import {serveStatic} from 'test/nbrowser/customUtil';
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('CopyPaste', function() {
+  this.timeout(60000);
+  const cleanup = setupTestSuite();
+  const clipboard = gu.getLockableClipboard();
+  afterEach(() => gu.checkForErrors());
+  gu.bigScreen();
+
+  after(async function() {
+    await driver.executeScript(removeDummyTextArea);
+  });
+
+  it('should allow pasting merged cells', async function() {
+    // Test that we can paste uneven data, i.e. containing merged cells.
+
+    // Serve a static file with a page containing a table with some merged cells.
+    const serving = await serveStatic(path.join(gu.fixturesRoot, "sites/paste"));
+    await driver.get(`${serving.url}/paste.html`);
+
+    // Select everything in our little page.
+    await driver.executeScript(`
+      let range = document.createRange();
+      range.selectNodeContents(document.querySelector('table'));
+      let sel = window.getSelection();
+      sel.removeAllRanges();
+      sel.addRange(range);
+    `);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      try {
+        await cb.copy();
+      } finally {
+        await serving?.shutdown();
+      }
+
+      const session = await gu.session().login();
+      await session.tempNewDoc(cleanup, 'CopyPaste');
+
+      await gu.getCell({col: 'A', rowNum: 1}).click();
+      await gu.waitAppFocus();
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    await gu.checkForErrors();
+    assert.deepEqual(await gu.getVisibleGridCells({rowNums: [1, 2, 3, 4], cols: ['A', 'B']}), [
+      'a', 'b',
+      'c', '',
+      'd', 'e',
+      'f', '',
+    ]);
+  });
+
+  it('should parse pasted numbers', async function() {
+    const session = await gu.session().teamSite.login();
+    await session.tempDoc(cleanup, 'PasteParsing.grist');
+    await driver.executeScript(createDummyTextArea);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '$1',        '1',
+        '(2)',       '-2',
+        '3e4',       '30000',
+        '5,678.901', '5678.901',
+        '23%',       '0.23',
+        '45 678',    '45678',
+
+        // . is a decimal separator in this locale (USA) so this can't be parsed
+        '1.234.567', '1.234.567 INVALID',
+
+        // Doesn't match the default currency of the document, whereas $ above does
+        '€89',       '€89 INVALID',
+      ], true);
+    });
+
+    // Open the side panel for the numeric column.
+    await gu.toggleSidePanel('right', 'open');
+    await driver.find('.test-right-tab-field').click();
+
+    // Switch to currency mode, and check the result.
+    await driver.findContent('.test-numeric-mode .test-select-button', /\$/).click();
+
+    // Same data, just formatted differently
+    await checkGridCells([
+      '$1',        '$1.00',
+      '(2)',       '-$2.00',
+      '3e4',       '$30,000.00',
+      '5,678.901', '$5,678.90',
+      '23%',       '$0.23',
+      '45 678',    '$45,678.00',
+      '1.234.567', '1.234.567 INVALID',
+      '€89',       '€89 INVALID',
+    ]);
+
+    // Check that currency is set to 'Default currency' by default (where the default is local currency).
+    assert.equal(await driver.find('.test-currency-autocomplete input').value(), 'Default currency (USD)');
+
+    // Change column setting for currency to Euros
+    await driver.findWait('.test-currency-autocomplete', 500).click();
+    await driver.sendKeys("eur", Key.ENTER);
+    await gu.waitForServer();
+
+    // Same data, just formatted differently
+    await checkGridCells([
+      '$1',        '€1.00',
+      '(2)',       '-€2.00',
+      '3e4',       '€30,000.00',
+      '5,678.901', '€5,678.90',
+      '23%',       '€0.23',
+      '45 678',    '€45,678.00',
+      '1.234.567', '1.234.567 INVALID',
+      '€89',       '€89 INVALID',
+    ]);
+
+    // Copy the numbers column into itself.
+    // Values which were already parsed remain parsed since it copies the underlying numbers.
+    await clipboard.lockAndPerform(async (cb) => {
+      await copy(cb, 'Parsed');
+    });
+    await checkGridCells([
+      '$1',        '€1.00',
+      '(2)',       '-€2.00',
+      '3e4',       '€30,000.00',
+      '5,678.901', '€5,678.90',
+      '23%',       '€0.23',
+      '45 678',    '€45,678.00',
+      '1.234.567', '1.234.567 INVALID',
+
+      // This was invalid before, so it was copied as text.
+      // This time it parsed successfully because the currency matches.
+      '€89',       '€89.00',
+    ]);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        // Now we're copying from the text column so everything is parsed again.
+        // $ can no longer be parsed now the currency is euros.
+        '$1',        '$1 INVALID',
+
+        '(2)',       '-€2.00',
+        '3e4',       '€30,000.00',
+        '5,678.901', '€5,678.90',
+        '23%',       '€0.23',
+        '45 678',    '€45,678.00',
+        '1.234.567', '1.234.567 INVALID',
+        '€89',       '€89.00',
+      ], true);
+    });
+
+    // Change the document locale
+    await gu.openDocumentSettings();
+    await driver.findWait('.test-locale-autocomplete', 500).click();
+    await driver.sendKeys("Germany", Key.ENTER);
+    await gu.waitForServer();
+    await driver.navigate().back();
+
+    // Same data, just formatted differently
+    // Currency sign has moved to the end
+    // Decimal separator is now ','
+    // Digit group separator is now '.'
+    await checkGridCells([
+      '$1',        '$1 INVALID',
+      '(2)',       '-2,00 €',
+      '3e4',       '30.000,00 €',
+      '5,678.901', '5.678,90 €',
+      '23%',       '0,23 €',
+      '45 678',    '45.678,00 €',
+      '1.234.567', '1.234.567 INVALID',
+      '€89',       '89,00 €',
+    ]);
+
+    // Copy the numbers column into itself.
+    // Values which were already parsed don't change since it copies the underlying numbers.
+    await clipboard.lockAndPerform(async (cb) => {
+      await copy(cb, 'Parsed');
+    });
+    await checkGridCells([
+      '$1',        '$1 INVALID',
+      '(2)',       '-2,00 €',
+      '3e4',       '30.000,00 €',
+      '5,678.901', '5.678,90 €',
+      '23%',       '0,23 €',
+      '45 678',    '45.678,00 €',
+
+      // This can be parsed for the first time now that '.'
+      // is seen as a digit group separator
+      '1.234.567', '1.234.567,00 €',
+
+      '€89',       '89,00 €',
+    ]);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '$1',        '$1 INVALID',
+        '(2)',       '-2,00 €',
+        '3e4',       '30.000,00 €',
+
+        // Now we're copying from the text column so everything is parsed again.
+        // The result in this case is not good:
+        // '.' was simply removed because we don't check where it is
+        // ',' is the decimal separator
+        // So this is parsed as 5.678901
+        // which rounds to 5.68 to two decimal places for the currency format
+        '5,678.901', '5,68 €',
+
+        '23%',       '0,23 €',
+        '45 678',    '45.678,00 €',
+        '1.234.567', '1.234.567,00 €',
+        '€89',       '89,00 €',
+      ], true);
+    });
+  });
+
+  it('should parse pasted dates', async function() {
+    await gu.getPageItem("Dates").click();
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '01-02-03',   '01-02-2003',
+        '01 02 2003', '01-02-2003',
+        '1/02/03',    '01-02-2003',
+        '01/2/03',    '01-02-2003',
+        '1/2/03',     '01-02-2003',
+        '1/2/3',      '1/2/3 INVALID',
+        '20/10/03',   '20-10-2003',
+        '10/20/03',   '10/20/03 INVALID',
+      ]);
+    });
+
+    await gu.getCell({col: 'Parsed', rowNum: 1}).click();
+    assert.equal(await gu.getDateFormat(), "DD-MM-YYYY");
+    await gu.setDateFormat("MM-DD-YYYY");
+
+    // Same data, just formatted differently
+    await checkGridCells([
+      '01-02-03',   '02-01-2003',
+      '01 02 2003', '02-01-2003',
+      '1/02/03',    '02-01-2003',
+      '01/2/03',    '02-01-2003',
+      '1/2/03',     '02-01-2003',
+      '1/2/3',      '1/2/3 INVALID',
+      '20/10/03',   '10-20-2003',
+      '10/20/03',   '10/20/03 INVALID',
+    ]);
+
+    // Copy the parsed column into itself.
+    // Values which were already parsed don't change since it copies the underlying values.
+    await clipboard.lockAndPerform(async (cb) => {
+      await copy(cb, 'Parsed');
+    });
+
+    await checkGridCells([
+      '01-02-03',   '02-01-2003',
+      '01 02 2003', '02-01-2003',
+      '1/02/03',    '02-01-2003',
+      '01/2/03',    '02-01-2003',
+      '1/2/03',     '02-01-2003',
+      '1/2/3',      '1/2/3 INVALID',
+      '20/10/03',   '10-20-2003',
+      '10/20/03',   '10-20-2003',  // can be parsed now
+    ]);
+
+    // Copy from the text column again, things get re-parsed
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '01-02-03',   '01-02-2003',
+        '01 02 2003', '01-02-2003',
+        '1/02/03',    '01-02-2003',
+        '01/2/03',    '01-02-2003',
+        '1/2/03',     '01-02-2003',
+        '1/2/3',      '1/2/3 INVALID',
+        '20/10/03',   '20/10/03 INVALID',  // newly invalid
+        '10/20/03',   '10-20-2003',
+      ]);
+    });
+  });
+
+  // Note that these tests which reference other tables
+  // assume that the previous tests have run.
+  it('should parse pasted references', async function() {
+    await gu.getPageItem("References").click();
+    await gu.getCell({col: 'Parsed', rowNum: 1}).click();
+    assert.equal(await gu.getRefTable(), "Dates");
+    assert.equal(await gu.getRefShowColumn(), "Text");
+
+    // Initially the References.Parsed column is displaying Dates.Text
+    // No date parsing happens, we just see which strings exist in that column
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '20/10/03', '20/10/03',
+        '10/20/03', '10/20/03',
+        '1/2/3',    '1/2/3',
+        'foo',      'foo INVALID',
+        '3',        '3 INVALID',
+        '-2',       '-2 INVALID',
+        '$1',       '$1 INVALID',
+        '€89',      '€89 INVALID',
+      ], true);
+    });
+
+    await gu.setRefShowColumn("Parsed");
+
+    // // Same data, just formatted differently
+    await checkGridCells([
+      // In the Parsed column, only the second value was parsed as an actual date
+      // The others look invalid in the Dates table, but here they're valid references
+      '20/10/03', '20/10/03',
+      '10/20/03', '10-20-2003',
+      '1/2/3',    '1/2/3',
+
+      'foo',      'foo INVALID',
+      '3',        '3 INVALID',
+      '-2',       '-2 INVALID',
+      '$1',       '$1 INVALID',
+      '€89',      '€89 INVALID',
+    ]);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '20/10/03', '20/10/03',
+        '10/20/03', '10-20-2003',
+        '1/2/3',    '1/2/3',
+        'foo',      'foo INVALID',
+        '3',        `3 INVALID`,
+        '-2',       `-2 INVALID`,
+        '$1',       `$1 INVALID`,
+        '€89',      '€89 INVALID',
+      ]);
+    });
+
+    await gu.setRefShowColumn("Row ID");
+
+    // Same data, just formatted differently
+    await checkGridCells([
+      '20/10/03', 'Dates[5]',
+      '10/20/03', 'Dates[6]',
+      '1/2/3',    'Dates[4]',
+      'foo',      'foo INVALID',
+      '3',        `3 INVALID`,
+      '-2',       `-2 INVALID`,
+      '$1',       `$1 INVALID`,
+      '€89',      '€89 INVALID',
+    ]);
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '20/10/03', '20/10/03 INVALID',
+        '10/20/03', '10/20/03 INVALID',
+        '1/2/3',    '1/2/3 INVALID',
+        'foo',      'foo INVALID',
+        '3',        'Dates[3]',  // 3 is the only valid Row ID
+        '-2',       '-2 INVALID',
+        '$1',       '$1 INVALID',
+        '€89',      '€89 INVALID',
+      ]);
+    });
+
+    await gu.setRefTable("Numbers");
+
+    // These checks run with References.Parsed as both a Reference and Reference List column.
+    async function checkRefsToNumbers() {
+      await gu.setRefShowColumn("Row ID");
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '20/10/03', '20/10/03 INVALID',
+          '10/20/03', '10/20/03 INVALID',
+          '1/2/3',    '1/2/3 INVALID',
+          'foo',      'foo INVALID',
+          '3',        'Numbers[3]',
+          '-2',       '-2 INVALID',
+          '$1',       '$1 INVALID',
+          '€89',      '€89 INVALID',
+        ], true);
+      });
+
+      await gu.setRefShowColumn("Text");
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '20/10/03', '20/10/03 INVALID',
+          '10/20/03', '10/20/03 INVALID',
+          '1/2/3',    '1/2/3 INVALID',
+          'foo',      'foo INVALID',
+          '3',        '3 INVALID',
+          '-2',       '-2 INVALID',
+          // These are the only strings that appear in Numbers.Text verbatim
+          '$1',       '$1',
+          '€89',      '€89',
+        ]);
+      });
+
+      await gu.setRefShowColumn("Parsed");
+
+      // Same data, just formatted differently
+      await checkGridCells([
+        '20/10/03', '20/10/03 INVALID',
+        '10/20/03', '10/20/03 INVALID',
+        '1/2/3',    '1/2/3 INVALID',
+        'foo',      'foo INVALID',
+        '3',        '3 INVALID',
+        '-2',       '-2 INVALID',
+        '$1',       '$1',
+        '€89',      '89,00 €',
+      ]);
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '20/10/03', '20/10/03 INVALID',
+          '10/20/03', '10/20/03 INVALID',
+          '1/2/3',    '1/2/3 INVALID',
+          'foo',      'foo INVALID',
+          '3',        '3 INVALID',  // parsed, but not a valid reference
+          '-2',       '-2,00 €',
+          '$1',       '$1',  // invalid in Numbers.parsed, but a valid reference
+          '€89',      '89,00 €',
+        ]);
+      });
+    }
+
+    await checkRefsToNumbers();
+
+    // Copy the Parsed column into the same column in a forked document.
+    // Because it's a different document, it uses the display values instead of the raw values (row IDs)
+    // to avoid referencing the wrong rows.
+    await clipboard.lockAndPerform(async (cb) => {
+      await copy(cb, 'Parsed');
+      await driver.get(await driver.getCurrentUrl() + "/m/fork");
+      await gu.waitForDocToLoad();
+      await driver.executeScript(createDummyTextArea);
+      await gu.setRefShowColumn("Text");
+      await paste(cb);
+    });
+    await checkGridCells([
+      '20/10/03', '20/10/03 INVALID',
+      '10/20/03', '10/20/03 INVALID',
+      '1/2/3',    '1/2/3 INVALID',
+      'foo',      'foo INVALID',
+      '3',        '3 INVALID',
+      '-2',       '-2,00 € INVALID',
+      '$1',       '$1',
+      '€89',      '89,00 € INVALID',
+    ]);
+
+    // Test the main copies with the Numbers table data not loaded in the browser
+    // so the lookups get done in the data engine.
+    await checkRefsToNumbers();
+
+    // Now test that pasting the same values into a Reference List column
+    // produces the same result (reflists containing a single reference)
+    await gu.setType(/Reference List/);
+    await gu.applyTypeTransform();
+    await gu.waitForServer();
+
+    // Clear the Parsed column. Make sure we don't edit the column header.
+    await gu.getCell({col: "Parsed", rowNum: 1}).click();
+    await gu.getColumnHeader({col: "Parsed"}).click();
+    await gu.sendKeys(Key.BACK_SPACE);
+    await gu.waitForServer();
+
+    await checkRefsToNumbers();
+  });
+
+  it('should parse pasted reference lists containing multiple values', async function() {
+    async function checkMultiRefs() {
+      await gu.setRefShowColumn("Row ID");
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '"(2)",$1',     '"(2)",$1 INVALID',
+          '$1,(2),22',    '$1,(2),22 INVALID',
+          '["$1",-2]',    '["$1",-2] INVALID',
+          '1,-2',         '1,-2 INVALID',
+          '3,5',          'Numbers[3]\nNumbers[5]',  // only valid row IDs
+          '-2,30000',     '-2,30000 INVALID',
+          '7,0',          '7,0 INVALID',  // 0 is not a valid row ID
+          '',             '',
+        ]);
+      });
+
+      await gu.setRefShowColumn("Text");
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '"(2)",$1',     '(2)\n$1',  // only verbatim text
+          '$1,(2),22',    '$1,(2),22 INVALID',  // 22 is invalid so whole thing fails
+          '["$1",-2]',    '["$1",-2] INVALID',  // -2 is invalid because this is text, not parsed
+          '1,-2',         '1,-2 INVALID',
+          '3,5',          '3,5 INVALID',
+          '-2,30000',     '-2,30000 INVALID',
+          '7,0',          '7,0 INVALID',
+          '',             '',
+        ]);
+      });
+
+      await gu.setRefShowColumn("Parsed");
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await copyAndCheck(cb, [
+          '"(2)",$1',     '-2,00 €\n$1',
+          '$1,(2),22',    '$1,(2),22 INVALID',
+          '["$1",-2]',    '$1\n-2,00 €',
+          '1,-2',         '1,-2 INVALID',
+          '3,5',          '3,5 INVALID',
+          '-2,30000',     '-2,00 €\n30.000,00 €',
+          '7,0',          '7,0 INVALID',
+          '',             '',
+        ], true);
+      });
+    }
+
+    await gu.getPageItem("Multi-References").click();
+    await gu.waitForServer();
+    await gu.getCell({col: 'Parsed', rowNum: 1}).click();
+
+    await checkMultiRefs();
+
+    // Load the Numbers table data in the browser and check again
+    await gu.getPageItem("Numbers").click();
+    await gu.getPageItem("Multi-References").click();
+    await gu.waitForServer();
+    await checkMultiRefs();
+  });
+
+  it('should parse pasted choice lists', async function() {
+    await gu.getPageItem("ChoiceLists").click();
+    await gu.waitForServer();
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '',                            '',
+        'a',                           'a',
+
+        // On the left, \n in text affects parsing and separates choices
+        // On the right, \n is how choices are separated in .getText()
+        // So the newlines on the two sides match, but also "e,f" -> "e\nf"
+        'a b\nc d\ne,f',               'a b\nc d\ne\nf',
+
+        // CSVs
+        'a,b  ',                       'a\nb',
+        '  "a  ", b,"a,b  "  ',        'a\nb\na,b',
+
+        // JSON. Empty strings and null are removed
+        ' ["a","b","a,b", null] ',     'a\nb\na,b',
+
+        // Nested JSON is formatted as JSON or CSV depending on nesting level
+        '["a","b",["a,b"], [["a,b"]], [["a", "b"], "c", "d"], "", "  "]',
+        'a\nb\n"a,b"\n[["a,b"]]\n[["a", "b"], "c", "d"]',
+
+        '[]',                          '',
+      ], true);
+    });
+  });
+
+  it('should parse pasted datetimes', async function() {
+    await gu.getPageItem("DateTimes").click();
+    await gu.waitForServer();
+
+    await clipboard.lockAndPerform(async (cb) => {
+      await copyAndCheck(cb, [
+        '2021-11-12 22:57:17+03:00', '12-11-2021 21:57 SAST',  // note the 1-hour difference
+        '2021-11-12 22:57:17+02:00', '12-11-2021 22:57 SAST',
+        '12-11-2021 22:57:17 SAST',  '12-11-2021 22:57 SAST',
+        '12-11-2021 22:57:17',       '12-11-2021 22:57 SAST',
+        '12-11-2021 22:57:17 UTC',   '13-11-2021 00:57 SAST',  // note the 2-hour difference
+        '12-11-2021 22:57:17 Z',     '13-11-2021 00:57 SAST',  // note the 2-hour difference
+        // EST doesn't match the current timezone so it's rejected
+        '12-11-2021 22:57:17 EST',   '12-11-2021 22:57:17 EST INVALID',
+        // Date without time is allowed
+        '12-11-2021',                '12-11-2021 00:00 SAST',
+      ]);
+    });
+  });
+});
+
+
+// mapper for getVisibleGridCells to get both text and whether the cell is invalid (pink).
+// Invalid cells mean text that was not parsed to the column type.
+async function mapper(el: WebElement) {
+  let text = await el.getText();
+  if (await el.find(".field_clip").matches(".invalid")) {
+    text += " INVALID";
+  }
+  return text;
+}
+
+// Checks that the full grid is equal to the given argument
+// The first column never changes, it's only included for readability of the test
+async function checkGridCells(expected: string[]) {
+  const actual = await gu.getVisibleGridCells({rowNums: _.range(1, 9), cols: ['Text', 'Parsed'], mapper});
+  assert.deepEqual(actual, expected);
+}
+
+// Paste whatever's in the clipboard into the Parsed column
+async function paste(cb: gu.IClipboard) {
+  // Click the first cell rather than the column header so that it doesn't try renaming the column
+  await gu.getCell({col: 'Parsed', rowNum: 1}).click();
+  await cb.paste();
+  await gu.waitForServer();
+  await gu.checkForErrors();
+}
+
+// Copy the contents of fromCol into the Parsed column
+async function copy(cb: gu.IClipboard, fromCol: 'Text' | 'Parsed') {
+  await gu.getColumnHeader({col: fromCol}).click();
+  await cb.copy();
+  await paste(cb);
+}
+
+async function copyAndCheck(cb: gu.IClipboard, expected: string[], extraChecks: boolean = false) {
+  // Copy Text cells into the Parsed column
+  await copy(cb, 'Text');
+  await checkGridCells(expected);
+
+  // Tests some extra features of parsing that don't really depend on the column
+  // type and so don't need to be checked with every call to copyAndCheck
+  if (extraChecks) {
+    // With the text cells still in the clipboard, convert the clipboard from
+    // rich data (cells) to plain text and confirm that it gets parsed the same way.
+    // The cells are still selected, clear them all.
+    await gu.sendKeys(Key.BACK_SPACE);
+    await gu.waitForServer();
+    assert.deepEqual(
+      await gu.getVisibleGridCells({rowNums: _.range(1, 9), cols: ['Parsed']}),
+      arrayRepeat(8, ''),
+    );
+
+    // Paste the text cells to the dummy textarea and copy.
+    await driver.find('#dummyText').click();
+    await gu.waitAppFocus(false);
+    await cb.paste();
+    await gu.sendKeys(await gu.selectAllKey());
+    await cb.copy();
+    await gu.sendKeys(Key.BACK_SPACE);
+
+    // Paste the now plain text and confirm that the resulting data is still the same.
+    await gu.getCell({col: 'Text', rowNum: 1}).click();
+    await gu.waitAppFocus();
+    await paste(cb);
+    await checkGridCells(expected);
+
+    // Check that copying from the Parsed column back into itself doesn't change anything.
+    await copy(cb, 'Parsed');
+    await checkGridCells(expected);
+  }
+}
+
+function createDummyTextArea() {
+  const textarea = document.createElement('textarea');
+  textarea.style.position = "absolute";
+  textarea.style.top = "0";
+  textarea.style.height = "2rem";
+  textarea.style.width = "16rem";
+  textarea.id = 'dummyText';
+  window.document.body.appendChild(textarea);
+}
+
+function removeDummyTextArea() {
+  const textarea = document.getElementById('dummyText');
+  if (textarea) {
+    window.document.body.removeChild(textarea);
+  }
+}

test/nbrowser/CopyPaste2.ntest.js

@@ -0,0 +1,329 @@
+/* global window */
+
+import { assert, driver } from 'mocha-webdriver';
+import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
+
+// Helper that returns the cell text prefixed by "+" if the cell is selected, "-" if not.
+async function selText(cell) {
+  const isSelected = await cell.hasClass('selected');
+  const text = await cell.getAttribute('textContent');
+  return (isSelected ? "+" : "-") + text;
+}
+
+describe('CopyPaste2.ntest', function() {
+  const cleanup = test.setupTestSuite(this);
+  const clipboard = gu.getLockableClipboard();
+
+  before(async function() {
+    await gu.supportOldTimeyTestCode();
+    await gu.useFixtureDoc(cleanup, "CopyPaste2.grist", true);
+  });
+
+  afterEach(function() {
+    return gu.checkForErrors();
+  });
+
+  it('should highlight correct cells after paste', async function() {
+    // After paste, the right cells should be highlighted (there was a bug with it when cursor was
+    // not in the top-left corner of the destination selection).
+
+    // Select a 3x2 rectangle, and check that the data and selection is as we expect.
+    await gu.clickCell({rowNum: 3, col: 0});
+    await gu.sendKeys([$.SHIFT, $.RIGHT], [$.SHIFT, $.DOWN, $.DOWN]);
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '+A3', '+B3', '-C3',  // rowNum 3
+      '+A4', '+B4', '-C4',  // rowNum 4
+      '+A5', '+B5', '-C5',  // rowNum 5
+      '-A6', '-B6', '-C6',  // rowNum 6
+      '-A7', '-B7', '-C7',  // rowNum 7
+    ]);
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+
+      // For destination, select rows 5-6, but with cursor in the bottom-right corner of them.
+      await gu.clickCell({rowNum: 6, col: 1});
+      await gu.sendKeys([$.SHIFT, $.LEFT], [$.SHIFT, $.UP]);
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // The result should have 3 rows selected starting from row 5, col 0.
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 6, col: 1});
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '-A3', '-B3', '-C3',  // rowNum 3
+      '-A4', '-B4', '-C4',  // rowNum 4
+      '+A3', '+B3', '-C5',  // rowNum 5
+      '+A4', '+B4', '-C6',  // rowNum 6
+      '+A5', '+B5', '-C7',  // rowNum 7
+    ]);
+
+    await gu.undo();    // Go back to initial state.
+  });
+
+  it('should allow paste into sorted grids', async function() {
+    // Sort by column A.
+    await gu.clickCell({rowNum: 1, col: 0});
+    await gu.openColumnMenu('A');
+    await $('.grist-floating-menu .test-sort-asc').click();
+    await gu.clickCell({rowNum: 1, col: 0});
+
+    // Check the initial state. Refer to this when trying to understand the results of each step.
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '-A3', '-B3', '-C3',  // rowNum 3
+      '-A4', '-B4', '-C4',  // rowNum 4
+      '-A5', '-B5', '-C5',  // rowNum 5
+      '-A6', '-B6', '-C6',  // rowNum 6
+      '-A7', '-B7', '-C7',  // rowNum 7
+    ]);
+
+    // First test pasting columns B,C: order of rows is not affected.
+    await gu.clickCell({rowNum: 3, col: 1});
+    await gu.sendKeys([$.SHIFT, $.RIGHT], [$.SHIFT, $.DOWN, $.DOWN]);
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 5, col: 1});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // Check values, and also that the selection is in the paste destination.
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 5, col: 1});
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '-A3', '-B3', '-C3',  // rowNum 3
+      '-A4', '-B4', '-C4',  // rowNum 4
+      '-A5', '+B3', '+C3',  // rowNum 5
+      '-A6', '+B4', '+C4',  // rowNum 6
+      '-A7', '+B5', '+C5',  // rowNum 7
+    ]);
+
+    await gu.undo();    // Go back to initial state.
+
+    // Now test pasting columns A,B. First a single row: it jumps but cursor should stay in it.
+    await gu.clickCell({rowNum: 7, col: 0});
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 3, col: 0});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // Check values, and also that the selection is in the paste destination.
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 6, col: 0});
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '-A4', '-B4', '-C4',  // rowNum 3
+      '-A5', '-B5', '-C5',  // rowNum 4
+      '-A6', '-B6', '-C6',  // rowNum 5
+      '-A7', '-B3', '-C3',  // rowNum 6
+      '-A7', '-B7', '-C7',  // rowNum 7
+    ]);
+
+    await gu.undo();    // Go back to initial state.
+
+    // Now multiple rows / columns, including adding records.
+    await gu.clickCell({rowNum: 3, col: 0});
+    await gu.sendKeys([$.SHIFT, $.RIGHT], [$.SHIFT, $.DOWN, $.DOWN]);
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 6, col: 0});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // Cursor should be in the row which used to be row 6 (has C5 in it); selection is lost
+    // because rows are no longer contiguous (and better behavior is not yet implemented).
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 4, col: 0});
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7, 8], cols: [0, 1, 2], mapper: selText}), [
+      '-A3', '-B3', '-C3',  // rowNum 3
+      '-A3', '-B3', '-C6',  // rowNum 4
+      '-A4', '-B4', '-C4',  // rowNum 5
+      '-A4', '-B4', '-C7',  // rowNum 6
+      '-A5', '-B5', '-C5',  // rowNum 7
+      '-A5', '-B5', '-',    // rowNum 8
+    ]);
+
+    await gu.undo();    // Go back to initial state.
+
+    // Now B/C column into A/B column, with a row shift. This happens to keep destination rows
+    // together, so we check that the selection is maintained.
+    await gu.clickCell({rowNum: 3, col: 1});
+    await gu.sendKeys([$.SHIFT, $.RIGHT], [$.SHIFT, $.DOWN]);
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 5, col: 0});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 6, col: 0});
+    assert.deepEqual(await gu.getGridValues({rowNums: [3, 4, 5, 6, 7], cols: [0, 1, 2], mapper: selText}), [
+      '-A3', '-B3', '-C3',  // rowNum 3
+      '-A4', '-B4', '-C4',  // rowNum 4
+      '-A7', '-B7', '-C7',  // rowNum 5
+      '+B3', '+C3', '-C5',  // rowNum 6
+      '+B4', '+C4', '-C6',  // rowNum 7
+    ]);
+
+    await gu.undo();    // Go back to initial state.
+
+    // Undo the sorting.
+    $('.test-section-menu-small-btn-revert').click();
+  });
+
+  it.skip('should copy formatted values to clipboard', async function() {
+    // Formatted values should be copied to the clipboard as the user sees them (particularly for
+    // Dates and Reference columns).
+    //
+    // FIXME: this test currently fails in headless environments, seemingly due to changes to
+    // clipboard behavior in recent versions of chromedriver.
+
+    // Select a 3x2 rectangle, and check that the data and selection is as we expect.
+    await gu.clickCell({rowNum: 3, col: 2});
+    await gu.sendKeys([$.SHIFT, $.RIGHT, $.RIGHT, $.RIGHT, $.RIGHT, $.RIGHT], [$.SHIFT, $.DOWN, $.DOWN]);
+    assert.deepEqual(await gu.getGridValues({rowNums: [1, 2, 3, 4, 5, 6, 7], cols: [2, 3, 4, 5, 6, 7], mapper: selText}), [
+      '-C1', '-17.504',   '-02/29/16', '-2016-02-29 9:30am', '-April 13, 1743',    '-Jefferson',
+      '-C2', '--3.222',   '-03/31/16', '-2016-03-31 9:30am', '-March 16, 1751',    '-Madison',
+      '+C3', '+-4.018',   '+04/30/16', '+2016-04-30 9:30am', '+October 30, 1735',  '+Adams',
+      '+C4', '+1829.324', '+05/31/16', '+2016-05-31 9:30am', '+February 22, 1732', '+Washington',
+      '+C5', '+9402.556', '+06/30/16', '+2016-06-30 9:30am', '+',                   '+',
+      '-C6', '-12.000',   '-07/31/16', '-2016-07-31 9:30am', '-February 22, 1732', '-Washington',
+      '-C7', '-0.001',    '-08/31/16', '-2016-08-31 9:30am', '-April 13, 1743',    '-Jefferson',
+    ]);
+
+    // Paste data as the text into the open editor of top-left cell, and save.
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 1, col: 0});
+      await gu.sendKeys($.ENTER, $.SELECT_ALL);
+      await cb.paste();
+    });
+    await gu.sendKeys($.ENTER);
+    await gu.waitForServer();
+
+    // Note how all values are formatted in the same way as above.
+    assert.deepEqual(await gu.getCell({rowNum: 1, col: 0}).getAttribute('textContent'),
+      'C3\t-4.018\t04/30/16\t2016-04-30 9:30am\tOctober 30, 1735\tAdams\n' +
+      'C4\t1829.324\t05/31/16\t2016-05-31 9:30am\tFebruary 22, 1732\tWashington\n' +
+      'C5\t9402.556\t06/30/16\t2016-06-30 9:30am\t\t');
+    await gu.undo();    // Go back to initial state.
+  });
+
+  it.skip('should copy properly in the presence of special characters', async function() {
+    // If we copy multiple cells (generating text/html to clipboard) and the cells contain special
+    // html characters (such as angle brackets), those should be escaped.
+    //
+    // FIXME: this test currently fails in headless environments, seemingly due to changes to
+    // clipboard behavior in recent versions of chromedriver.
+
+    await gu.clickCell({rowNum: 1, col: 1});
+    await gu.sendKeys($.ENTER, $.SELECT_ALL, "<tag> for", [$.SHIFT, $.ENTER], "you & me;", $.ENTER);
+    await gu.waitForServer();
+
+    // Add a listener that will save the prepared clipboard data, so that we can examine it.
+    await driver.executeScript(function() {
+      window.gristCopyHandler = ev => {
+        window.copiedClipboardData = {};
+        for (let t of ev.clipboardData.types) {
+          window.copiedClipboardData[t] = ev.clipboardData.getData(t);
+        }
+      };
+      window.addEventListener('copy', window.gristCopyHandler);
+    });
+
+    try {
+      // Now copy a multi-cell selection including this cell.
+      await gu.clickCell({rowNum: 1, col: 0});
+      await gu.sendKeys([$.SHIFT, $.RIGHT], [$.SHIFT, $.DOWN]);
+      assert.deepEqual(await gu.getGridValues({rowNums: [1, 2], cols: [0, 1, 2], mapper: selText}), [
+        '+A1', '+<tag> for\nyou & me;', '-C1',
+        '+A2', '+B2',                   '-C2',
+      ]);
+
+      await clipboard.lockAndPerform(async (cb) => {
+        await cb.copy();
+
+        // Firefox and Chrome actually produce slightly different html, so we just check the part that
+        // matters: that angle brackets and ampersand got escaped.
+        assert.include(await driver.executeScript(() => window.copiedClipboardData['text/html']),
+                      '<td>A1</td><td>&lt;tag&gt; for\nyou &amp; me;</td>');
+
+        // Check the contents of text that got copied to the clipboard
+        assert.equal(await driver.executeScript(() => window.copiedClipboardData['text/plain']),
+                    'A1\t"<tag> for\nyou & me;"\n' +
+                    'A2\tB2'
+                    );
+
+        // We can check that we also accept such text correctly by pasting as text inside a cell, and
+        // then copy-pasting from there.
+        await gu.clickCell({rowNum: 3, col: 0});
+        await gu.sendKeys($.ENTER, $.SELECT_ALL);
+        await cb.paste();
+      });
+      await gu.sendKeys($.ENTER);
+      await gu.waitForServer();
+
+      await gu.clickCell({rowNum: 3, col: 0});
+      await gu.sendKeys($.ENTER, $.SELECT_ALL);
+      await clipboard.lockAndPerform(async (cb) => {
+        await cb.copy();
+        await gu.sendKeys($.ESCAPE);
+        await gu.clickCell({rowNum: 4, col: 0});
+        await cb.paste();
+      });
+
+      await gu.waitForServer();
+      assert.deepEqual(await gu.getGridValues({rowNums: [1, 2, 3, 4, 5], cols: [0, 1, 2], mapper: selText}), [
+        '-A1', '-<tag> for\nyou & me;', '-C1',
+        '-A2', '-B2',                   '-C2',
+        '-A1\t"<tag> for\nyou & me;"\nA2\tB2', '-B3', '-C3',
+        '+A1', '+<tag> for\nyou & me;', '-C4',
+        '+A2', '+B2',                   '-C5',
+      ]);
+
+      await gu.undo(3);    // Go back to initial state.
+    } finally {
+      await driver.executeScript(function() {
+        window.removeEventListener('copy', window.gristCopyHandler);
+      });
+    }
+  });
+
+  it('should paste correctly when values contain commas', async function() {
+    // When pasting, split only on tabs, not on commas. (We used to split on both, or guess what
+    // to split on, which resulted in unexpected and unpleasant surprises when a legitimate value
+    // contained a comma.)
+
+    // Create a value with commas.
+    await gu.clickCell({rowNum: 1, col: 0});
+    await gu.sendKeys($.ENTER, $.SELECT_ALL, "this,is,a,test", $.ENTER);
+    await gu.waitForServer();
+
+    // Copy a single value, and paste to another cell.
+    await gu.clickCell({rowNum: 1, col: 0});
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 2, col: 0});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+    assert.deepEqual(await gu.getGridValues({rowNums: [1, 2], cols: [0, 1, 2], mapper: selText}), [
+      '-this,is,a,test', '-B1', '-C1',
+      '-this,is,a,test', '-B2', '-C2',
+    ]);
+
+    // Now copy multiple values, and paste to other cells.
+    await gu.sendKeys([$.SHIFT, $.UP], [$.SHIFT, $.RIGHT]);
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.clickCell({rowNum: 1, col: 1});
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    assert.deepEqual(await gu.getGridValues({rowNums: [1, 2], cols: [0, 1, 2], mapper: selText}), [
+      '-this,is,a,test', '+this,is,a,test', '+B1',
+      '-this,is,a,test', '+this,is,a,test', '+B2',
+    ]);
+
+    await gu.undo(3);    // Go back to initial state.
+  });
+});

test/nbrowser/CopyPasteColumnOptions.ts

@@ -0,0 +1,363 @@
+/**
+ * Test for copy-pasting from a Grist column into a blank column, which should copy the options.
+ */
+import {safeJsonParse} from 'app/common/gutil';
+import {GristObjCode} from 'app/plugin/GristData';
+import {assert} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('CopyPasteColumnOptions', function() {
+  this.timeout(10000);
+  const cleanup = setupTestSuite();
+  const clipboard = gu.getLockableClipboard();
+  afterEach(() => gu.checkForErrors());
+  gu.bigScreen();
+
+  it('should copy column options into blank columns', async function() {
+    const session = await gu.session().login();
+    const doc = await session.tempDoc(cleanup, 'CopyOptions.grist');
+    const api = session.createHomeApi().getDocAPI(doc.id);
+    const data1 = await api.getRows("Table1");
+    const data2 = await api.getRows("Table2");
+
+    assert.deepEqual(data1, {
+      "id": [1],
+      "manualSort": [1],
+      "A": [1041465600],
+      "B": [1044057600],
+      "C": [1],
+      "D": [[GristObjCode.List, 1, 1]],
+      "E": ["01/02/03"],
+      "F": [[GristObjCode.List, "01/02/03"]],
+      "G": ["01/02/03"],
+      "gristHelper_Display": [[GristObjCode.Date, 1041465600]],
+      "gristHelper_Display2": [[GristObjCode.List, [GristObjCode.Date, 1044057600], [GristObjCode.Date, 1044057600]]],
+      "gristHelper_ConditionalRule": [true],
+    });
+
+    // Initially Table2 is completely empty, all the columns are blank and of type Any
+    assert.deepEqual(data2, {
+      "id": [],
+      "manualSort": [],
+      "A": [],
+      "B": [],
+      "C2": [],
+      "D2": [],
+      "E": [],
+      "F": [],
+      "G": [],
+    });
+
+    // Copy all the data from Table1 to Table2, which will copy the column options
+    await gu.getCell({section: 'TABLE1', col: 0, rowNum: 1}).click();
+    await gu.sendKeys(await gu.selectAllKey());
+    await clipboard.lockAndPerform(async (cb) => {
+      await cb.copy();
+      await gu.getCell({section: 'TABLE2', col: 0, rowNum: 1}).click();
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // Now Table2 contains essentially the same data as Table1
+    // Table2 just has slightly different column names to test display formulas,
+    // and conditional formatting is not copied at the moment.
+    data1.C2 = data1.C;
+    data1.D2 = data1.D;
+    delete data1.C;
+    delete data1.D;
+    delete data1.gristHelper_ConditionalRule;
+    // Actual difference: G is a Text column, so its type was guessed as Date and the string was parsed
+    data1.G = [981158400];
+    assert.deepEqual(await api.getRows("Table2"), data1);
+
+    // Second check that the data is the same, and also that it's formatted the same
+    const cols1 = ["A", "B", "C", "D", "E", "F", "G"];
+    const cols2 = ["A", "B", "C2", "D2", "E", "F", "G"];
+    assert.deepEqual(
+      await gu.getVisibleGridCells({cols: cols1, rowNums: [1], section: "TABLE1"}),
+      await gu.getVisibleGridCells({cols: cols2, rowNums: [1], section: "TABLE2"}),
+    );
+
+    // Check that the column options are essentially the same in both tables
+    const cols = await api.getRecords("_grist_Tables_column");
+    const cleanCols = cols.map(
+      ({
+         id,
+         fields: {
+           parentId,
+           colId,
+           type,
+           visibleCol,
+           displayCol,
+           rules,
+           widgetOptions,
+           formula
+         }
+       }) => ({
+        id,
+        parentId,
+        colId,
+        type,
+        visibleCol,
+        displayCol,
+        rules,
+        formula,
+        widgetOptions: safeJsonParse(widgetOptions as string, ""),
+      }));
+
+    assert.deepEqual(cleanCols, [
+      {
+        "id": 1,
+        "parentId": 1,
+        "colId": "manualSort",
+        "type": "ManualSortPos",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": ""
+      }, {
+        "id": 2,
+        "parentId": 1,
+        "colId": "A",
+        "type": "Date",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {
+          "widget": "TextBox",
+          "dateFormat": "MM/DD/YY",
+          "isCustomDateFormat": false,
+          "alignment": "left"
+        }
+      }, {
+        "id": 3,
+        "parentId": 1,
+        "colId": "B",
+        "type": "Date",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {
+          "widget": "TextBox",
+          "dateFormat": "DD/MM/YY",
+          "isCustomDateFormat": true,
+          "alignment": "center"
+        }
+      }, {
+        "id": 4,
+        "parentId": 1,
+        "colId": "C",
+        "type": "Ref:Table1",
+        "visibleCol": 2,
+        "displayCol": 5,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "Reference", "alignment": "left", "fillColor": "#FECC81"}
+      }, {
+        "id": 5,
+        "parentId": 1,
+        "colId": "gristHelper_Display",
+        "type": "Any",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "$C.A",
+        "widgetOptions": ""
+      }, {
+        "id": 6,
+        "parentId": 1,
+        "colId": "D",
+        "type": "RefList:Table1",
+        "visibleCol": 3,
+        "displayCol": 7,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "Reference", "alignment": "left", "rulesOptions": [], "wrap": true}
+      }, {
+        "id": 7,
+        "parentId": 1,
+        "colId": "gristHelper_Display2",
+        "type": "Any",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "$D.B",
+        "widgetOptions": ""
+      }, {
+        "id": 8,
+        "parentId": 1,
+        "colId": "E",
+        "type": "Choice",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "TextBox", "alignment": "left", "choices": ["01/02/03"], "choiceOptions": {}}
+      }, {
+        "id": 9,
+        "parentId": 1,
+        "colId": "F",
+        "type": "ChoiceList",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": [GristObjCode.List, 21],  // Not copied into the new table
+        "formula": "",
+        "widgetOptions": {
+          "widget": "TextBox",
+          "choices": ["01/02/03", "foo"],
+          "choiceOptions": {},
+          "alignment": "left",
+          "rulesOptions": [{"fillColor": "#BC77FC", "textColor": "#000000"}]  // Not copied into the new table
+        }
+      }, {
+        "id": 10,
+        "parentId": 1,
+        "colId": "G",
+        "type": "Text",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "TextBox", "alignment": "left", "rulesOptions": []}
+      },
+
+      /////////////////
+      ///// Table2 starts here. Most of the column options are now the same.
+      /////////////////
+      {
+        "id": 13,
+        "parentId": 2,
+        "colId": "manualSort",
+        "type": "ManualSortPos",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": ""
+      }, {
+        "id": 14,
+        "parentId": 2,
+        "colId": "A",
+        "type": "Date",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {
+          "widget": "TextBox",
+          "dateFormat": "MM/DD/YY",
+          "isCustomDateFormat": false,
+          "alignment": "left"
+        }
+      }, {
+        "id": 15,
+        "parentId": 2,
+        "colId": "B",
+        "type": "Date",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {
+          "widget": "TextBox",
+          "dateFormat": "DD/MM/YY",
+          "isCustomDateFormat": true,
+          "alignment": "center"
+        }
+      }, {
+        "id": 16,
+        "parentId": 2,
+        "colId": "C2",
+        "type": "Ref:Table1",
+        "visibleCol": 2,
+        "displayCol": 22,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "Reference", "alignment": "left", "fillColor": "#FECC81"}
+      }, {
+        "id": 17,
+        "parentId": 2,
+        "colId": "D2",
+        "type": "RefList:Table1",
+        "visibleCol": 3,
+        "displayCol": 23,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "Reference", "alignment": "left", "wrap": true}
+      }, {
+        "id": 18,
+        "parentId": 2,
+        "colId": "E",
+        "type": "Choice",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "TextBox", "alignment": "left", "choices": ["01/02/03"], "choiceOptions": {}}
+      }, {
+        "id": 19,
+        "parentId": 2,
+        "colId": "F",
+        "type": "ChoiceList",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {"widget": "TextBox", "choices": ["01/02/03", "foo"], "choiceOptions": {}, "alignment": "left"}
+      }, {
+        // Actual difference: the original 'G' is a Text column, so in the new column the type was guessed as Date
+        "id": 20,
+        "parentId": 2,
+        "colId": "G",
+        "type": "Date",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "",
+        "widgetOptions": {
+          "timeFormat": "",
+          "isCustomTimeFormat": true,
+          "isCustomDateFormat": true,
+          "dateFormat": "YY/MM/DD"
+        }
+      }, {
+        "id": 21,
+        // This is in Table1, it's here because it was created in the fixture after Table2
+        // No similar column is in Table2 because conditional formatting is not copied
+        "parentId": 1,
+        "colId": "gristHelper_ConditionalRule",
+        "type": "Any",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "True",
+        "widgetOptions": ""
+      }, {
+        "id": 22,
+        "parentId": 2,
+        "colId": "gristHelper_Display",
+        "type": "Any",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "$C2.A",  // Correctly 'renamed' from $C.A
+        "widgetOptions": ""
+      }, {
+        "id": 23,
+        "parentId": 2,
+        "colId": "gristHelper_Display2",
+        "type": "Any",
+        "visibleCol": 0,
+        "displayCol": 0,
+        "rules": null,
+        "formula": "$D2.B",  // Correctly 'renamed' from $D.A
+        "widgetOptions": ""
+      }]
+    );
+  });
+
+});

test/nbrowser/CopyPasteLinked.ts

@@ -0,0 +1,64 @@
+/**
+ * Test for pasting into a linked GridView.
+ *
+ * In particular, when multiple rows are selected in GridView, on switching to a different linked
+ * record, the selection should be cleared, or else paste will misbehave.
+ */
+import {assert, Key, WebElement} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+
+describe('CopyPasteLinked', function() {
+  this.timeout(30000);
+  const cleanup = setupTestSuite();
+  const clipboard = gu.getLockableClipboard();
+
+  it('should clear internal selection when link record changes', async function() {
+    const mainSession = await gu.session().login();
+    await mainSession.tempDoc(cleanup, 'Landlord.grist');
+    await gu.getPageItem(/Current Signers/).click();
+    await gu.waitForServer();
+
+    let cell: WebElement;
+
+    // Select a cell.
+    cell = await gu.getCell({section: 'Tenants', col: 'Tenant', rowNum: 1});
+    await cell.click();
+    assert.equal(await cell.getText(), 'John Malik');
+
+    await clipboard.lockAndPerform(async (cb) => {
+      // Copy the cell's value to the clipboard.
+      await cb.copy();
+
+      // Now select multiple cells.
+      await gu.sendKeys(Key.chord(Key.SHIFT, Key.DOWN), Key.chord(Key.SHIFT, Key.DOWN));
+
+      // Check that 3 cells are indeed selected.
+      assert.deepEqual(await gu.getVisibleGridCells({col: 'Tenant', rowNums: [1, 2, 3, 4],
+        mapper: (el) => el.matches('.selected')}),
+        [true, true, true, false]);
+
+      // Switch to a different Apartments row that drives the filtering in the Tenants section.
+      await gu.getCell({section: 'Apartments', col: 0, rowNum: 2}).click();
+      cell = await gu.getCell({section: 'Tenants', col: 'Tenant', rowNum: 1});
+      await cell.click();
+      assert.equal(await cell.getText(), 'Fred Brown');
+
+      // Paste the copied value. It doesn't work reliably in a test, so try until it works. (The
+      // reasons seems to be that 'body' has focus briefly, rather than Clipboard component.)
+      await gu.waitAppFocus();
+      await cb.paste();
+    });
+    await gu.waitForServer();
+
+    // Check that only one value was copied, and that there are not multiple cells selected.
+    assert.deepEqual(await gu.getVisibleGridCells({col: 'Tenant', rowNums: [1, 2, 3, 4]}),
+      ['John Malik', 'Fred Brown', 'Susan Sharp', 'Owen Sharp']);
+    assert.deepEqual(await gu.getVisibleGridCells({col: 'Tenant', rowNums: [1, 2, 3, 4],
+      mapper: (el) => el.matches('.selected')}),
+      [false, false, false, false]);
+
+    await gu.checkForErrors();
+    await gu.undo();
+  });
+});

test/nbrowser/CustomWidgetsConfig.ts

@@ -194,6 +194,13 @@ describe('CustomWidgetsConfig', function () {
     // Wait for a frame.
     public async waitForFrame() {
       await driver.findWait(`iframe.test-custom-widget-ready`, 1000);
+      await driver.wait(async () => await driver.find('iframe').isDisplayed(), 1000);
+      await widget.waitForPendingRequests();
+    }
+    public async waitForPendingRequests() {
+      await this._inWidgetIframe(async () => {
+        await driver.executeScript('grist.testWaitForPendingRequests();');
+      });
     }
     public async content() {
       return await this._read('body');
@@ -232,13 +239,12 @@ describe('CustomWidgetsConfig', function () {
     }
     // Wait for the onOptions event, and return its value.
     public async onOptions() {
-      const iframe = driver.find('iframe');
+      const text = await this._inWidgetIframe(async () => {
-      await driver.switchTo().frame(iframe);
+        // Wait for options to get filled, initially this div is empty,
-      // Wait for options to get filled, initially this div is empty,
+        // as first message it should get at least null as an options.
-      // as first message it should get at least null as an options.
+        await driver.wait(async () => await driver.find('#onOptions').getText(), 3000);
-      await driver.wait(async () => await driver.find('#onOptions').getText(), 3000);
+        return await driver.find('#onOptions').getText();
-      const text = await driver.find('#onOptions').getText();
+      });
-      await driver.switchTo().defaultContent();
       return JSON.parse(text);
     }
     public async wasConfigureCalled() {
@@ -308,11 +314,15 @@ describe('CustomWidgetsConfig', function () {
     }
 
     private async _read(selector: string) {
+      return this._inWidgetIframe(() => driver.find(selector).getText());
+    }
+
+    private async _inWidgetIframe<T>(callback: () => Promise<T>) {
       const iframe = driver.find('iframe');
       await driver.switchTo().frame(iframe);
-      const text = await driver.find(selector).getText();
+      const retVal = await callback();
       await driver.switchTo().defaultContent();
-      return text;
+      return retVal;
     }
   }
   // Rpc for main widget (Custom Widget).
@@ -337,7 +347,9 @@ describe('CustomWidgetsConfig', function () {
       })
     );
 
+    await widget.waitForFrame();
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
 
     // Get the drop for M2 mappings.
     const mappingsForM2 = () => driver.find(pickerDrop('M2'));
@@ -387,7 +399,9 @@ describe('CustomWidgetsConfig', function () {
       })
     );
 
+    await widget.waitForFrame();
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
 
     // Get the drop for M2 mappings.
     const mappingsForM2 = () => driver.find(pickerDrop('M2'));
@@ -428,6 +442,7 @@ describe('CustomWidgetsConfig', function () {
     await clickOption(COLUMN_WIDGET);
     await widget.waitForFrame();
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
     // Visible columns section should be hidden.
     assert.isFalse(await driver.find('.test-vfc-visible-fields-select-all').isPresent());
     // Record event should be fired.
@@ -444,7 +459,7 @@ describe('CustomWidgetsConfig', function () {
     await toggleDrop(pickerDrop('Column'));
     assert.deepEqual(await getOptions(), ['A']);
     await clickOption('A');
-    await gu.waitForServer();
+    await widget.waitForPendingRequests();
     // Widget should receive mappings
     assert.deepEqual(await widget.onRecordsMappings(), {Column: 'A'});
     await revert();
@@ -462,9 +477,10 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
-    const empty = {M1: null, M2: null, M3: null, M4: null};
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
+    // Mappings should be empty
     assert.isNull(await widget.onRecordsMappings());
     // We should see 4 pickers
     assert.isTrue(await driver.find(pickerLabel('M1')).isPresent());
@@ -481,29 +497,29 @@ describe('CustomWidgetsConfig', function () {
     assert.equal(await driver.find(pickerDrop('M2')).getText(), 'Pick a column');
     assert.equal(await driver.find(pickerDrop('M3')).getText(), 'Pick a column');
     assert.equal(await driver.find(pickerDrop('M4')).getText(), 'Pick a text column');
-    // Mappings should be empty
-    assert.isNull(await widget.onRecordsMappings());
     // Should be able to select column A for all options
     await toggleDrop(pickerDrop('M1'));
     await clickOption('A');
-    await gu.waitForServer();
+    await widget.waitForPendingRequests();
+    const empty = {M1: null, M2: null, M3: null, M4: null};
     assert.deepEqual(await widget.onRecordsMappings(), {... empty, M1: 'A'});
     await toggleDrop(pickerDrop('M2'));
     await clickOption('A');
-    await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {... empty, M1: 'A', M2: 'A'});
     await toggleDrop(pickerDrop('M3'));
     await clickOption('A');
-    await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {... empty, M1: 'A', M2: 'A', M3: 'A'});
     await toggleDrop(pickerDrop('M4'));
     await clickOption('A');
-    await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: 'A', M2: 'A', M3: 'A', M4: 'A'});
     // Single record should also receive update.
     assert.deepEqual(await widget.onRecordMappings(), {M1: 'A', M2: 'A', M3: 'A', M4: 'A'});
     // Undo should revert mappings - there should be only 3 operations to revert to first mapping.
     await gu.undo(3);
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {... empty, M1: 'A'});
     // Add another columns, numeric B and any C.
     await gu.selectSectionByTitle('Table');
@@ -519,6 +535,7 @@ describe('CustomWidgetsConfig', function () {
     assert.deepEqual(await getOptions(), ['A', 'C']);
     await toggleDrop(pickerDrop('M1'));
     await clickOption('B');
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {...empty, M1: 'B'});
     await revert();
   });
@@ -528,7 +545,9 @@ describe('CustomWidgetsConfig', function () {
 
     await toggleWidgetMenu();
     await clickOption(COLUMN_WIDGET);
+    await widget.waitForFrame();
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
 
     // Make sure columns are there to pick.
 
@@ -540,7 +559,6 @@ describe('CustomWidgetsConfig', function () {
     // Pick first column
     await toggleDrop(pickerDrop('Column'));
     await clickOption('A');
-    await gu.waitForServer();
 
     // Now change to a widget without columns
     await toggleWidgetMenu();
@@ -560,7 +578,9 @@ describe('CustomWidgetsConfig', function () {
     // Now go back to the widget with mappings.
     await toggleWidgetMenu();
     await clickOption(COLUMN_WIDGET);
+    await widget.waitForFrame();
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
     assert.equal(await driver.find(pickerDrop('Column')).getText(), 'Pick a column');
     assert.isFalse(await driver.find('.test-vfc-visible-fields-select-all').isPresent());
     assert.isTrue(await driver.find('.test-config-widget-label-for-Column').isPresent());
@@ -580,9 +600,8 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
-    const empty = {M1: [], M2: []};
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
     // Add some columns, numeric B and any C.
     await gu.selectSectionByTitle('Table');
     await gu.addColumn('B');
@@ -590,6 +609,7 @@ describe('CustomWidgetsConfig', function () {
     await gu.enterCell('99');
     await gu.addColumn('C');
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
     // Make sure we have no mappings
     assert.deepEqual(await widget.onRecordsMappings(), null);
     // Map all columns to M1
@@ -600,6 +620,8 @@ describe('CustomWidgetsConfig', function () {
     await clickMenuItem('B');
     await click(pickerAdd('M1'));
     await clickMenuItem('C');
+    await widget.waitForPendingRequests();
+    const empty = {M1: [], M2: []};
     assert.deepEqual(await widget.onRecordsMappings(), {...empty, M1: ['A', 'B', 'C']});
     // Map A and C to M2
     await click(pickerAdd('M2'));
@@ -609,6 +631,7 @@ describe('CustomWidgetsConfig', function () {
     await clickMenuItem('A');
     await click(pickerAdd('M2'));
     await clickMenuItem('C');
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: ['A', 'B', 'C'], M2: ['A', 'C']});
     function dragItem(column: string, item: string) {
       return driver.findContent(`.test-config-widget-map-list-for-${column} .kf_draggable`, item);
@@ -623,6 +646,7 @@ describe('CustomWidgetsConfig', function () {
         .release()
     );
     await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: ['B', 'C', 'A'], M2: ['A', 'C']});
     // Should support removing
     const removeButton = (column: string, item: string) => {
@@ -630,9 +654,11 @@ describe('CustomWidgetsConfig', function () {
     };
     await removeButton('M1', 'B').click();
     await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: ['C', 'A'], M2: ['A', 'C']});
     // Should undo removing
     await gu.undo();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: ['B', 'C', 'A'], M2: ['A', 'C']});
     await removeButton('M1', 'B').click();
     await gu.waitForServer();
@@ -640,6 +666,7 @@ describe('CustomWidgetsConfig', function () {
     await gu.waitForServer();
     await removeButton('M2', 'C').click();
     await gu.waitForServer();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: ['A'], M2: ['A']});
     await revert();
   });
@@ -657,8 +684,8 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
     // Add B=Date, C=DateTime, D=Numeric
     await gu.sendActions([
       ['AddVisibleColumn', 'Table1', 'B', {type: 'Any'}],
@@ -670,6 +697,7 @@ describe('CustomWidgetsConfig', function () {
     ]);
 
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
     // Make sure we have no mappings
     assert.deepEqual(await widget.onRecordsMappings(), null);
     // Now see what we are offered for M1.
@@ -719,8 +747,8 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
     await gu.sendActions([
       ['AddVisibleColumn', 'Table1', 'Any', {type: 'Any'}],
       ['AddVisibleColumn', 'Table1', 'Date', {type: 'Date'}],
@@ -728,6 +756,8 @@ describe('CustomWidgetsConfig', function () {
     ]);
 
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
+
     // Make sure we have no mappings
     assert.deepEqual(await widget.onRecordsMappings(), null);
 
@@ -759,20 +789,22 @@ describe('CustomWidgetsConfig', function () {
       })
     );
 
+    await widget.waitForFrame();
+    await gu.acceptAccessRequest();
+
     const widgetOptions = {
       choices: ['A'],
       choiceOptions: {A: {textColor: 'red'}}
     };
-
     await gu.sendActions([
       ['AddVisibleColumn', 'Table1', 'Choice', {type: 'Choice', widgetOptions: JSON.stringify(widgetOptions)}]
     ]);
-    await gu.acceptAccessRequest();
-    await widget.waitForFrame();
-
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
+
     await toggleDrop(pickerDrop('Choice'));
     await clickOption('Choice');
+    await widget.waitForPendingRequests();
 
     // Clear logs
     await widget.clearLog();
@@ -803,22 +835,25 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
     // Add some columns, to remove later
     await gu.selectSectionByTitle('Table');
     await gu.addColumn('B');
     await gu.addColumn('C');
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
     // Make sure we have no mappings
     assert.deepEqual(await widget.onRecordsMappings(), null);
     // Map B to M1
     await toggleDrop(pickerDrop('M1'));
     await clickOption('B');
+    await widget.waitForPendingRequests();
     // Map all columns to M2
     for (const col of ['A', 'B', 'C']) {
       await click(pickerAdd('M2'));
       await clickMenuItem(col);
+      await widget.waitForPendingRequests();
     }
     assert.deepEqual(await widget.onRecordsMappings(), {M1: 'B', M2: ['A', 'B', 'C']});
     assert.deepEqual(await widget.onRecords(), [
@@ -830,10 +865,12 @@ describe('CustomWidgetsConfig', function () {
       await gu.selectSectionByTitle('Table');
       await gu.openColumnMenu(col, 'Delete column');
       await gu.waitForServer();
+      await widget.waitForPendingRequests();
       await gu.selectSectionByTitle('Widget');
     };
     // Remove B column
     await removeColumn('B');
+    await widget.waitForPendingRequests();
     // Mappings should be updated
     assert.deepEqual(await widget.onRecordsMappings(), {M1: null, M2: ['A', 'C']});
     // Records should not have B column
@@ -848,6 +885,7 @@ describe('CustomWidgetsConfig', function () {
     await gu.selectSectionByTitle('Table');
     await gu.addColumn('B');
     await gu.selectSectionByTitle('Widget');
+    await widget.waitForPendingRequests();
     // Adding the same column should not add it to mappings or records (as this is a new Id)
     assert.deepEqual(await widget.onRecordsMappings(), {M1: null, M2: ['A', 'C']});
     assert.deepEqual(await widget.onRecords(), [
@@ -861,9 +899,11 @@ describe('CustomWidgetsConfig', function () {
     // Make sure it is there to select.
     assert.deepEqual(await getOptions(), ['A', 'C', 'B']);
     await clickOption('B');
+    await widget.waitForPendingRequests();
     await click(pickerAdd('M2'));
     assert.deepEqual(await getMenuOptions(), ['B']); // multiple selection will only show not selected columns
     await clickMenuItem('B');
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), {M1: 'B', M2: ['A', 'C', 'B']});
     assert.deepEqual(await widget.onRecords(), [
       {id: 1, B: null, A: 'A', C: null},
@@ -884,8 +924,9 @@ describe('CustomWidgetsConfig', function () {
         requiredAccess: 'read table',
       })
     );
-    await gu.acceptAccessRequest();
     await widget.waitForFrame();
+    await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
     assert.deepEqual(await widget.onRecordsMappings(), null);
     assert.deepEqual(await widget.onRecords(), [
       {id: 1, A: 'A'},
@@ -910,8 +951,10 @@ describe('CustomWidgetsConfig', function () {
     await gu.setType(/Numeric/);
     await gu.selectSectionByTitle('Widget');
     await driver.find(".test-right-tab-pagewidget").click();
+    await widget.waitForPendingRequests();
     // Drop should be empty,
-    assert.equal(await driver.find(pickerDrop("M1")).getText(), "No text columns in table.");
+    await driver.wait(async () =>
+      await driver.find(pickerDrop("M1")).getText() == "No text columns in table.", 1000);
     assert.isEmpty(await getListItems("M2"));
     // And drop is disabled.
     assert.isTrue(await driver.find(pickerDrop("M1")).matches(".test-config-widget-disabled"));
@@ -1124,25 +1167,30 @@ describe('CustomWidgetsConfig', function () {
     // Select widget without request
     await toggleWidgetMenu();
     await clickOption(NORMAL_WIDGET);
+    await widget.waitForFrame();
     assert.isFalse(await gu.hasAccessPrompt());
     assert.equal(await gu.widgetAccess(), AccessLevel.none);
     assert.equal(await widget.access(), AccessLevel.none);
     // Select widget that requests read access.
     await toggleWidgetMenu();
     await clickOption(READ_WIDGET);
+    await widget.waitForFrame();
     assert.isTrue(await gu.hasAccessPrompt());
     assert.equal(await gu.widgetAccess(), AccessLevel.none);
     assert.equal(await widget.access(), AccessLevel.none);
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
     assert.equal(await gu.widgetAccess(), AccessLevel.read_table);
     assert.equal(await widget.access(), AccessLevel.read_table);
     // Select widget that requests full access.
     await toggleWidgetMenu();
     await clickOption(FULL_WIDGET);
+    await widget.waitForFrame();
     assert.isTrue(await gu.hasAccessPrompt());
     assert.equal(await gu.widgetAccess(), AccessLevel.none);
     assert.equal(await widget.access(), AccessLevel.none);
     await gu.acceptAccessRequest();
+    await widget.waitForPendingRequests();
     assert.equal(await gu.widgetAccess(), AccessLevel.full);
     assert.equal(await widget.access(), AccessLevel.full);
     await gu.undo(5);

test/nbrowser/DetailView.ntest.js

@@ -3,6 +3,7 @@ import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
 
 describe("DetailView.ntest", function () {
   const cleanup = test.setupTestSuite(this);
+  const clipboard = gu.getLockableClipboard();
   gu.bigScreen();
 
   before(async function() {
@@ -110,8 +111,6 @@ describe("DetailView.ntest", function () {
   it('should include an add record row', async function() {
     // Should include an add record row which works in card view and detail view.
     // Check that adding 'Jurassic Park' to the card view add record row adds it as a row.
-    // await gu.selectSectionByTitle("Performances detail");
-    //await gu.sendKeys([$.MOD, $.DOWN]);
     await $('.g_record_detail:nth-child(14) .field_clip').eq(1).wait().click();
     await gu.sendKeys('Jurassic Park', $.ENTER);
     await gu.waitForServer();
@@ -130,11 +129,14 @@ describe("DetailView.ntest", function () {
 
     // Should allow pasting into the add record row.
     await gu.getDetailCell('Actor', 1).click();
-    await gu.sendKeys($.COPY);
+    await clipboard.lockAndPerform(async (cb) => {
-    await $('.detail-add-btn').click();
+      await cb.copy();
-    // Paste '100' into the last field of the row and check that it is added as its own row.
+      await $('.detail-add-btn').click();
-    await gu.getDetailCell('Character', 1).click();
+      await gu.waitForServer();
-    await gu.sendKeys($.PASTE);
+      // Paste '100' into the last field of the row and check that it is added as its own row.
+      await gu.getDetailCell('Character', 1).click();
+      await cb.paste();
+    });
     await gu.waitForServer();
     assert.deepEqual(await gu.getDetailCell('Character', 1).text(), '100');
 

test/nbrowser/DocTutorial.ts

@@ -7,7 +7,6 @@ import {EnvironmentSnapshot} from 'test/server/testUtils';
 
 describe('DocTutorial', function () {
   this.timeout(60000);
-  setupTestSuite();
 
   gu.bigScreen();
 
@@ -196,8 +195,10 @@ describe('DocTutorial', function () {
       const windowWidth: any = await driver.executeScript('return window.innerWidth');
       assert.equal(dims.y + dims.height, windowHeight - 16);
 
-      // Now move it offscreen as low as possible.
+      // Now we'll test moving the window outside the viewport. Selenium throws when
-      await move({y: windowHeight - dims.y - 10});
+      // the mouse exceeds the bounds of the viewport, so we can't test every scenario.
+      // Start by moving the window as low as possible.
+      await move({y: windowHeight - dims.y - 32});
 
       // Make sure it is still visible.
       dims = await driver.find('.test-floating-popup-window').getRect();
@@ -213,71 +214,33 @@ describe('DocTutorial', function () {
       assert.equal(dims.x, windowWidth - 32 * 4);
 
       // Now move it to the left as far as possible.
-      await move({x: -3000});
+      await move({x: -windowWidth + 128});
 
       // Make sure it is still visible.
       dims = await driver.find('.test-floating-popup-window').getRect();
-      assert.isBelow(dims.x, 0);
+      assert.equal(dims.x, 0);
       assert.isAbove(dims.x + dims.width, 30);
 
-      const miniButton = driver.find(".test-floating-popup-minimize-maximize");
-      // Now move it back, but this time manually as the move handle is off screen.
-      await driver.withActions((a) => a
-        .move({origin: miniButton })
-        .press()
-        .move({origin: miniButton, x: Math.abs(dims.x) + 20})
-        .release()
-      );
-
-      // Maximize it (it was minimized as we used the button to move it).
-      await driver.find(".test-floating-popup-minimize-maximize").click();
-
       // Now move it to the top as far as possible.
       // Move it a little right, so that we don't end up on the logo. Driver is clicking logo sometimes.
-      await move({y: -windowHeight, x: 100});
+      await move({y: -windowHeight + 16, x: 100});
 
       // Make sure it is still visible.
       dims = await driver.find('.test-floating-popup-window').getRect();
       assert.equal(dims.y, 16);
-      assert.isAbove(dims.x, 100);
+      assert.equal(dims.x, 100);
-      assert.isBelow(dims.x, windowWidth);
 
       // Move back where it was.
-      let moverNow = await driver.find('.test-floating-popup-move-handle').getRect();
+      const moverNow = await driver.find('.test-floating-popup-move-handle').getRect();
       await move({x: moverInitial.x - moverNow.x});
       // And restore the size by double clicking the resizer.
       await driver.withActions((a) => a.doubleClick(driver.find('.test-floating-popup-resize-handle')));
-
-      // Now test if we can't resize it offscreen.
-      await move({y: 10000});
-      await move({y: -100});
-      // Header is about 100px above the viewport.
-      dims = await driver.find('.test-floating-popup-window').getRect();
-      assert.isBelow(dims.y, windowHeight);
-      assert.isAbove(dims.x, windowHeight - 140);
-
-      // Now resize as far as possible.
-      await resize({y: 10});
-      await resize({y: 300});
-
-      // Make sure it is still visible.
-      dims = await driver.find('.test-floating-popup-window').getRect();
-      assert.isBelow(dims.y, windowHeight - 16);
-
-      // Now move back and resize.
-      moverNow = await driver.find('.test-floating-popup-move-handle').getRect();
-      await move({x: moverInitial.x - moverNow.x, y: moverInitial.y - moverNow.y});
-      await driver.withActions((a) => a.doubleClick(driver.find('.test-floating-popup-resize-handle')));
-
-      dims = await driver.find('.test-floating-popup-window').getRect();
-      assert.equal(dims.height, initialDims.height);
-      assert.equal(dims.y, initialDims.y);
-      assert.equal(dims.x, initialDims.x);
     });
 
     it('is visible on all pages', async function() {
       for (const page of ['access-rules', 'raw', 'code', 'settings']) {
         await driver.find(`.test-tools-${page}`).click();
+        if (['access-rules', 'code'].includes(page)) { await gu.waitForServer(); }
         assert.isTrue(await driver.find('.test-doc-tutorial-popup').isDisplayed());
       }
     });

test/nbrowser/FieldSettings.ntest.js

@@ -8,6 +8,7 @@ import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
 
 describe('FieldSettings.ntest', function() {
   const cleanup = test.setupTestSuite(this);
+  gu.bigScreen();
 
   before(async function() {
     await gu.supportOldTimeyTestCode();

test/nbrowser/FillLinkedRecords.ntest.js

@@ -7,6 +7,7 @@ import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
  */
 describe('FillLinkedRecords.ntest', function() {
   const cleanup = test.setupTestSuite(this);
+  const clipboard = gu.getLockableClipboard();
 
   gu.bigScreen();
 
@@ -130,9 +131,12 @@ describe('FillLinkedRecords.ntest', function() {
 
     // Copy a range of three values, and paste them into the Add-New row.
     await gu.clickCell({col: 2, rowNum: 1});
-    await gu.sendKeys([$.SHIFT, $.DOWN, $.DOWN], $.COPY);
+    await gu.sendKeys([$.SHIFT, $.DOWN, $.DOWN]);
-    await gu.clickCell({col: 2, rowNum: 5});
+    await clipboard.lockAndPerform(async (cb) => {
-    await gu.sendKeys($.PASTE);
+      await cb.copy();
+      await gu.clickCell({col: 2, rowNum: 5});
+      await cb.paste();
+    });
     await gu.waitForServer();
 
     // Verify that three new rows now show up, with Film auto-filled.

test/nbrowser/Pages.ts

@@ -8,7 +8,6 @@ import values = require('lodash/values');
 
 describe('Pages', function() {
   this.timeout(60000);
-  setupTestSuite();
   let doc: DocCreationInfo;
   let api: UserAPI;
   let session: Session;

test/nbrowser/SavePosition.ntest.js

@@ -3,6 +3,7 @@ import { $, gu, test } from 'test/nbrowser/gristUtil-nbrowser';
 
 describe('SavePosition.ntest', function() {
   const cleanup = test.setupTestSuite(this);
+  const clipboard = gu.getLockableClipboard();
 
   before(async function() {
     this.timeout(Math.max(this.timeout(), 20000)); // Long-running test, unfortunately
@@ -96,9 +97,11 @@ describe('SavePosition.ntest', function() {
   it('should paste into saved position', async function() {
     await gu.getCell({col: 1, rowNum: 9, section: 'Country'}).click();
     await gu.actions.selectTabView('City');
-    await gu.sendKeys($.COPY);
+    await clipboard.lockAndPerform(async (cb) => {
-    await gu.actions.selectTabView('Country');
+      await cb.copy();
-    await gu.sendKeys($.PASTE);
+      await gu.actions.selectTabView('Country');
+      await cb.paste();
+    });
     await gu.waitForServer();
     assert.deepEqual(await gu.getVisibleGridCells(1, [8, 9, 10]),
       ['United Arab Emirates', 'Pará', 'Armenia']);

test/nbrowser/ViewLayoutCollapse.ts

@@ -0,0 +1,875 @@
+import * as gu from 'test/nbrowser/gristUtils';
+import {setupTestSuite} from 'test/nbrowser/testUtils';
+import {getCollapsedSection, openCollapsedSectionMenu} from 'test/nbrowser/ViewLayoutUtils';
+import {assert, driver, Key, WebElement, WebElementPromise} from 'mocha-webdriver';
+import {arrayRepeat} from 'app/plugin/gutil';
+import {addStatic, serveSomething} from 'test/server/customUtil';
+
+const GAP = 16;     // Distance between buttons representing collapsed widgets.
+
+describe("ViewLayoutCollapse", function() {
+  this.timeout(40000);
+  const cleanup = setupTestSuite();
+  gu.bigScreen();
+  let session: gu.Session;
+
+  before(async () => {
+    session = await gu.session().login();
+    await session.tempDoc(cleanup, 'Investment Research.grist');
+    await gu.openPage("Overview");
+  });
+
+  it('fix: custom widget should not throw errors when collapsed', async function() {
+    const revert = await gu.begin();
+
+    // Add custom section.
+    await gu.addNewPage('Table', 'Companies');
+    await gu.addNewSection('Custom', 'Companies', { selectBy: 'COMPANIES'});
+
+    // Serve custom widget.
+    const widgetServer = await serveSomething(app => {
+      addStatic(app);
+    });
+    cleanup.addAfterAll(widgetServer.shutdown);
+    await gu.openWidgetPanel();
+    await gu.setWidgetUrl(widgetServer.url + '/probe/index.html');
+    await driver.find('.test-config-widget-access .test-select-open').click();
+    await driver.findContent('.test-select-menu li', 'Full document access').click();
+    await gu.waitForServer();
+
+    // Collapse it.
+    await collapseByMenu('COMPANIES Custom');
+
+    // Change cursor position in the active section.
+    await gu.getCell(2, 4).click();
+
+    // Put custom section in popup.
+    await openCollapsedSection('COMPANIES Custom');
+
+    // Close it by pressing escape.
+    await gu.sendKeys(Key.ESCAPE);
+
+    // Change cursor once again.
+    await gu.getCell(2, 5).click();
+
+    // Make sure we don't have an error (there was a bug here).
+    await gu.checkForErrors();
+
+    await revert();
+  });
+
+  it('fix: should resize other sections correctly when maximized and linked', async function() {
+    const revert = await gu.begin();
+    // If there are two sections linked, but one is collapsed, and user is changing the row
+    // in the popup of the maximized section, the other section should resize correctly.
+
+    // Add two linked tables.
+    await gu.addNewTable('Table1');
+    await gu.addNewSection('Table', 'New Table');
+
+    await gu.toggleSidePanel('right', 'open');
+
+    // Set A in Table2 to be linked in Table1, by ref column
+    await gu.openColumnMenu('A', 'Column Options');
+
+    // Change it to the Ref column of TABLE1
+    await gu.setType('Reference');
+    await gu.setRefTable('Table1');
+    await gu.setRefShowColumn('A');
+
+    // Select it by Table1.
+    await gu.selectBy('TABLE1');
+
+    // Now add 2 records with 'White' and 'Black' in Table1
+    await gu.sendActions([
+      ['BulkAddRecord', 'Table1', arrayRepeat(2, null), { A: ['White', 'Black'] }],
+      // And 30 records in Table2 that are connected to White.
+      ['BulkAddRecord', 'Table2', arrayRepeat(30, null), { A: arrayRepeat(30, 1) }],
+      // And 30 records in Table2 that are connected to Black.
+      ['BulkAddRecord', 'Table2', arrayRepeat(30, null), { A: arrayRepeat(30, 2) }],
+    ]);
+
+    // Now select White in Table1.
+    await gu.getCell('A', 1, 'Table1').click();
+
+    // Now expand Table1.
+    await gu.expandSection();
+
+    // Change to black.
+    await gu.getCell('A', 2, 'Table1').click();
+
+    // Close popup by sending ESCAPE
+    await gu.sendKeys(Key.ESCAPE);
+
+    // Make sure we see 30 records in Table2.
+    const count = await driver.executeScript(`
+      const section = Array.from(document.querySelectorAll('.test-widget-title-text'))
+                           .find(e => e.textContent === 'TABLE2')
+                           .closest('.viewsection_content');
+      return Array.from(section.querySelectorAll('.gridview_data_row_num')).length;
+    `);
+
+    assert.equal(count, 30 + 1);
+
+    await revert();
+  });
+
+
+  it('fix: should support searching', async function() {
+    // Collapse Companies section (a first one).
+    await collapseByMenu(COMPANIES);
+
+    // Clear any saved position state.
+    await driver.executeScript('window.sessionStorage.clear(); window.localStorage.clear();');
+
+    // Refresh.
+    await driver.navigate().refresh();
+    await gu.waitForDocToLoad();
+
+    // Here we had a bug that the hidden section was active, and the search was not working as it was
+    // starting with the hidden section.
+
+    // Now search (for something in the INVESTMENTS section)
+    await gu.search('2006');
+    await gu.closeSearch();
+
+    // Make sure we don't have an error.
+    await gu.checkForErrors();
+
+    assert.equal(await gu.getActiveSectionTitle(), INVESTMENTS);
+    // Make sure we are in 1column 9th row.
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 9, col: 0});
+
+    // Hide companies chart, and search for mobile (should show no results).
+    await collapseByMenu(COMPANIES_CHART);
+    await gu.search('mobile');
+    await gu.hasNoResult();
+    await gu.closeSearch();
+
+    // Open companies in the popup.
+    await openCollapsedSection(COMPANIES);
+    // Search for 2006, there will be no results.
+    await gu.search('2006');
+    await gu.hasNoResult();
+    // Now search for web.
+    await gu.closeSearch();
+    await gu.search('web');
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 5, col: 0});
+
+    // Recreate document (can't undo).
+    await session.tempDoc(cleanup, 'Investment Research.grist');
+  });
+
+
+  it('fix: should not dispose the instance when drag is cancelled', async function() {
+    const revert = await gu.begin();
+
+    // Collapse a section.
+    await collapseByMenu(INVESTMENTS);
+
+    // Drag it and then cancel.
+    await dragCollapsed(INVESTMENTS);
+    const logo = driver.find('.test-dm-logo');
+    await move(logo, {y:  0});
+    await move(logo, {y:  -1});
+    // Drop it here.
+    await driver.withActions(actions => actions.release());
+
+    // Now open it in the full view.
+    await openCollapsedSection(INVESTMENTS);
+
+    // And make sure we can move cursor.
+    await gu.getCell(1, 1).click();
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 1, col: 1});
+    await gu.getCell(1, 2).click();
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 2, col: 1});
+
+    // Change its type, and check that it works.
+    await gu.changeWidget('Card List');
+    // Undo it.
+    await gu.undo();
+    await gu.getCell(1, 3).click();
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 3, col: 1});
+
+    await gu.sendKeys(Key.ESCAPE);
+
+    // Move it back
+    await dragCollapsed(INVESTMENTS);
+
+    // Move back and drop.
+    await gu.getSection(COMPANIES_CHART).getRect();
+    await move(getDragElement(COMPANIES_CHART));
+    await driver.sleep(100);
+    await move(getDragElement(COMPANIES_CHART), {x : 200});
+    await driver.sleep(300);
+    assert.lengthOf(await driver.findAll(".layout_editor_drop_target.layout_hover"), 1);
+
+    await driver.withActions(actions => actions.release());
+    await driver.sleep(600);
+
+    // And make sure we can move cursor.
+    await gu.getCell(1, 1).click();
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 1, col: 1});
+    await gu.getCell(1, 2).click();
+    assert.deepEqual(await gu.getCursorPosition(), {rowNum: 2, col: 1});
+
+    await waitForSave();
+    await revert();
+  });
+
+
+  it('fix: should work when the page is refreshed', async function() {
+    const revert = await gu.begin();
+
+    await gu.openPage("Companies");
+    await gu.selectSectionByTitle("Companies");
+    // Go to second row.
+    await gu.getCell(0, 2).click();
+
+    // Make sure we see correct company card.
+    assert.equal(await gu.getCardCell('name', 'COMPANIES Card').getText(), '#NAME?');
+
+    // Hide first section.
+    await collapseByMenu("Companies");
+    await waitForSave();
+
+    // Refresh the page.
+    await driver.navigate().refresh();
+    await gu.waitForDocToLoad();
+
+    // Make sure card is still at the correct row.
+    await gu.waitToPass(async () => {
+      assert.equal(await gu.getCardCell('name', 'COMPANIES Card').getText(), '#NAME?');
+    });
+
+    await addToMainByMenu("Companies");
+    await revert();
+  });
+
+  it('fix: should support anchor links', async function() {
+    const revert = await gu.begin();
+
+    // Open 42Floors in Companies section.
+    assert.equal(await gu.getActiveSectionTitle(), "COMPANIES");
+    await gu.getCell('Link', 11).click();
+    assert.equal(await gu.getActiveCell().getText(), '42Floors');
+
+    // Open 12 row (Alex Bresler, angel).
+    await gu.getCell('funding_round_type', 12, 'Investments').click();
+    assert.equal(await gu.getActiveCell().getText(), 'angel');
+
+    // Copy anchor link.
+    const link = await gu.getAnchor();
+
+    // Collapse first section.
+    await collapseByMenu("COMPANIES");
+
+    // Clear any saved position state.
+    await driver.executeScript('window.sessionStorage.clear(); window.localStorage.clear();');
+
+    // Navigate to the home screen.
+    await gu.loadDocMenu('/o/docs');
+
+    // Now go to the anchor.
+    await driver.get(link);
+    await gu.waitForAnchor();
+
+    const cursor = await gu.getCursorPosition();
+    assert.equal(cursor.rowNum, 12);
+    assert.equal(cursor.col, 1);
+    assert.equal(await gu.getActiveCell().getText(), 'angel');
+    assert.equal(await gu.getActiveSectionTitle(), 'INVESTMENTS');
+    assert.match(await driver.getCurrentUrl(), /\/o\/docs\/[^/]+\/Investment-Research\/p\/1$/);
+    await revert();
+  });
+
+  it("should not autoexpand the tray on a page with a single widget", async () => {
+    await gu.openPage("Investments");
+    assert.equal((await driver.findAll(".viewsection_content")).length, 1);
+
+    // Start drag the main section.
+    await dragMain("INVESTMENTS");
+
+    // Move it over the logo, so that the tray thinks that it should expand.
+    const logo = driver.find('.test-dm-logo');
+    await move(logo, {y:  0});
+    await move(logo, {y:  -1});
+    await driver.sleep(100);
+
+    // Make sure the tray was not tricked into expanding itself.
+    assert.isFalse(await layoutTray().isDisplayed());
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 0); // No empty box
+
+    // Drop it on the button, it should go back to where it was.
+    await driver.withActions(actions => actions.release());
+  });
+
+  it("should autoexpand the tray", async () => {
+    await gu.openPage("Overview");
+
+    // Get one of the sections and start dragging it.
+    await dragMain(COMPANIES_CHART);
+
+    // The tray should not be expanded.
+    assert.isFalse(await layoutTray().isDisplayed());
+
+    const logo = driver.find('.test-dm-logo');
+    // Now move it to the top, so that tray should be expanded.
+    await move(logo, {y:  0});
+    await driver.sleep(100);
+
+    // Now the tray is visible
+    assert.isTrue(await layoutTray().isDisplayed());
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 1); // One empty box
+    assert.isTrue(await layoutEditor().matches('[class*=-is-active]')); // Is active
+    assert.isFalse(await layoutEditor().matches('[class*=-is-target]')); // Is not a target
+
+    // Drop it on the button, it should go back to where it was.
+    await driver.withActions(actions => actions.release());
+
+    // The tray should not be expanded.
+    assert.isFalse(await layoutTray().isDisplayed());
+
+    await gu.checkForErrors();
+  });
+
+  it("should drag onto main area", async () => {
+    const revert = await gu.begin();
+    await collapseByMenu(COMPANIES);
+    await collapseByMenu(INVESTMENTS);
+
+    await dragCollapsed(COMPANIES);
+    const chartCords = await gu.getSection(COMPANIES_CHART).getRect();
+    await move(getDragElement(COMPANIES_CHART));
+    await driver.sleep(100);
+    await move(getDragElement(COMPANIES_CHART), {x : 10});
+    await driver.sleep(300);
+
+    // We should have a drop target.
+    const dropTarget = await driver.find(".layout_editor_drop_target.layout_hover");
+    const dCords = await dropTarget.getRect();
+    // It should be more or less on the left of the chart.
+    assertDistance(dCords.x, chartCords.x, 20);
+    assertDistance(dCords.y, chartCords.y, 20);
+
+    // Move away from the drop target.
+    const addButton = driver.find('.test-dp-add-new');
+    await move(addButton);
+    await driver.sleep(300);
+
+    // Drop target should be gone.
+    assert.lengthOf(await driver.findAll(".layout_editor_drop_target.layout_hover"), 0);
+
+    // Move back and drop.
+    await move(getDragElement(COMPANIES_CHART));
+    await driver.sleep(100);
+    // Split the movement into two parts, to make sure layout sees the mouse move.
+    await move(getDragElement(COMPANIES_CHART), {x : 10});
+    await driver.sleep(200);
+    assert.lengthOf(await driver.findAll(".layout_editor_drop_target.layout_hover"), 1);
+    await driver.withActions(actions => actions.release());
+    await driver.sleep(600); // This animation can be longer.
+
+    // Make sure it was dropped.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-leaf-box"), 1); // Only one collapsed box.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 0); // No empty box.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 0); // No target box.
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS]); // Only investments is collapsed.
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, COMPANIES_CHART, INVESTMENTS_CHART]);
+    // Check that it was dropped on the left top side.
+    const companiesCords = await gu.getSection(COMPANIES).getRect();
+    assertDistance(companiesCords.x, chartCords.x, 20);
+    assertDistance(companiesCords.y, chartCords.y, 20);
+    // It should be half as tall as the main layout.
+    const root = await driver.find(".layout_root").getRect();
+    assertDistance(companiesCords.height, root.height / 2, 30);
+    // And almost half as wide.
+    assertDistance(companiesCords.width, root.width / 2, 30);
+
+    // Now move it back to the tray. But first collapse another section (so we can test inbetween target).
+    await collapseByMenu(COMPANIES_CHART);
+    await dragMain(COMPANIES);
+
+    // Try to move it as the first element.
+    const firstLeafSize = await firstLeaf().getRect();
+    await move(firstLeaf(), { x: -firstLeafSize.width / 2 });
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 1);
+    // Make sure that the target is in right place.
+    let target = await layoutTray().find(".test-layoutTray-target-box").getRect();
+    assertDistance(target.x, firstLeafSize.x, 10);
+
+    // Now as the second element.
+    await move(firstLeaf(), { x: firstLeafSize.width / 2 + GAP });
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 1);
+    target = await layoutTray().find(".test-layoutTray-target-box").getRect();
+    assertDistance(target.x, firstLeafSize.x + firstLeafSize.width + GAP, 10);
+
+    // Move away to make sure the target is gone.
+    await move(addButton);
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 0);
+
+    // Move back and drop.
+    await move(firstLeaf(), { x: firstLeafSize.width / 2 + GAP });
+    await driver.sleep(300);
+    await driver.withActions(actions => actions.release());
+    await driver.sleep(600);
+
+    // Make sure it was dropped.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-leaf-box"), 3);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 0);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 0);
+
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS, COMPANIES, COMPANIES_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS_CHART]);
+
+    await waitForSave(); // Layout save is debounced 1s.
+
+    // Test couple of undo steps.
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS, COMPANIES_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS_CHART]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, COMPANIES_CHART, INVESTMENTS_CHART]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES, INVESTMENTS]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART, INVESTMENTS]);
+
+    await gu.redo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES, INVESTMENTS]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART]);
+
+    await revert();
+    assert.deepEqual(await collapsedSectionTitles(), []);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, COMPANIES, INVESTMENTS_CHART, INVESTMENTS]);
+    await gu.checkForErrors();
+  });
+
+  it("should reorder collapsed sections", async () => {
+    const revert = await gu.begin();
+    await collapseByMenu(COMPANIES);
+    await collapseByMenu(INVESTMENTS);
+    await collapseByMenu(COMPANIES_CHART);
+
+    await dragCollapsed(COMPANIES);
+
+    // We should see the empty box in the layout.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 1);
+    // The section is actually removed from the layout tray.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-leaf-box"), 2);
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS, COMPANIES_CHART]);
+
+    // Layout should be active and accepting.
+    assert.isTrue(await layoutEditor().matches('[class*=-is-active]'));
+    assert.isTrue(await layoutEditor().matches('[class*=-is-target]'));
+
+    // Move mouse somewhere else, layout should not by highlighted.
+    const addButton = driver.find('.test-dp-add-new');
+    await move(addButton);
+    assert.isTrue(await layoutEditor().matches('[class*=-is-active]'));
+    assert.isFalse(await layoutEditor().matches('[class*=-is-target]'));
+
+    // Move to the first leaf, and wait for the target to show up.
+    const first = await firstLeaf().getRect();
+    await move(firstLeaf(), {x : -first.width / 2});
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 1);
+    // Make sure that the target is in right place.
+    let target = await layoutTray().find(".test-layoutTray-target-box").getRect();
+    assert.isBelow(Math.abs(target.x - first.x), 10);
+    assert.isBelow(Math.abs(target.y - first.y), 10);
+    assert.isBelow(Math.abs(target.height - first.height), 10);
+
+    // Move away and make sure the target is gone.
+    await move(addButton);
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 0);
+
+    // Move between first and second leaf.
+    await move(firstLeaf(), {x : first.width / 2 + GAP});
+    await driver.sleep(300);
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-target-box"), 1);
+    target = await layoutTray().find(".test-layoutTray-target-box").getRect();
+    assert.isBelow(Math.abs(target.height - first.height), 2);
+    assert.isBelow(Math.abs(target.y - first.y), 2);
+    // Should be between first and second leaf.
+    assert.isBelow(Math.abs(target.x - (first.x + first.width + GAP)), 10);
+
+    // Drop here.
+    await driver.withActions(actions => actions.release());
+    await waitForSave(); // Wait for layout to be saved.
+    // Target is gone.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-empty-box"), 0);
+    // And we have 3 sections in the layout.
+    assert.lengthOf(await layoutTray().findAll(".test-layoutTray-leaf-box"), 3);
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS, COMPANIES, COMPANIES_CHART]);
+
+    // Undo.
+    await gu.undo();
+    // Order should be restored.
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES, INVESTMENTS, COMPANIES_CHART]);
+
+    await revert();
+    await gu.checkForErrors();
+  });
+
+  it("should collapse sections and expand using menu", async () => {
+    await collapseByMenu(COMPANIES_CHART);
+    await gu.checkForErrors();
+
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART]);
+    // Make sure that other sections are not collapsed.
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS_CHART, INVESTMENTS]);
+
+    await collapseByMenu(INVESTMENTS_CHART);
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS]);
+
+    await collapseByMenu(COMPANIES);
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    // The last section is INVESTMENTS, which can't be collapsed.
+    await gu.openSectionMenu('viewLayout', INVESTMENTS);
+    assert.equal(await driver.find('.test-section-collapse').matches('[class*=disabled]'), true);
+    await driver.sendKeys(Key.ESCAPE);
+
+    // Now expand them one by one and test.
+    await addToMainByMenu(COMPANIES_CHART);
+    await gu.checkForErrors();
+
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS, COMPANIES_CHART]);
+
+    await addToMainByMenu(INVESTMENTS_CHART);
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS, COMPANIES_CHART, INVESTMENTS_CHART]);
+    await gu.checkForErrors();
+
+    await addToMainByMenu(COMPANIES);
+    assert.deepEqual(await collapsedSectionTitles(), []);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS, COMPANIES_CHART, INVESTMENTS_CHART, COMPANIES]);
+    await gu.checkForErrors();
+
+    // Now revert everything using undo but test each step.
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS, COMPANIES_CHART, INVESTMENTS_CHART]);
+    await gu.checkForErrors();
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS, COMPANIES_CHART]);
+    await gu.checkForErrors();
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+    await gu.checkForErrors();
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS]);
+    await gu.checkForErrors();
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS_CHART, INVESTMENTS]);
+    await gu.checkForErrors();
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), []);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, COMPANIES, INVESTMENTS_CHART, INVESTMENTS]);
+    await gu.checkForErrors();
+  });
+
+  it("should remove sections from collapsed tray", async () => {
+    const revert = await gu.begin();
+    // Collapse everything we can.
+    await collapseByMenu(COMPANIES_CHART);
+    await collapseByMenu(INVESTMENTS_CHART);
+    await collapseByMenu(COMPANIES);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    // Now remove them using menu.
+    await removeMiniSection(COMPANIES_CHART);
+    await gu.checkForErrors();
+
+    // Check that the section is removed from the collapsed tray.
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS_CHART, COMPANIES]);
+    // Make sure it is stays removed when we move to the other page.
+    await gu.openPage("Investments");
+    // Go back.
+    await gu.openPage("Overview");
+    await gu.checkForErrors();
+
+    // Test if we see everything as it was.
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS_CHART, COMPANIES]);
+    // Make sure that visible sections are not affected.
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    // Remove the other sections.
+    await removeMiniSection(INVESTMENTS_CHART);
+    await removeMiniSection(COMPANIES);
+    assert.deepEqual(await collapsedSectionTitles(), []);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    // Now revert everything using undo but test each step.
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART, COMPANIES]);
+    assert.deepEqual(await mainSectionTitles(), [INVESTMENTS]);
+
+    await gu.undo();
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART]);
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS]);
+
+    // Ok, we are good, revert back to the original state.
+    await revert();
+    await gu.checkForErrors();
+  });
+
+  it("should switch active section when collapsed", async () => {
+    const revert = await gu.begin();
+    await gu.selectSectionByTitle(gu.exactMatch(COMPANIES));
+    // Make sure we are active.
+    assert.equal(await gu.getActiveSectionTitle(), COMPANIES);
+    // Collapse it.
+    await collapseByMenu(COMPANIES);
+    // Make sure it is collapsed.
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES]);
+    // Make sure that now COMPANIES_CHART is active. (first one).
+    assert.equal(await gu.getActiveSectionTitle(), COMPANIES_CHART);
+    // Expand COMPANIES.
+    await addToMainByMenu(COMPANIES);
+    // Make sure that now it is active.
+    assert.equal(await gu.getActiveSectionTitle(), COMPANIES);
+    await revert();
+    await gu.checkForErrors();
+  });
+
+  it("should show section on popup when clicked", async () => {
+    const revert = await gu.begin();
+    await collapseByMenu(COMPANIES);
+    await openCollapsedSection(COMPANIES);
+    // Make sure it is expanded.
+    assert.isTrue(await driver.find(".test-viewLayout-overlay").matches("[class*=-active]"));
+    assert.equal(await gu.getActiveSectionTitle(), COMPANIES);
+    // Make sure that the panel shows it.
+    await gu.toggleSidePanel('right', 'open');
+    await driver.find('.test-config-widget').click();
+    assert.equal(await driver.find('.test-right-widget-title').value(), COMPANIES);
+    // Make sure we see proper items in the menu.
+    await gu.openSectionMenu('viewLayout', COMPANIES);
+    // Collapse widget menu item is disabled.
+    assert.equal(await driver.find('.test-section-collapse').matches('[class*=disabled]'), true);
+    // Delete widget is enabled.
+    assert.equal(await driver.find('.test-section-delete').matches('[class*=disabled]'), false);
+    await driver.sendKeys(Key.ESCAPE);
+    // Expand button is not visible
+    assert.lengthOf(await driver.findAll(".active_section .test-section-menu-expandSection"), 0);
+    // We can rename a section using the popup.
+    await gu.renameActiveSection("New name");
+    assert.equal(await gu.getActiveSectionTitle(), "New name");
+    // Make sure the name is reflected in the collapsed tray.
+    await gu.sendKeys(Key.ESCAPE);
+    assert.deepEqual(await collapsedSectionTitles(), ["New name"]);
+    // Open it back.
+    await openCollapsedSection("New name");
+    // Rename it back using undo.
+    await gu.undo();
+    assert.equal(await gu.getActiveSectionTitle(), COMPANIES);
+    // Now remove it.
+    await gu.openSectionMenu('viewLayout', COMPANIES);
+    await driver.find('.test-section-delete').click();
+    await gu.waitForServer();
+    // Make sure it is closed.
+    assert.isFalse(await driver.find(".test-viewLayout-overlay").matches("[class*=-active]"));
+    // Make sure it is removed.
+    assert.deepEqual(await collapsedSectionTitles(), []);
+    // Make sure it didn't reappear on the main area.
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES_CHART, INVESTMENTS_CHART, INVESTMENTS]);
+
+    await revert();
+    await gu.checkForErrors();
+  });
+
+  it("should collapse and expand charts without an error", async () => {
+    const revert = await gu.begin();
+    await collapseByMenu(INVESTMENTS);
+    await dragMain(COMPANIES_CHART);
+    const firstRect = await firstLeaf().getRect();
+    await move(firstLeaf(), { x: firstRect.width / 2 + GAP });
+    await driver.sleep(300);
+    await driver.withActions(actions => actions.release());
+    await waitForSave(); // Resize is delayed.
+    await gu.checkForErrors();
+    await revert();
+  });
+
+  it("should drop on the empty space", async () => {
+    const revert = await gu.begin();
+    // Get one of the sections and start dragging it.
+    await dragMain(COMPANIES_CHART);
+    // Move it over the logo to show the tray.
+    const logo = driver.find('.test-dm-logo');
+    await move(logo, {y:  0});
+    await move(logo, {y:  -20});
+    await driver.sleep(100);
+    // Now the tray is visible
+    assert.isTrue(await layoutTray().isDisplayed());
+    // Move it on the empty space just after the empty box
+    const emptyBox = await layoutTray().find(".test-layoutTray-empty-box");
+    const emptyBoxCords = await emptyBox.getRect();
+    await move(emptyBox, {x: emptyBoxCords.width + 100 });
+    // Make sure that the empty box is not active.
+    assert.isFalse(await emptyBox.matches('[class*=-is-active]'));
+    // Drop it here
+    await driver.withActions(actions => actions.release());
+    await driver.sleep(600); // Wait for animation to finish.
+    await waitForSave();
+    // The tray should stay expanded.
+    assert.isTrue(await layoutTray().isDisplayed());
+
+    // Check that the section was collapsed.
+    assert.deepEqual(await collapsedSectionTitles(), [COMPANIES_CHART]);
+    // And other sections are still there.
+    assert.deepEqual(await mainSectionTitles(), [COMPANIES, INVESTMENTS_CHART, INVESTMENTS]);
+    await gu.checkForErrors();
+    await revert();
+    await gu.checkForErrors();
+  });
+
+  it("should clear layout when dropped section is removed", async () => {
+    await session.tempNewDoc(cleanup, 'CollapsedBug.grist');
+    // Add a new section based on current table.
+    await gu.addNewSection('Table', 'Table1');
+    // It will have id 3 (1 is raw, 2 is visible).
+    // Collapse it.
+    await gu.renameActiveSection('ToDelete');
+    await collapseByMenu('ToDelete');
+    // Remove it from the tray.
+    await openCollapsedSectionMenu('ToDelete');
+    await driver.find('.test-section-delete').click();
+    await gu.waitForServer();
+    await waitForSave();
+    // Now add another one, it will have the same id (3) and it used to be collapsed when added
+    // as the layout was not cleared.
+    await gu.addNewSection('Table', 'Table1');
+    // Make sure it is expanded.
+    assert.deepEqual(await mainSectionTitles(), ['TABLE1', 'TABLE1']);
+    assert.deepEqual(await collapsedSectionTitles(), []);
+  });
+});
+
+async function addToMainByMenu(section: string) {
+  await openCollapsedSectionMenu(section);
+  await driver.find('.test-section-expand').click();
+  await gu.waitForServer();
+  await gu.checkForErrors();
+}
+
+async function dragCollapsed(section: string) {
+  const handle = getCollapsedSection(section).find('.draggable-handle');
+  await driver.withActions((actions) => actions
+    .move({origin: handle})
+    .press());
+  await move(handle, {x : 10, y: 10});
+  return handle;
+}
+
+async function dragMain(section: string) {
+  const handle = gu.getSection(section).find('.viewsection_drag_indicator');
+  await driver.withActions((actions) => actions
+    .move({origin: handle}));
+  await driver.withActions((actions) => actions
+    .move({origin: handle, x : 1}) // This is needed to show the drag element.
+    .press());
+  await move(handle, {x : 10, y: 10});
+  return handle;
+}
+
+async function openCollapsedSection(section: string) {
+  await getCollapsedSection(section).find('.draggable-handle').click();
+}
+
+async function removeMiniSection(section: string) {
+  await openCollapsedSectionMenu(section);
+  await driver.find('.test-section-delete').click();
+  await gu.waitForServer();
+  await gu.checkForErrors();
+}
+
+async function collapseByMenu(section: string) {
+  await gu.openSectionMenu('viewLayout', section);
+  await driver.find('.test-section-collapse').click();
+  await gu.waitForServer();
+  await gu.checkForErrors();
+}
+
+// Returns the titles of all collapsed sections.
+async function collapsedSectionTitles() {
+  return await layoutTray().findAll('.test-layoutTray-leaf-box .test-collapsed-section-title', e => e.getText());
+}
+
+// Returns titles of all sections in the view layout.
+async function mainSectionTitles() {
+  return await driver.findAll('.layout_root .test-viewsection-title', e => e.getText());
+}
+
+async function move(element: WebElementPromise|WebElement, offset: {x?: number, y?: number} = {x: 0, y: 0}) {
+  // With current version of webdriver, a fractional values will get ignored, so round to nearest.
+  if (offset.x) { offset.x = Math.round(offset.x); }
+  if (offset.y) { offset.y = Math.round(offset.y); }
+  await driver.withActions(actions => actions.move({origin: element, ...offset}));
+}
+
+
+function getDragElement(section: string) {
+  return gu.getSection(section).find('.viewsection_drag_indicator');
+}
+
+function layoutTray() {
+  return driver.find(".test-layoutTray-layout");
+}
+
+function firstLeaf() {
+  return layoutTray().find(".test-layoutTray-leaf-box");
+}
+
+function layoutEditor() {
+  return driver.find(".test-layoutTray-editor");
+}
+
+const COMPANIES_CHART = 'COMPANIES [by category_code] Chart';
+const INVESTMENTS_CHART = 'INVESTMENTS [by funded_year] Chart';
+const COMPANIES = 'COMPANIES [by category_code]';
+const INVESTMENTS = 'INVESTMENTS [by funded_year]';
+
+function assertDistance(left: number, right: number, max: number) {
+  return assert.isBelow(Math.abs(left - right), max);
+}
+
+async function waitForSave() {
+  await gu.waitToPass(async () => {
+    const pending = await driver.findAll(".test-viewLayout-save-pending");
+    assert.isTrue(pending.length === 0);
+    await gu.waitForServer();
+  }, 3000);
+}

test/nbrowser/ViewLayoutUtils.ts

@@ -0,0 +1,46 @@
+import {assert, driver} from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+
+export async function closeExpandedSection() {
+  await driver.find(".test-viewLayout-overlay .test-close-button").click();
+  await gu.waitToPass(async () => {
+    assert.isFalse(await driver.find(".test-viewLayout-overlay").matches("[class*=-active]"));
+  });
+}
+
+export async function sectionIsExpanded() {
+  // Check that we see the overlay
+  assert.isTrue(await driver.find(".test-viewLayout-overlay").matches("[class*=-active]"));
+
+  // Visually check that the section is expanded.
+  assert.isTrue(await driver.find(".active_section").isDisplayed());
+  const section = await driver.find(".active_section").getRect();
+  const doc = await driver.find(".test-gristdoc").getRect();
+  assert.isTrue(Math.abs(section.height + 48 - doc.height) < 4);
+  assert.isTrue(Math.abs(section.width + 112 - doc.width) < 4);
+
+  // Get all other sections on the page and make sure they are hidden.
+  const otherSections = await driver.findAll(".view_leaf:not(.active_section)");
+  for (const otherSection of otherSections) {
+    assert.isFalse(await otherSection.isDisplayed());
+  }
+
+  // Make sure we see the close button.
+  assert.isTrue(await driver.find(".test-viewLayout-overlay .test-close-button").isDisplayed());
+}
+
+/**
+ * Opens the section menu for a collapsed section.
+ */
+export async function openCollapsedSectionMenu(section: string|RegExp) {
+  await getCollapsedSection(section).find(`.test-section-menu-viewLayout`).click();
+  await driver.findWait('.grist-floating-menu', 100);
+}
+
+export function getCollapsedSection(section: string|RegExp) {
+  if (typeof section === 'string') {
+    section = gu.exactMatch(section, 'i');
+  }
+  return driver.findContentWait('.test-collapsed-section .test-collapsed-section-title', section, 100)
+               .findClosest('.test-collapsed-section');
+}

test/nbrowser/WebhookPage.ts

@@ -8,6 +8,7 @@ import {EnvironmentSnapshot} from 'test/server/testUtils';
 describe('WebhookPage', function () {
   this.timeout(60000);
   const cleanup = setupTestSuite();
+  const clipboard = gu.getLockableClipboard();
 
   let session: gu.Session;
   let oldEnv: EnvironmentSnapshot;
@@ -192,7 +193,7 @@ describe('WebhookPage', function () {
     await openWebhookPage();
 
     // Open another tab.
-    await driver.executeScript("return window.open('about:blank', '_blank')");
+    await driver.executeScript("window.open('about:blank', '_blank')");
     const [ownerTab, owner2Tab] = await driver.getAllWindowHandles();
 
     await driver.switchTo().window(owner2Tab);
@@ -256,9 +257,11 @@ describe('WebhookPage', function () {
     await openWebhookPage();
     await setField(1, 'Name', '1234');
     await gu.waitForServer();
-    await gu.sendKeys(await gu.copyKey());
+    await clipboard.lockAndPerform(async (cb) => {
-    await gu.getDetailCell({col: 'Memo', rowNum: 1}).click();
+      await cb.copy();
-    await gu.sendKeys(await gu.pasteKey());
+      await gu.getDetailCell({col: 'Memo', rowNum: 1}).click();
+      await cb.paste();
+    });
     await gu.waitForServer();
     assert.equal(await getField(1, 'Memo'), '1234');
   });

test/nbrowser/chartViewTestUtils.ts

@@ -0,0 +1,99 @@
+import {assert, driver, WebElement} from 'mocha-webdriver';
+import {Layout, LayoutAxis, PlotData} from 'plotly.js';
+import * as gu from 'test/nbrowser/gristUtils';
+import isString = require('lodash/isString');
+import isUndefined = require('lodash/isUndefined');
+
+export interface ChartData {
+  data: Partial<PlotData>[];
+  layout: Layout;
+}
+
+export async function selectChartType(chartType: string) {
+  await driver.find('.test-chart-type').click();
+  await driver.findContent('.test-select-row', chartType).click();
+  return gu.waitForServer();
+}
+
+export async function getChartData(chartElem?: WebElement|string): Promise<ChartData> {
+  if (isString(chartElem) || isUndefined(chartElem)) {
+    const section = isString(chartElem) ?
+      await gu.getSection(chartElem) :
+      await driver.findWait('.active_section', 4000);
+    chartElem = await section.find('.test-chart-container');
+  }
+  return driver.executeScript((el: any) => ({data: el.data, layout: el.layout}), chartElem);
+}
+
+export function checkAxisRange({layout}: ChartData, xMin: number, xMax: number, yMin: number, yMax: number) {
+  assert.closeTo(layout.xaxis.range![0], xMin, xMin * 0.1);
+  assert.closeTo(layout.xaxis.range![1], xMax, xMax * 0.1);
+  assert.closeTo(layout.yaxis.range![0], yMin, yMin * 0.1);
+  assert.closeTo(layout.yaxis.range![1], yMax, yMax * 0.1);
+}
+
+export function getAxisTitle(axis: Partial<LayoutAxis>): string|undefined {
+  return axis.title && (axis.title as any).text;
+}
+
+export function findYAxis(name: string) {
+  return driver.findContent('.test-chart-y-axis', name);
+}
+
+export async function removeYAxis(name: string) {
+  await findYAxis(name).mouseMove().find('.test-chart-ref-select-remove').click();
+  await gu.waitForServer();
+}
+
+export async function checkAxisConfig(expected: {groupingByColumn?: string|false,
+                                                 xaxis: string|undefined, yaxis: string[]}) {
+  const isGroupByPresent = await driver.find('.test-chart-group-by-column').isPresent();
+  let groupingByColumn = isGroupByPresent ? await driver.find('.test-chart-group-by-column').getText() : false;
+  if (groupingByColumn === 'Pick a column') {
+    groupingByColumn = false;
+  }
+  const xaxis = await driver.find('.test-chart-x-axis').getText();
+  assert.deepEqual({
+    groupingByColumn,
+    xaxis: xaxis === 'Pick a column' ? undefined : xaxis,
+    yaxis: await driver.findAll('.test-chart-y-axis', (e) => e.getText()),
+  }, {...expected, groupingByColumn: expected.groupingByColumn || false});
+}
+
+export async function setSplitSeries(name: string|false, section?: string) {
+  await gu.openSectionMenu('viewLayout', section);
+  await driver.findContent('.grist-floating-menu li', 'Widget options').click();
+
+  const isChecked = await driver.findContent('label', /Split series/).find('input').matches(':checked');
+  if (name === false && isChecked === true ||
+      name && isChecked === false) {
+    await driver.findContent('label', /Split series/).click();
+  }
+  if (name) {
+    await driver.find('.test-chart-group-by-column').click();
+    await driver.findContent('.test-select-menu li', name || 'Pick a column').click();
+  }
+  await gu.waitForServer();
+}
+
+export async function selectXAxis(name: string, opt: {noWait?: boolean} = {}) {
+  await driver.find('.test-chart-x-axis').click();
+  await driver.findContent('.test-select-menu li', name).click();
+  if (!opt.noWait) {
+    await gu.waitForServer();
+  }
+}
+
+
+export async function setYAxis(names: string[]) {
+  // let's first remove all yaxis and then add new ones
+  const toRemove = await driver.findAll('.test-chart-y-axis', (e) => e.getText());
+  for (const n of toRemove) { await removeYAxis(n); }
+  for (const n of names) { await addYAxis(n); }
+}
+
+export async function addYAxis(name: string) {
+  await driver.find('.test-chart-add-y-axis').click();
+  await driver.findContent('.grist-floating-menu li', name).click();
+  await gu.waitForServer();
+}

test/nbrowser/gristUtil-nbrowser.js

@@ -17,24 +17,11 @@ function $(key) {
   return _webdriverjqFactory(key);
 }
 
-function rep(n, value) {
-  return _.times(n, () => value);
-}
-
 // The "$" object needs some setup done asynchronously.
 // We do that later, during test initialization.
 async function applyPatchesToJquerylikeObject($) {
   $.MOD = await guBase.modKey();
-  $.COPY = await guBase.copyKey();
-  $.CUT = await guBase.cutKey();
-  $.PASTE = await guBase.pasteKey();
   $.SELECT_ALL = await guBase.selectAllKey();
-  const capabilities = await driver.getCapabilities();
-  if (capabilities.getBrowserName() === 'chrome' && await guBase.isMac()) {
-    $.SELECT_ALL_LINES = (n) => [...rep(n, $.UP), $.HOME, $.SHIFT, ...rep(n, $.DOWN), $.END]
-  } else {
-    $.SELECT_ALL_LINES = () => $.SELECT_ALL;
-  }
   $.getPage = async (url) => {
     return driver.get(url);
   };
@@ -227,8 +214,8 @@ const gu = {
     return gu.toggleSidePanel('right', 'close');
   },
 
-  clickVisibleDetailCells(column, rowNums) {
+  clickVisibleDetailCells(column, rowNums, section) {
-    return gu.getDetailCell(column, rowNums[0]).click();
+    return gu.getDetailCell(column, rowNums[0], section).click();
   },
 
   async clickRowMenuItem(rowNum, item) {
@@ -302,9 +289,15 @@ const gu = {
   *                         endCell[1]: 0-based column index.
   **/
   async selectGridArea(startCell, endCell) {
-    let start = await gu.getCell({rowNum: startCell[0], col: startCell[1]});
+    const [startRowNum, startCol] = startCell;
-    let end = await gu.getCell({rowNum: endCell[0], col: endCell[1]});
+    const [endRowNum, endCol] = endCell;
-    await driver.withActions(a => a.click(start).keyDown($.SHIFT).click(end).keyUp($.SHIFT));
+    if (startRowNum === endRowNum && startCol === endCol) {
+      await gu.getCell({rowNum: endRowNum, col: endCol}).click();
+    } else {
+      const start = await gu.getCell({rowNum: startRowNum, col: startCol});
+      const end = await gu.getCell({rowNum: endRowNum, col: endCol});
+      await driver.withActions(a => a.click(start).keyDown($.SHIFT).click(end).keyUp($.SHIFT));
+    }
   },
 
   /**

test/nbrowser/gristUtils.ts

@@ -24,6 +24,7 @@ import { Organization as APIOrganization, DocStateComparison,
 import { Organization } from 'app/gen-server/entity/Organization';
 import { Product } from 'app/gen-server/entity/Product';
 import { create } from 'app/server/lib/create';
+import { getAppRoot } from 'app/server/lib/places';
 
 import { GristWebDriverUtils, PageWidgetPickerOptions,
          WindowDimensions as WindowDimensionsBase } from 'test/nbrowser/gristWebDriverUtils';
@@ -33,6 +34,7 @@ import { Cleanup } from 'test/nbrowser/testUtils';
 import * as testUtils from 'test/server/testUtils';
 import type { AssertionError } from 'assert';
 import axios from 'axios';
+import { lock } from 'proper-lockfile';
 
 // tslint:disable:no-namespace
 // Wrap in a namespace so that we can apply stackWrapOwnMethods to all the exports together.
@@ -923,6 +925,10 @@ export async function userActionsVerify(expectedUserActions: unknown[]): Promise
     if (!Array.isArray(assertError.actual)) {
       throw new Error('userActionsVerify: no user actions, run userActionsCollect() first');
     }
+    if (!Array.isArray(assertError.expected)) {
+      throw new Error('userActionsVerify: no expected user actions');
+    }
+
     assertError.actual = assertError.actual.map((a: any) => JSON.stringify(a) + ",").join("\n");
     assertError.expected = assertError.expected.map((a: any) => JSON.stringify(a) + ",").join("\n");
     assert.deepEqual(assertError.actual, assertError.expected);
@@ -1402,6 +1408,9 @@ export async function closeSearch() {
 }
 
 export async function closeTooltip() {
+  if (!await driver.find('.test-tooltip').isPresent()) { return; }
+
+  await driver.find('.test-tooltip').mouseMove();
   await driver.mouseMoveBy({x : 100, y: 100});
   await waitToPass(async () => {
     assert.equal(await driver.find('.test-tooltip').isPresent(), false);
@@ -1580,29 +1589,16 @@ export async function applyTypeTransform() {
 }
 
 export async function isMac(): Promise<boolean> {
-  return /Darwin|Mac|iPod|iPhone|iPad/i.test((await driver.getCapabilities()).get('platform'));
+  const platform = (await driver.getCapabilities()).getPlatform() ?? '';
+  return /Darwin|Mac|mac os x|iPod|iPhone|iPad/i.test(platform);
 }
 
 export async function modKey() {
   return await isMac() ? Key.COMMAND : Key.CONTROL;
 }
 
-// For copy-pasting, use different key combinations for Chrome on Mac.
-// See http://stackoverflow.com/a/41046276/328565
-export async function copyKey() {
-  return await isMac() ? Key.chord(Key.CONTROL, Key.INSERT) : Key.chord(Key.CONTROL, 'c');
-}
-
-export async function cutKey() {
-  return await isMac() ? Key.chord(Key.CONTROL, Key.DELETE) : Key.chord(Key.CONTROL, 'x');
-}
-
-export async function pasteKey() {
-  return await isMac() ? Key.chord(Key.SHIFT, Key.INSERT) : Key.chord(Key.CONTROL, 'v');
-}
-
 export async function selectAllKey() {
-  return await isMac() ? Key.chord(Key.HOME, Key.SHIFT, Key.END) : Key.chord(Key.CONTROL, 'a');
+  return await isMac() ? Key.chord(Key.COMMAND, 'a') : Key.chord(Key.CONTROL, 'a');
 }
 
 /**
@@ -1616,14 +1612,13 @@ export async function sendKeys(...keys: string[]) {
     const toRelease: string[] =  [];
     for (const part of keys) {
       for (const key of part) {
-        if ([Key.SHIFT, Key.CONTROL, Key.ALT, Key.META].includes(key)) {
+        if ([Key.ALT, Key.CONTROL, Key.SHIFT, Key.COMMAND, Key.META].includes(key)) {
           a.keyDown(key);
           toRelease.push(key);
         } else if (key === Key.NULL) {
           toRelease.splice(0).reverse().forEach(k => a.keyUp(k));
         } else {
-          a.keyDown(key);
+          a.sendKeys(key);
-          a.keyUp(key);
         }
       }
     }
@@ -1631,11 +1626,10 @@ export async function sendKeys(...keys: string[]) {
 }
 
 /**
- * Clears active input/textarea by sending CTRL HOME + CTRL + SHIFT END + DELETE.
+ * Clears active input/textarea.
  */
 export async function clearInput() {
-  const ctrl = await modKey();
+  return sendKeys(await selectAllKey(), Key.DELETE);
-  return sendKeys(Key.chord(ctrl, Key.HOME), Key.chord(ctrl, Key.SHIFT, Key.END), Key.DELETE);
 }
 
 /**
@@ -2720,10 +2714,10 @@ export async function getEnabledOptions(): Promise<SortOption[]> {
 /**
  * Runs action in a separate tab, closing the tab after.
  * In case of an error tab is not closed, consider using cleanupExtraWindows
- * on whole test suit if needed.
+ * on whole test suite if needed.
  */
 export async function onNewTab(action: () => Promise<void>) {
-  await driver.executeScript("return window.open('about:blank', '_blank')");
+  await driver.executeScript("window.open('about:blank', '_blank')");
   const tabs = await driver.getAllWindowHandles();
   await driver.switchTo().window(tabs[tabs.length - 1]);
   await action();
@@ -3273,6 +3267,126 @@ export async function changeWidgetAccess(access: 'read table'|'full'|'none') {
   }
 }
 
+/*
+ * Returns an instance of `LockableClipboard`, making sure to unlock it after
+ * each test.
+ *
+ * Recommended for use in contexts where the system clipboard may be accessed by
+ * multiple parallel processes, such as Mocha tests.
+ */
+export function getLockableClipboard() {
+  const cb = new LockableClipboard();
+
+  afterEach(async () => {
+    await cb.unlock();
+  });
+
+  return cb;
+}
+
+export interface ILockableClipboard {
+  lockAndPerform(callback: (clipboard: IClipboard) => Promise<void>): Promise<void>;
+  unlock(): Promise<void>;
+}
+
+class LockableClipboard implements ILockableClipboard {
+  private _unlock: (() => Promise<void>) | null = null;
+
+  constructor() {
+
+  }
+
+  public async lockAndPerform(callback: (clipboard: IClipboard) => Promise<void>) {
+    this._unlock = await lock(path.resolve(getAppRoot(), 'test'), {
+      lockfilePath: path.join(path.resolve(getAppRoot(), 'test'), '.clipboard.lock'),
+      retries: {
+        /* The clipboard generally isn't locked for long, so retry frequently. */
+        maxTimeout: 1000,
+        retries: 20,
+      },
+    });
+    try {
+      await callback(new Clipboard());
+    } finally {
+      await this.unlock();
+    }
+  }
+
+  public async unlock() {
+    await this._unlock?.();
+    this._unlock = null;
+  }
+}
+
+export type ClipboardAction = 'copy' | 'cut' | 'paste';
+
+export interface ClipboardActionOptions {
+  method?: 'keyboard' | 'menu';
+}
+
+export interface IClipboard {
+  copy(options?: ClipboardActionOptions): Promise<void>;
+  cut(options?: ClipboardActionOptions): Promise<void>;
+  paste(options?: ClipboardActionOptions): Promise<void>;
+}
+
+class Clipboard implements IClipboard {
+  constructor() {
+
+  }
+
+  public async copy(options: ClipboardActionOptions = {}) {
+    await this._performAction('copy', options);
+  }
+
+  public async cut(options: ClipboardActionOptions = {}) {
+    await this._performAction('cut', options);
+  }
+
+  public async paste(options: ClipboardActionOptions = {}) {
+    await this._performAction('paste', options);
+  }
+
+  private async _performAction(action: ClipboardAction, options: ClipboardActionOptions) {
+    const {method = 'keyboard'} = options;
+    switch (method) {
+      case 'keyboard': {
+        await this._performActionWithKeyboard(action);
+        break;
+      }
+      case 'menu': {
+        await this._performActionWithMenu(action);
+        break;
+      }
+    }
+  }
+
+  private async _performActionWithKeyboard(action: ClipboardAction) {
+    switch (action) {
+      case 'copy': {
+        await sendKeys(Key.chord(await isMac() ? Key.COMMAND : Key.CONTROL, 'c'));
+        break;
+      }
+      case 'cut': {
+        await sendKeys(Key.chord(await isMac() ? Key.COMMAND : Key.CONTROL, 'x'));
+        break;
+      }
+      case 'paste': {
+        await sendKeys(Key.chord(await isMac() ? Key.COMMAND : Key.CONTROL, 'v'));
+        break;
+      }
+    }
+  }
+
+  private async _performActionWithMenu(action: ClipboardAction) {
+    const field = await driver.find('.active_section .field_clip.has_cursor');
+    await driver.withActions(actions => { actions.contextClick(field); });
+    await driver.findWait('.grist-floating-menu', 1000);
+    const menuItemName = action.charAt(0).toUpperCase() + action.slice(1);
+    await driver.findContent('.grist-floating-menu li', menuItemName).click();
+  }
+}
+
 
 } // end of namespace gristUtils
 

test/server/Comm.ts

@@ -278,37 +278,49 @@ describe('Comm', function() {
     });
 
     async function testMissedResponses(sendShouldFail: boolean) {
-      const logMessages = await testUtils.captureLog('debug', async () => {
+      let failedSendCount = 0;
-        const {cliComm, forwarder} = await startManagedConnection({...assortedMethods,
-          // An extra method that simulates a lost connection on server side prior to response.
-          testDisconnect: async function(client, x, y) {
-            await delay(0);     //  disconnect on the same tick.
-            await forwarder.disconnectServerSide();
-            if (!sendShouldFail) {
-              // Add a delay to let the 'close' event get noticed first.
-              await delay(20);
-            }
-            return {x: x, y: y, name: "testDisconnect"};
-          },
-        });
 
-        const resp1 = await cliComm._makeRequest(null, null, "methodSync", "foo", 1);
+      const {cliComm, forwarder} = await startManagedConnection({...assortedMethods,
-        assert.deepEqual(resp1, {name: 'methodSync', x: "foo", y: 1});
+        // An extra method that simulates a lost connection on server side prior to response.
-
+        testDisconnect: async function(client, x, y) {
-        // Make more calls, with a disconnect before they return. The server should queue up responses.
+          setTimeout(() => forwarder.disconnectServerSide(), 0);
-        const resp2Promise = cliComm._makeRequest(null, null, "testDisconnect", "foo", 2);
+          if (!sendShouldFail) {
-        const resp3Promise = cliComm._makeRequest(null, null, "methodAsync", "foo", 3);
+            // Add a delay to let the 'close' event get noticed first.
-        assert.equal(await isLongerThan(resp2Promise, 250), true);
+            await delay(20);
-
+          }
-        // Once we reconnect, the response should arrive.
+          return {x: x, y: y, name: "testDisconnect"};
-        await forwarder.connect();
+        },
-        assert.deepEqual(await resp2Promise, {name: 'testDisconnect', x: "foo", y: 2});
-        assert.deepEqual(await resp3Promise, {name: 'methodAsync', x: "foo", y: 3});
       });
 
-      // Check that we saw the situations we were hoping to test.
+      const resp1 = await cliComm._makeRequest(null, null, "methodSync", "foo", 1);
-      assert.equal(logMessages.some(m => /^warn: .*send error.*WebSocket is not open/.test(m)), sendShouldFail,
+      assert.deepEqual(resp1, {name: 'methodSync', x: "foo", y: 1});
-        `Expected to see a failed send:\n${logMessages.join('\n')}`);
+
+      if (sendShouldFail) {
+        // In Node 18, the socket is closed during the call to 'testDisconnect'.
+        // In prior versions of Node, the socket was still disconnecting.
+        // This test is sensitive to timing and only passes in the latter, unless we
+        // stub the method below to produce similar behavior in the former.
+        sandbox.stub(Client.prototype as any, '_sendToWebsocket')
+          .onFirstCall()
+          .callsFake(() => {
+            failedSendCount += 1;
+            throw new Error('WebSocket is not open');
+          })
+          .callThrough();
+      }
+
+      // Make more calls, with a disconnect before they return. The server should queue up responses.
+      const resp2Promise = cliComm._makeRequest(null, null, "testDisconnect", "foo", 2);
+      const resp3Promise = cliComm._makeRequest(null, null, "methodAsync", "foo", 3);
+      assert.equal(await isLongerThan(resp2Promise, 250), true);
+
+      // Once we reconnect, the response should arrive.
+      await forwarder.connect();
+      assert.deepEqual(await resp2Promise, {name: 'testDisconnect', x: "foo", y: 2});
+      assert.deepEqual(await resp3Promise, {name: 'methodAsync', x: "foo", y: 3});
+
+      // Check that we saw the situation we were hoping to test.
+      assert.equal(failedSendCount, sendShouldFail ? 1 : 0, 'Expected to see a failed send');
     }
 
     it("should receive all server messages (small) in order when send doesn't fail", async function() {

test/server/customUtil.ts

@@ -20,7 +20,7 @@ export function addStatic(app: express.Express, rootDir?: string) {
           res.sendFile(req.params[0], {root:
                                         path.resolve(getAppRoot(), "static")}));
   app.use(express.static(rootDir || path.resolve(fixturesRoot, "sites"), {
-    setHeaders: (res) => {
+    setHeaders: (res: express.Response) => {
       res.set("Access-Control-Allow-Origin", "*");
     }
   }));

test/server/lib/DocApi.ts

@@ -17,8 +17,8 @@ import {
 import {delayAbort} from 'app/server/lib/serverUtils';
 import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
 import {delay} from 'bluebird';
-import * as bodyParser from 'body-parser';
 import {assert} from 'chai';
+import * as express from 'express';
 import FormData from 'form-data';
 import * as fse from 'fs-extra';
 import * as _ from 'lodash';
@@ -3786,7 +3786,7 @@ function testDocApi() {
 
       // TODO test retries on failure and slowness in a new test
       serving = await serveSomething(app => {
-        app.use(bodyParser.json());
+        app.use(express.json());
         app.post('/200', ({body}, res) => {
           successCalled.emit(body[0].A);
           res.sendStatus(200);

test/server/lib/Telemetry.ts

@@ -8,12 +8,24 @@ import {assert} from 'chai';
 import * as sinon from 'sinon';
 import {TestServer} from 'test/gen-server/apiUtils';
 import {configForUser} from 'test/gen-server/testUtils';
+import * as testUtils from 'test/server/testUtils';
 
 const chimpy = configForUser('Chimpy');
 const kiwi = configForUser('Kiwi');
 const anon = configForUser('Anonymous');
 
 describe('Telemetry', function() {
+  let oldEnv: testUtils.EnvironmentSnapshot;
+
+  before(async function() {
+    oldEnv = new testUtils.EnvironmentSnapshot();
+    process.env.TYPEORM_DATABASE = ':memory:';
+  });
+
+  after(function() {
+    oldEnv.restore();
+  });
+
   const variants: [GristDeploymentType, TelemetryLevel, PrefSource][] = [
     ['saas', 'off', 'environment-variable'],
     ['saas', 'limited', 'environment-variable'],

test/server/lib/Webhooks-Proxy.ts

@@ -1,8 +1,8 @@
 import {UserAPIImpl} from 'app/common/UserAPI';
 import {WebhookSubscription} from 'app/server/lib/DocApi';
 import axios from 'axios';
-import * as bodyParser from 'body-parser';
 import {assert} from 'chai';
+import * as express from 'express';
 import {tmpdir} from 'os';
 import * as path from 'path';
 import {createClient} from 'redis';
@@ -218,7 +218,7 @@ describe('Webhooks-Proxy', function () {
         this.timeout(30000);
         serving = await serveSomething(app => {
 
-          app.use(bodyParser.json());
+          app.use(express.json());
           app.post('/200', ({body}, res) => {
             successCalled.emit(body[0].A);
             res.sendStatus(200);

test/server/lib/helpers/TestProxyServer.ts

@@ -1,6 +1,5 @@
 import {serveSomething, Serving} from "test/server/customUtil";
-import * as bodyParser from "body-parser";
+import * as express from "express";
-import {Request, Response} from "express-serve-static-core";
 import axios from "axios";
 
 export class TestProxyServer {
@@ -27,8 +26,8 @@ export class TestProxyServer {
 
   private async _prepare(portNumber: number) {
     this._proxyServing = await serveSomething(app => {
-      app.use(bodyParser.json());
+      app.use(express.json());
-      app.all('*', async (req: Request, res: Response) => {
+      app.all('*', async (req: express.Request, res: express.Response) => {
         this._proxyCallsCounter += 1;
         let responseCode;
         try {

test/split-tests.js

@@ -1,6 +1,6 @@
 /**
  * This module handles splitting tests for parallelizing them. This module is imported by any run
- * of mocha, due by being listed in test/mocha.opts.
+ * of mocha, due by being listed in package.json.
  *
  * It only does anything if TEST_SPLITS is set, which must have the form "3-of-8".
  *