gristlabs_grist-core/test/server/lib/DocApi.ts

import {ActionSummary} from 'app/common/ActionSummary';
import {BulkColValues, UserAction} from 'app/common/DocActions';
import {arrayRepeat} from 'app/common/gutil';
import {DocState, UserAPIImpl} from 'app/common/UserAPI';
import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
import {CellValue, GristObjCode} from 'app/plugin/GristData';
import {
  applyQueryParameters,
  docApiUsagePeriods,
  docPeriodicApiUsageKey,
  getDocApiUsageKeysToIncr
} from 'app/server/lib/DocApi';
import * as log from 'app/server/lib/log';
import {exitPromise} from 'app/server/lib/serverUtils';
import {connectTestingHooks, TestingHooksClient} from 'app/server/lib/TestingHooks';
import axios, {AxiosResponse} from 'axios';
import {delay} from 'bluebird';
import * as bodyParser from 'body-parser';
import {assert} from 'chai';
import {ChildProcess, execFileSync, spawn} from 'child_process';
import * as FormData from 'form-data';
import * as fse from 'fs-extra';
import * as _ from 'lodash';
import * as LRUCache from 'lru-cache';
import * as moment from 'moment';
import fetch from 'node-fetch';
import {tmpdir} from 'os';
import * as path from 'path';
import {createClient, RedisClient} from 'redis';
import {configForUser} from 'test/gen-server/testUtils';
import {serveSomething, Serving} from 'test/server/customUtil';
import * as testUtils from 'test/server/testUtils';
import clone = require('lodash/clone');
import defaultsDeep = require('lodash/defaultsDeep');

const chimpy = configForUser('Chimpy');
const kiwi = configForUser('Kiwi');
const charon = configForUser('Charon');
const nobody = configForUser('Anonymous');
const support = configForUser('support');

// some doc ids
const docIds: {[name: string]: string} = {
  ApiDataRecordsTest: 'sample_7',
  Timesheets: 'sample_13',
  Bananas: 'sample_6',
  Antartic: 'sample_11'
};

// A testDir of the form grist_test_{USER}_{SERVER_NAME}
const username = process.env.USER || "nobody";
const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi`);

let dataDir: string;
let suitename: string;
let serverUrl: string;
let homeUrl: string;
let hasHomeApi: boolean;
let home: TestServer;
let docs: TestServer;
let userApi: UserAPIImpl;

describe('DocApi', function() {
  this.timeout(20000);
  testUtils.setTmpLogLevel('error');
  const oldEnv = clone(process.env);

  before(async function() {
    // Create the tmp dir removing any previous one
    await fse.remove(tmpDir);
    await fse.mkdirs(tmpDir);
    log.warn(`Test logs and data are at: ${tmpDir}/`);

    // Let's create a sqlite db that we can share with servers that run in other processes, hence
    // not an in-memory db. Running seed.ts directly might not take in account the most recent value
    // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
    // configuration (in-memory for instance). Spawning a process is one way to make sure that the
    // latest value prevail.
    process.env.TYPEORM_DATABASE = path.join(tmpDir, 'landing.db');
    const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
    execFileSync('node', [seed, 'init'], {
      env: process.env,
      stdio: 'inherit'
    });
  });

  after(() => {
    Object.assign(process.env, oldEnv);
  });

  /**
   * Doc api tests are run against three different setup:
   *  - a merged server: a single server serving both as a home and doc worker
   *  - two separated servers: requests are sent to a home server which then forward them to a doc worker
   *  - a doc worker: request are sent directly to the doc worker (note that even though it is not
   *    used for testing we starts anyway a home server, needed for setting up the test cases)
   *
   *  Future tests must be added within the testDocApi() function.
   */

  describe("should work with a merged server", async () => {
    setup('merged', async () => {
      home = docs = await startServer('home,docs');
      homeUrl = serverUrl = home.serverUrl;
      hasHomeApi = true;
    });
    testDocApi();
  });

  // the way these tests are written, non-merged server requires redis.
  if (process.env.TEST_REDIS_URL) {
    describe("should work with a home server and a docworker", async () => {
      setup('separated', async () => {
        home = await startServer('home');
        docs = await startServer('docs', home.serverUrl);
        homeUrl = serverUrl = home.serverUrl;
        hasHomeApi = true;
      });
      testDocApi();
    });

    describe("should work directly with a docworker", async () => {
      setup('docs', async () => {
        home = await startServer('home');
        docs = await startServer('docs', home.serverUrl);
        homeUrl = home.serverUrl;
        serverUrl = docs.serverUrl;
        hasHomeApi = false;
      });
      testDocApi();
    });
  }

  describe("QueryParameters", async () => {

    function makeExample() {
      return {
        id:    [   1,        2,       3,      7,       8,       9  ],
        color: ['red', 'yellow', 'white', 'blue', 'black', 'purple'],
        spin:  [ 'up',     'up',  'down', 'down',    'up',     'up'],
      };
    }

    it("supports ascending sort", async function() {
      assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['color']}, null), {
        id: [8, 7, 9, 1, 3, 2],
        color: ['black', 'blue', 'purple', 'red', 'white', 'yellow'],
        spin: ['up', 'down', 'up', 'up', 'down', 'up']
      });
    });

    it("supports descending sort", async function() {
      assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-id']}, null), {
        id: [9, 8, 7, 3, 2, 1],
        color: ['purple', 'black', 'blue', 'white', 'yellow', 'red'],
        spin: ['up', 'up', 'down', 'down', 'up', 'up'],
      });
    });

    it("supports multi-key sort", async function() {
      assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-spin', 'color']}, null), {
        id: [8, 9, 1, 2, 7, 3],
        color: ['black', 'purple', 'red', 'yellow', 'blue', 'white'],
        spin: ['up', 'up', 'up', 'up', 'down', 'down'],
      });
    });

    it("does not freak out sorting mixed data", async function() {
      const example = {
        id:    [   1,       2,       3,    4, 5,    6,          7,               8,     9],
        mixed: ['red', 'green', 'white', 2.5, 1, null, ['zing', 3] as any, 5, 'blue']
      };
      assert.deepEqual(applyQueryParameters(example, {sort: ['mixed']}, null), {
        mixed: [1, 2.5, 5, null, ['zing', 3] as any, 'blue', 'green', 'red', 'white'],
        id: [5, 4, 8, 6, 7, 9, 2, 1, 3],
      });
    });

    it("supports limit", async function() {
      assert.deepEqual(applyQueryParameters(makeExample(), {limit: 1}),
                       { id: [1], color: ['red'], spin: ['up'] });
    });

    it("supports sort and limit", async function() {
      assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-color'], limit: 2}, null),
                       { id: [2, 3], color: ['yellow', 'white'], spin: ['up', 'down'] });
    });
  });
});

// Contains the tests. This is where you want to add more test.
function testDocApi() {

  it("guesses types of new columns", async () => {
    const userActions = [
      ['AddTable', 'GuessTypes', []],
      // Make 5 blank columns of type Any
      ['AddColumn', 'GuessTypes', 'Date', {}],
      ['AddColumn', 'GuessTypes', 'DateTime', {}],
      ['AddColumn', 'GuessTypes', 'Bool', {}],
      ['AddColumn', 'GuessTypes', 'Numeric', {}],
      ['AddColumn', 'GuessTypes', 'Text', {}],
      // Add string values from which the initial type will be guessed
      ['AddRecord', 'GuessTypes', null, {
        Date: "1970-01-02",
        DateTime: "1970-01-02 12:00",
        Bool: "true",
        Numeric: "1.2",
        Text: "hello",
      }],
    ];
    const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
    assert.equal(resp.status, 200);

    // Check that the strings were parsed to typed values
    assert.deepEqual(
      (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/records`, chimpy)).data,
      {
        records: [
          {
            id: 1,
            fields: {
              Date: 24 * 60 * 60,
              DateTime: 36 * 60 * 60,
              Bool: true,
              Numeric: 1.2,
              Text: "hello",
            },
          },
        ],
      },
    );

    // Check the column types
    assert.deepEqual(
      (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/columns`, chimpy))
        .data.columns.map((col: any) => col.fields.type),
      ["Date", "DateTime:UTC", "Bool", "Numeric", "Text"],
    );
  });

  for (const mode of ['logged in', 'anonymous']) {
    for (const content of ['with content', 'without content']) {
      it(`POST /api/docs ${content} creates an unsaved doc when ${mode}`, async function() {
        const user = (mode === 'logged in') ? chimpy : nobody;
        const formData = new FormData();
        formData.append('upload', 'A,B\n1,2\n3,4\n', 'table1.csv');
        const config = defaultsDeep({headers: formData.getHeaders()}, user);
        let resp = await axios.post(`${serverUrl}/api/docs`,
          ...(content === 'with content' ? [formData, config] : [null, user]));
        assert.equal(resp.status, 200);
        const urlId = resp.data;
        if (mode === 'logged in') {
          assert.match(urlId, /^new~[^~]*~[0-9]+$/);
        } else {
          assert.match(urlId, /^new~[^~]*$/);
        }

        // Access information about that document should be sane for current user
        resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, user);
        assert.equal(resp.status, 200);
        assert.equal(resp.data.name, 'Untitled');
        assert.equal(resp.data.workspace.name, 'Examples & Templates');
        assert.equal(resp.data.access, 'owners');
        if (mode === 'anonymous') {
          resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, chimpy);
          assert.equal(resp.data.access, 'owners');
        } else {
          resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, charon);
          assert.equal(resp.status, 403);
          resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, nobody);
          assert.equal(resp.status, 403);
        }

        // content was successfully stored
        resp = await axios.get(`${serverUrl}/api/docs/${urlId}/tables/Table1/data`, user);
        if (content === 'with content') {
          assert.deepEqual(resp.data, { id: [ 1, 2 ], manualSort: [ 1, 2 ], A: [ 1, 3 ], B: [ 2, 4 ] });
        } else {
          assert.deepEqual(resp.data, { id: [], manualSort: [], A: [], B: [], C: [] });
        }
      });
    }
  }

  it("GET /docs/{did}/tables/{tid}/data retrieves data in column format", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data, {
      id: [1, 2, 3, 4],
      A: ['hello', '', '', ''],
      B: ['', 'world', '', ''],
      C: ['', '', '', ''],
      D: [null, null, null, null],
      E: ['HELLO', '', '', ''],
      manualSort: [1, 2, 3, 4]
    });
  });

  it("GET /docs/{did}/tables/{tid}/records retrieves data in records format", async function () {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data,
      {
        records:
          [
            {
              id: 1,
              fields: {
                A: 'hello',
                B: '',
                C: '',
                D: null,
                E: 'HELLO',
              },
            },
            {
              id: 2,
              fields: {
                A: '',
                B: 'world',
                C: '',
                D: null,
                E: '',
              },
            },
            {
              id: 3,
              fields: {
                A: '',
                B: '',
                C: '',
                D: null,
                E: '',
              },
            },
            {
              id: 4,
              fields: {
                A: '',
                B: '',
                C: '',
                D: null,
                E: '',
              },
            },
          ]
      });
  });

  it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () {
    let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data,
      {
        "records": [
          {
            "id": 1,
            "fields": {
              "A": null,
              "B": "Hi",
              "C": 1,
            },
            "errors": {
              "A": "ZeroDivisionError"
            }
          }
        ]
      }
    );

    // /data format for comparison: includes manualSort, gristHelper_Display, and ["E", "ZeroDivisionError"]
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data,
      {
        "id": [
          1
        ],
        "manualSort": [
          1
        ],
        "A": [
          [
            "E",
            "ZeroDivisionError"
          ]
        ],
        "B": [
          "Hi"
        ],
        "C": [
          1
        ],
        "gristHelper_Display": [
          "Hi"
        ]
      }
    );
  });

  it("GET /docs/{did}/tables/{tid}/columns retrieves columns", async function () {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data,
      {
        columns: [
          {
            id: 'A',
            fields: {
              colRef: 2,
              parentId: 1,
              parentPos: 1,
              type: 'Text',
              widgetOptions: '',
              isFormula: false,
              formula: '',
              label: 'A',
              untieColIdFromLabel: false,
              summarySourceCol: 0,
              displayCol: 0,
              visibleCol: 0,
              rules: null,
              recalcWhen: 0,
              recalcDeps: null
            }
          },
          {
            id: 'B',
            fields: {
              colRef: 3,
              parentId: 1,
              parentPos: 2,
              type: 'Text',
              widgetOptions: '',
              isFormula: false,
              formula: '',
              label: 'B',
              untieColIdFromLabel: false,
              summarySourceCol: 0,
              displayCol: 0,
              visibleCol: 0,
              rules: null,
              recalcWhen: 0,
              recalcDeps: null
            }
          },
          {
            id: 'C',
            fields: {
              colRef: 4,
              parentId: 1,
              parentPos: 3,
              type: 'Text',
              widgetOptions: '',
              isFormula: false,
              formula: '',
              label: 'C',
              untieColIdFromLabel: false,
              summarySourceCol: 0,
              displayCol: 0,
              visibleCol: 0,
              rules: null,
              recalcWhen: 0,
              recalcDeps: null
            }
          },
          {
            id: 'D',
            fields: {
              colRef: 5,
              parentId: 1,
              parentPos: 3,
              type: 'Any',
              widgetOptions: '',
              isFormula: true,
              formula: '',
              label: 'D',
              untieColIdFromLabel: false,
              summarySourceCol: 0,
              displayCol: 0,
              visibleCol: 0,
              rules: null,
              recalcWhen: 0,
              recalcDeps: null
            }
          },
          {
            id: 'E',
            fields: {
              colRef: 6,
              parentId: 1,
              parentPos: 4,
              type: 'Any',
              widgetOptions: '',
              isFormula: true,
              formula: '$A.upper()',
              label: 'E',
              untieColIdFromLabel: false,
              summarySourceCol: 0,
              displayCol: 0,
              visibleCol: 0,
              rules: null,
              recalcWhen: 0,
              recalcDeps: null
            }
          }
        ]
      }
    );
  });

  it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent doc", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /document not found/i);
  });

  it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent table", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /table not found/i);
  });

  it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent doc", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /document not found/i);
  });

  it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent table", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /table not found/i);
  });

  it("GET /docs/{did}/tables/{tid}/data supports filters", async function() {
    function makeQuery(filters: {[colId: string]: any[]}) {
      const query = "filter=" + encodeURIComponent(JSON.stringify(filters));
      return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy);
    }
    function checkResults(resp: AxiosResponse<any>, expectedData: any) {
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, expectedData);
    }

    checkResults(await makeQuery({B: ['world']}), {
      id: [2], A: [''], B: ['world'], C: [''], D: [null], E: [''], manualSort: [2],
    });

    // Can query by id
    checkResults(await makeQuery({id: [1]}), {
      id: [1], A: ['hello'], B: [''], C: [''], D: [null], E: ['HELLO'], manualSort: [1],
    });

    checkResults(await makeQuery({B: [''], A: ['']}), {
      id: [3, 4], A: ['', ''], B: ['', ''], C: ['', ''], D: [null, null], E: ['', ''], manualSort: [3, 4],
    });

    // Empty filter is equivalent to no filter and should return full data.
    checkResults(await makeQuery({}), {
      id: [1, 2, 3, 4],
      A: ['hello', '', '', ''],
      B: ['', 'world', '', ''],
      C: ['', '', '', ''],
      D: [null, null, null, null],
      E: ['HELLO', '', '', ''],
      manualSort: [1, 2, 3, 4]
    });

    // An impossible filter should succeed but return an empty set of rows.
    checkResults(await makeQuery({B: ['world'], C: ['Neptune']}), {
      id: [], A: [], B: [], C: [], D: [], E: [], manualSort: [],
    });

    // An invalid filter should return an error
    {
      const resp = await makeQuery({BadCol: ['']});
      assert.equal(resp.status, 400);
      assert.match(resp.data.error, /BadCol/);
    }

    {
      const resp = await makeQuery({B: 'world'} as any);
      assert.equal(resp.status, 400);
      assert.match(resp.data.error, /filter values must be arrays/);
    }
  });

  for (const mode of ['url', 'header']) {
    it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function() {
      function makeQuery(sort: string[]|null, limit: number|null) {
        const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
        const config = configForUser('chimpy');
        if (mode === 'url') {
          if (sort) { url.searchParams.append('sort', sort.join(',')); }
          if (limit) { url.searchParams.append('limit', String(limit)); }
        } else {
          if (sort)  { config.headers['x-sort'] = sort.join(','); }
          if (limit) { config.headers['x-limit'] = String(limit); }
        }
        return axios.get(url.href, config);
      }
      function checkResults(resp: AxiosResponse<any>, expectedData: any) {
        assert.equal(resp.status, 200);
        assert.deepEqual(resp.data, expectedData);
      }

      checkResults(await makeQuery(['-id'], null), {
        id: [4, 3, 2, 1],
        A: ['', '', '', 'hello'],
        B: ['', '', 'world', ''],
        C: ['', '', '', ''],
        D: [null, null, null, null],
        E: ['', '', '', 'HELLO'],
        manualSort: [4, 3, 2, 1]
      });

      checkResults(await makeQuery(['-id'], 2), {
        id: [4, 3],
        A: ['', ''],
        B: ['', ''],
        C: ['', ''],
        D: [null, null],
        E: ['', ''],
        manualSort: [4, 3]
      });
    });
  }

  it("GET /docs/{did}/tables/{tid}/data respects document permissions", async function() {
    // as not part of any group kiwi cannot fetch Timesheets
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, kiwi);
    assert.equal(resp.status, 403);
  });

  it("GET /docs/{did}/tables/{tid}/data returns matches /not found/ for bad table id", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /not found/);
  });

  it("POST /docs/{did}/apply applies user actions", async function() {
    const userActions = [
      ['AddTable', 'Foo', [{id: 'A'}, {id: 'B'}]],
      ['BulkAddRecord', 'Foo', [1, 2], {A: ["Santa", "Bob"], B: [1, 11]}]
    ];
    const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(
      (await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data,
      {id: [1, 2], A: ['Santa', 'Bob'], B: ['1', '11'], manualSort: [1, 2]});
  });

  it("POST /docs/{did}/apply respects document permissions", async function() {
    const userActions = [
      ['AddTable', 'FooBar', [{id: 'A'}]]
    ];
    let resp: AxiosResponse;

    // as a guest chimpy cannot edit Bananas
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Bananas}/apply`, userActions, chimpy);
    assert.equal(resp.status, 403);
    assert.deepEqual(resp.data, {error: 'No write access'});

    // check that changes did not apply
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.Bananas}/tables/FooBar/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /not found/);

    // as not in any group kiwi cannot edit TestDoc
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, kiwi);
    assert.equal(resp.status, 403);

    // check that changes did not apply
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/FooBar/data`, chimpy);
    assert.equal(resp.status, 404);
    assert.match(resp.data.error, /not found/);

  });

  it("POST /docs/{did}/tables/{tid}/data adds records", async function() {
    let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
      A: ['Alice', 'Felix'],
      B: [2, 22]
    }, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data, [3, 4]);
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
    assert.deepEqual(resp.data, {
      id: [1, 2, 3, 4],
      A: ['Santa', 'Bob', 'Alice', 'Felix'],
      B: ["1", "11", "2", "22"],
      manualSort: [1, 2, 3, 4]
    });
  });

  it("POST /docs/{did}/tables/{tid}/records adds records", async function() {
    let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
      records: [
        {fields: {A: 'John', B: 55}},
        {fields: {A: 'Jane', B: 0}},
      ]
    }, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data, {
      records: [
        {id: 5},
        {id: 6},
      ]
    });
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data,
      {
        records:
          [
            {
              id: 1,
              fields: {
                A: 'Santa',
                B: '1',
              },
            },
            {
              id: 2,
              fields: {
                A: 'Bob',
                B: '11',
              },
            },
            {
              id: 3,
              fields: {
                A: 'Alice',
                B: '2',
              },
            },
            {
              id: 4,
              fields: {
                A: 'Felix',
                B: '22',
              },
            },
            {
              id: 5,
              fields: {
                A: 'John',
                B: '55',
              },
            },
            {
              id: 6,
              fields: {
                A: 'Jane',
                B: '0',
              },
            },
          ]
      });
  });

  it("POST /docs/{did}/tables/{tid}/data/delete deletes records", async function() {
    let resp = await axios.post(
      `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data/delete`,
      [3, 4, 5, 6],
      chimpy,
    );
    assert.equal(resp.status, 200);
    assert.deepEqual(resp.data, null);
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
    assert.deepEqual(resp.data, {
      id: [1, 2],
      A: ['Santa', 'Bob'],
      B: ["1", "11"],
      manualSort: [1, 2]
    });

    // restore rows
    await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
      A: ['Alice', 'Felix'],
      B: [2, 22]
    }, chimpy);
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
    assert.deepEqual(resp.data, {
      id: [1, 2, 3, 4],
      A: ['Santa', 'Bob', 'Alice', 'Felix'],
      B: ["1", "11", "2", "22"],
      manualSort: [1, 2, 3, 4]
    });
  });

  function checkError(status: number, test: RegExp|object, resp: AxiosResponse<any>, message?: string) {
    assert.equal(resp.status, status);
    if (test instanceof RegExp) {
      assert.match(resp.data.error, test, message);
    } else {
      try {
      assert.deepEqual(resp.data, test, message);
      } catch(err) {
        console.log(JSON.stringify(resp.data));
        console.log(JSON.stringify(test));
        throw err;
      }
    }
  }

  it("parses strings in user actions", async () => {
    // Create a test document.
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
    const docUrl = `${serverUrl}/api/docs/${docId}`;
    const recordsUrl = `${docUrl}/tables/Table1/records`;

    // Make the column numeric, delete the other columns we don't care about
    await axios.post(`${docUrl}/apply`, [
      ['ModifyColumn', 'Table1', 'A', {type: 'Numeric'}],
      ['RemoveColumn', 'Table1', 'B'],
      ['RemoveColumn', 'Table1', 'C'],
    ], chimpy);

    // Add/update some records without and with string parsing
    // Specifically test:
    // 1. /apply, with an AddRecord
    // 2. POST  /records (BulkAddRecord)
    // 3. PATCH /records (BulkUpdateRecord)
    // Send strings that look like currency which need string parsing to become numbers
    for (const queryParams of ['?noparse=1', '']) {
      await axios.post(`${docUrl}/apply${queryParams}`, [
        ['AddRecord', 'Table1', null, {'A': '$1'}],
      ], chimpy);

      const response = await axios.post(`${recordsUrl}${queryParams}`,
        {
          records: [
            {fields: {'A': '$2'}},
            {fields: {'A': '$3'}},
          ]
        },
        chimpy);

      // Update $3 -> $4
      const rowId = response.data.records[1].id;
      await axios.patch(`${recordsUrl}${queryParams}`,
        {
          records: [
            {id: rowId, fields: {'A': '$4'}}
          ]
        },
        chimpy);
    }

    // Check the results
    const resp = await axios.get(recordsUrl, chimpy);
    assert.deepEqual(resp.data, {
        records:
          [
            // Without string parsing
            {id: 1, fields: {A: '$1'}},
            {id: 2, fields: {A: '$2'}},
            {id: 3, fields: {A: '$4'}},

            // With string parsing
            {id: 4, fields: {A: 1}},
            {id: 5, fields: {A: 2}},
            {id: 6, fields: {A: 4}},
          ]
      }
    );
  });

  describe("PUT /docs/{did}/tables/{tid}/records", async function() {
    it("should add or update records", async function() {
      // create sample document for testing
      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
      const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;

      async function check(records: AddOrUpdateRecord[], expectedTableData: BulkColValues, params: any={}) {
        const resp = await axios.put(url, {records}, {...chimpy, params});
        assert.equal(resp.status, 200);
        const table = await userApi.getTable(docId, "Table1");
        delete table.manualSort;
        delete table.C;
        assert.deepStrictEqual(table, expectedTableData);
      }

      // Add 3 new records, since the table is empty so nothing matches `requires`
      await check(
         [
            {
              require: {A: 1},
            },
            {
              // Since no record with A=2 is found, create a new record,
              // but `fields` overrides `require` for the value when creating,
              // so the new record has A=3
              require: {A: 2},
              fields: {A: 3},
            },
            {
              require: {A: 4},
              fields: {B: 5},
            },
          ],
        {id: [1, 2, 3], A: [1, 3, 4], B: [0, 0, 5]}
      );

      // Update all three records since they all match the `require` values here
      await check(
        [
            {
              // Does nothing
              require: {A: 1},
            },
            {
              // Changes A from 3 to 33
              require: {A: 3},
              fields: {A: 33},
            },
            {
              // Changes B from 5 to 6 in the third record where A=4
              require: {A: 4},
              fields: {B: 6},
            },
        ],
        {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}
      );

      // This would normally add a record, but noadd suppresses that
      await check([
          {
            require: {A: 100},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
        {noadd: "1"},
      );

      // This would normally update A from 1 to 11, bot noupdate suppresses that
      await check([
          {
            require: {A: 1},
            fields: {A: 11},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
        {noupdate: "1"},
      );

      // There are 2 records with B=0, update them both to B=1
      // Use onmany=all to specify that they should both be updated
      await check([
          {
            require: {B: 0},
            fields: {B: 1},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, 4], B: [1, 1, 6]},
        {onmany: "all"}
      );

      // In contrast to the above, the default behaviour for no value of onmany
      // is to only update the first matching record,
      // so only one of the records with B=1 is updated to B=2
      await check([
          {
            require: {B: 1},
            fields: {B: 2},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, 4], B: [2, 1, 6]},
      );

      // By default, strings in `require` and `fields` are parsed based on column type,
      // so these dollar amounts are treated as currency
      // and parsed as A=4 and A=44
      await check([
          {
            require: {A: "$4"},
            fields: {A: "$44"},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, 44], B: [2, 1, 6]},
      );

      // Turn off the default string parsing with noparse=1
      // Now we need A=44 to actually be a number to match,
      // A="$44" wouldn't match and would create a new record.
      // Because A="$55" isn't parsed, the raw string is stored in the table.
      await check([
          {
            require: {A: 44},
            fields: {A: "$55"},
          },
        ],
        {id: [1, 2, 3], A: [1, 33, "$55"], B: [2, 1, 6]},
        {noparse: 1}
      );

      await check([
          // First three records already exist and nothing happens
          {require: {A: 1}},
          {require: {A: 33}},
          {require: {A: "$55"}},
          // Without string parsing, A="$33" doesn't match A=33 and a new record is created
          {require: {A: "$33"}},
        ],
        {id: [1, 2, 3, 4], A: [1, 33, "$55", "$33"], B: [2, 1, 6, 0]},
        {noparse: 1}
      );

      // Checking that updating by `id` works.
      await check([
          {
            require: {id: 3},
            fields: {A: "66"},
          },
        ],
        {id: [1, 2, 3, 4], A: [1, 33, 66, "$33"], B: [2, 1, 6, 0]},
      );

      // allow_empty_require option with empty `require` updates all records
      await check([
          {
            require: {},
            fields: {A: 99, B: 99},
          },
        ],
        {id: [1, 2, 3, 4], A: [99, 99, 99, 99], B: [99, 99, 99, 99]},
        {allow_empty_require: "1", onmany: "all"},
      );
    });

    it("should 404 for missing tables", async () => {
      checkError(404, /Table not found "Bad_Foo_"/,
        await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/records`,
          {records: [{require: {id: 1}}]}, chimpy));
    });

    it("should 400 for missing columns", async () => {
      checkError(400, /Invalid column "no_such_column"/,
        await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
          {records: [{require: {no_such_column: 1}}]}, chimpy));
    });

    it("should 400 for an incorrect onmany parameter", async function() {
      checkError(400,
        /onmany parameter foo should be one of first,none,all/,
        await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
          {records: [{require: {id: 1}}]}, {...chimpy, params: {onmany: "foo"}}));
    });

    it("should 400 for an empty require without allow_empty_require", async function() {
      checkError(400,
        /require is empty but allow_empty_require isn't set/,
        await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
          {records: [{require: {}}]}, chimpy));
    });

    it("should validate request schema", async function() {
      const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
      const test = async (payload: any, error: { error: string, details: string }) => {
        const resp = await axios.put(url, payload, chimpy);
        checkError(400, error, resp);
      };
      await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
      await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
      await test({records: [{fields: {}}]},
        {
          error: 'Invalid payload',
          details: 'Error: ' +
            'body.records[0] is not a AddOrUpdateRecord; ' +
            'body.records[0].require is missing',
        });
      await test({records: [{require: {id: "1"}}]},
        {
          error: 'Invalid payload',
          details: 'Error: ' +
            'body.records[0] is not a AddOrUpdateRecord; ' +
            'body.records[0].require.id is not a number',
        });
    });
  });

  describe("POST /docs/{did}/tables/{tid}/records", async function() {
    it("POST should have good errors", async () => {
      checkError(404, /not found/,
                await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`,
                { A: ['Alice', 'Felix'], B: [2, 22] }, chimpy));

      checkError(400, /Invalid column "Bad"/,
                await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
                { A: ['Alice'], Bad: ['Monthy'] }, chimpy));

      // Other errors should also be maximally informative.
      checkError(400, /Error manipulating data/,
                await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
                { A: ['Alice'], B: null }, chimpy));
    });

    it("validates request schema", async function() {
      const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
      const test = async(payload: any, error: {error: string, details: string}) => {
        const resp = await axios.post(url, payload, chimpy);
        checkError(400, error, resp);
      };
      await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
      await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
      // All column types are allowed, except Arrays (or objects) without correct code.
      const testField = async (A: any) => {
        await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
                    'Error: body.records[0] is not a NewRecord; '+
                    'body.records[0].fields.A is not a CellValue; '+
                    'body.records[0].fields.A is none of number, '+
                    'string, boolean, null, 1 more; body.records[0].'+
                    'fields.A[0] is not a GristObjCode; body.records[0]'+
                    '.fields.A[0] is not a valid enum value'});
      };
      // test no code at all
      await testField([]);
      // test invalid code
      await testField(['ZZ']);
    });

    it("allows to create a blank record", async function() {
      // create sample document for testing
      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
      const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
      // Create two blank records
      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
      const resp = await axios.post(url, {records: [{}, { fields: {}}]}, chimpy);
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, { records : [{id: 1}, {id: 2}]});
    });

    it("allows to create partial records", async function() {
      // create sample document for testing
      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
      const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
      // create partial records
      const resp = await axios.post(url, {records: [{fields: { A: 1}}, { fields: {B: 2}}, {}]}, chimpy);
      assert.equal(resp.status, 200);
      const table = await userApi.getTable(docId, "Table1");
      delete table.manualSort;
      assert.deepStrictEqual(
        table,
        { id: [1, 2, 3], A: [1, null, null], B: [null, 2, null], C:[null, null, null]});
    });

    it("allows CellValue as a field", async function() {
      // create sample document
      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
      const docId = await userApi.newDoc({ name : 'PostTest'}, wid);
      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
      const testField = async(A?: CellValue, message?: string) =>{
        const resp = await axios.post(url, {records: [{ fields: { A } }]}, chimpy);
        assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
      };
      // test allowed types for a field
      await testField(1); // ints
      await testField(1.2); // floats
      await testField("string"); // strings
      await testField(true); // true and false
      await testField(false);
      await testField(null); // null
      // encoded values (though not all make sense)
      for (const code of [
          GristObjCode.List,
          GristObjCode.Dict,
          GristObjCode.DateTime,
          GristObjCode.Date,
          GristObjCode.Skip,
          GristObjCode.Censored,
          GristObjCode.Reference,
          GristObjCode.ReferenceList,
          GristObjCode.Exception,
          GristObjCode.Pending,
          GristObjCode.Unmarshallable,
          GristObjCode.Versions,
      ]) {
        await testField([code]);
      }
    });
  });

  it("POST /docs/{did}/tables/{tid}/data respects document permissions", async function() {
    let resp: AxiosResponse;
    const data = {
      A: ['Alice', 'Felix'],
      B: [2, 22]
    };

    // as a viewer charon cannot edit TestDoc
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
    assert.equal(resp.status, 403);
    assert.deepEqual(resp.data, {error: 'No write access'});

    // as not part of any group kiwi cannot edit TestDoc
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
    assert.equal(resp.status, 403);
    assert.deepEqual(resp.data, {error: 'No view access'});

    // check that TestDoc did not change
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
    assert.deepEqual(resp.data, {
      id: [1, 2, 3, 4],
      A: ['Santa', 'Bob', 'Alice', 'Felix'],
      B: ["1", "11", "2", "22"],
      manualSort: [1, 2, 3, 4]
    });
  });

  describe("PATCH /docs/{did}/tables/{tid}/records", function() {
    it("updates records", async function () {
      let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
        records: [
          {
            id: 1,
            fields: {
              A: 'Father Christmas',
            },
          },
        ],
      }, chimpy);
      assert.equal(resp.status, 200);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
      // check that rest of the data is left unchanged
      assert.deepEqual(resp.data, {
        records:
          [
            {
              id: 1,
              fields: {
                A: 'Father Christmas',
                B: '1',
              },
            },
            {
              id: 2,
              fields: {
                A: 'Bob',
                B: '11',
              },
            },
            {
              id: 3,
              fields: {
                A: 'Alice',
                B: '2',
              },
            },
            {
              id: 4,
              fields: {
                A: 'Felix',
                B: '22',
              },
            },
          ]
      });
    });

    it("validates request schema", async function() {
      const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
      async function failsWithError(payload: any, error: { error: string, details?: string }){
        const resp = await axios.patch(url, payload, chimpy);
        checkError(400, error, resp);
      }

      await failsWithError({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});

      await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});

      await failsWithError({records: []}, {error: 'Invalid payload', details:
                  'Error: body.records[0] is not a Record; body.records[0] is not an object'});

      await failsWithError({records: [{}]}, {error: 'Invalid payload', details:
                  'Error: body.records[0] is not a Record\n    '+
                  'body.records[0].id is missing\n    '+
                  'body.records[0].fields is missing'});

      await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details:
                  'Error: body.records[0] is not a Record\n' +
                  '    body.records[0].id is not a number\n' +
                  '    body.records[0].fields is missing'});

      await failsWithError(
        {records: [{id: 1, fields: {A : 1}}, {id: 2, fields: {B: 3}}]},
        {error: 'PATCH requires all records to have same fields'});

      // Test invalid object codes
      const fieldIsNotValid = async (A: any) => {
        await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
                    'Error: body.records[0] is not a Record; '+
                    'body.records[0].fields.A is not a CellValue; '+
                    'body.records[0].fields.A is none of number, '+
                    'string, boolean, null, 1 more; body.records[0].'+
                    'fields.A[0] is not a GristObjCode; body.records[0]'+
                    '.fields.A[0] is not a valid enum value'});
      };
      await fieldIsNotValid([]);
      await fieldIsNotValid(['ZZ']);
    });

    it("allows CellValue as a field", async function() {
      // create sample document for testing
      const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
      const docId = await userApi.newDoc({ name : 'PatchTest'}, wid);
      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
      // create record for patching
      const id = (await axios.post(url, { records: [{}] }, chimpy)).data.records[0].id;
      const testField = async(A?: CellValue, message?: string) =>{
        const resp = await axios.patch(url, {records: [{ id, fields: { A } }]}, chimpy);
        assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
      };
      await testField(1);
      await testField(1.2);
      await testField("string");
      await testField(true);
      await testField(false);
      await testField(null);
      for (const code of [
          GristObjCode.List,
          GristObjCode.Dict,
          GristObjCode.DateTime,
          GristObjCode.Date,
          GristObjCode.Skip,
          GristObjCode.Censored,
          GristObjCode.Reference,
          GristObjCode.ReferenceList,
          GristObjCode.Exception,
          GristObjCode.Pending,
          GristObjCode.Unmarshallable,
          GristObjCode.Versions,
      ]) {
        await testField([code]);
      }
    });
  });

  describe("PATCH /docs/{did}/tables/{tid}/data", function() {

    it("updates records", async function() {
      let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
        id: [1],
        A: ['Santa Klaus'],
      }, chimpy);
      assert.equal(resp.status, 200);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
      // check that rest of the data is left unchanged
      assert.deepEqual(resp.data, {
        id: [1, 2, 3, 4],
        A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
        B: ["1", "11", "2", "22"],
        manualSort: [1, 2, 3, 4]
      });

    });

    it("throws 400 for invalid row ids", async function() {

      // combination of valid and invalid ids fails
      let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
        id: [1, 5],
        A: ['Alice', 'Felix']
      }, chimpy);
      assert.equal(resp.status, 400);
      assert.match(resp.data.error, /Invalid row id 5/);

      // only invalid ids also fails
      resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
        id: [10, 5],
        A: ['Alice', 'Felix']
      }, chimpy);
      assert.equal(resp.status, 400);
      assert.match(resp.data.error, /Invalid row id 10/);

      // check that changes related to id 1 did not apply
      assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
        id: [1, 2, 3, 4],
        A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
        B: ["1", "11", "2", "22"],
        manualSort: [1, 2, 3, 4]
      });
    });

    it("throws 400 for invalid column", async function() {
      const resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
        id: [1],
        A: ['Alice'],
        C: ['Monthy']
      }, chimpy);
      assert.equal(resp.status, 400);
      assert.match(resp.data.error, /Invalid column "C"/);
    });

    it("respects document permissions", async function() {
      let resp: AxiosResponse;
      const data = {
        id: [1],
        A: ['Santa'],
      };

      // check data
      assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
        id: [1, 2, 3, 4],
        A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
        B: ["1", "11", "2", "22"],
        manualSort: [1, 2, 3, 4]
      });

      // as a viewer charon cannot patch TestDoc
      resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
      assert.equal(resp.status, 403);
      assert.deepEqual(resp.data, {error: 'No write access'});

      // as not part of any group kiwi cannot patch TestDoc
      resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
      assert.equal(resp.status, 403);
      assert.deepEqual(resp.data, {error: 'No view access'});

      // check that changes did not apply
      assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
        id: [1, 2, 3, 4],
        A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
        B: ["1", "11", "2", "22"],
        manualSort: [1, 2, 3, 4]
      });
    });

  });

  describe('attachments', function() {
    it("POST /docs/{did}/attachments adds attachments", async function() {
      let formData = new FormData();
      formData.append('upload', 'foobar', "hello.doc");
      formData.append('upload', '123456', "world.jpg");
      let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, chimpy));
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, [1, 2]);

      // Another upload gets the next number.
      formData = new FormData();
      formData.append('upload', 'abcdef', "hello.png");
      resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, chimpy));
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, [3]);
    });

    it("GET /docs/{did}/attachments/{id} returns attachment metadata", async function() {
      const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2`, chimpy);
      assert.equal(resp.status, 200);
      assert.include(resp.data, {fileName: "world.jpg", fileSize: 6});
      assert.match(resp.data.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
    });

    it("GET /docs/{did}/attachments/{id}/download downloads attachment contents", async function() {
      const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
        {...chimpy, responseType: 'arraybuffer'});
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
      assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
      assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
      assert.deepEqual(resp.data, Buffer.from('123456'));
    });

    it("GET /docs/{did}/attachments/{id}/download works after doc shutdown", async function() {
      // Check that we can download when ActiveDoc isn't currently open.
      let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/force-reload`, null, chimpy);
      assert.equal(resp.status, 200);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
        {...chimpy, responseType: 'arraybuffer'});
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
      assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
      assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
      assert.deepEqual(resp.data, Buffer.from('123456'));
    });

    it("GET /docs/{did}/attachments/{id}... returns 404 when attachment not found", async function() {
      let resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22`, chimpy);
      checkError(404, /Attachment not found: 22/, resp);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo`, chimpy);
      checkError(404, /Attachment not found: moo/, resp);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22/download`, chimpy);
      checkError(404, /Attachment not found: 22/, resp);
      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo/download`, chimpy);
      checkError(404, /Attachment not found: moo/, resp);
    });

    it("POST /docs/{did}/attachments produces reasonable errors", async function() {
      // Check that it produces reasonable errors if we try to use it with non-form-data
      let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, [4, 5, 6], chimpy);
      assert.equal(resp.status, 415);     // Wrong content-type

      // Check for an error if there is no data included.
      const formData = new FormData();
      resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, chimpy));
      assert.equal(resp.status, 400);
      // TODO The error here is "stream ended unexpectedly", which isn't really reasonable.
    });

    it("POST/GET /docs/{did}/attachments respect document permissions", async function() {
      const formData = new FormData();
      formData.append('upload', 'xyzzz', "wrong.png");
      let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, kiwi));
      checkError(403, /No view access/, resp);

      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3`, kiwi);
      checkError(403, /No view access/, resp);

      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3/download`, kiwi);
      checkError(403, /No view access/, resp);
    });

    it("POST /docs/{did}/attachments respects untrusted content-type only if valid", async function() {
      const formData = new FormData();
      formData.append('upload', 'xyz', {filename: "foo", contentType: "application/pdf"});
      formData.append('upload', 'abc', {filename: "hello.png", contentType: "invalid/content-type"});
      formData.append('upload', 'def', {filename: "world.doc", contentType: "text/plain\nbad-header: 1\n\nEvil"});
      let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, chimpy));
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, [4, 5, 6]);

      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/4/download`, chimpy);
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.headers['content-type'], 'application/pdf');    // A valid content-type is respected
      assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="foo.pdf"');
      assert.deepEqual(resp.data, 'xyz');

      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/5/download`, chimpy);
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.headers['content-type'], 'image/png');    // Did not pay attention to invalid header
      assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="hello.png"');
      assert.deepEqual(resp.data, 'abc');

      resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/6/download`, chimpy);
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.headers['content-type'], 'application/msword');    // Another invalid header ignored
      assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.doc"');
      assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
      assert.deepEqual(resp.headers['bad-header'], undefined);   // Attempt to hack in more headers didn't work
      assert.deepEqual(resp.data, 'def');
    });

    it("POST /docs/{did}/attachments/updateUsed updates timeDeleted on metadata", async function() {
      const wid = await getWorkspaceId(userApi, 'Private');
      const docId = await userApi.newDoc({name: 'TestDoc2'}, wid);

      // Apply the given user actions,
      // POST to /attachments/updateUsed
      // Check that Table1 and _grist_Attachments contain the expected rows
      async function check(
        actions: UserAction[],
        userData: { id: number, Attached: any }[],
        metaData: { id: number, deleted: boolean }[],
      ) {
        const docUrl = `${serverUrl}/api/docs/${docId}`;

        let resp = await axios.post(`${docUrl}/apply`, actions, chimpy);
        assert.equal(resp.status, 200);

        resp = await axios.post(`${docUrl}/attachments/updateUsed`, null, chimpy);
        assert.equal(resp.status, 200);

        resp = await axios.get(`${docUrl}/tables/Table1/records`, chimpy);
        const actualUserData = resp.data.records.map(
          ({id, fields: {Attached}}: ApiRecord) =>
          ({id, Attached})
        );
        assert.deepEqual(actualUserData, userData);

        resp = await axios.get(`${docUrl}/tables/_grist_Attachments/records`, chimpy);
        const actualMetaData = resp.data.records.map(
          ({id, fields: {timeDeleted}}: ApiRecord) =>
          ({id, deleted: Boolean(timeDeleted)})
        );
        assert.deepEqual(actualMetaData, metaData);
      }

      // Set up the document and initial data.
      await check(
        [
          ["AddColumn", "Table1", "Attached", {type: "Attachments"}],
          ["BulkAddRecord", "Table1", [1, 2], {Attached: [['L', 1], ['L', 2, 3]]}],
          // There's no actual attachments here but that doesn't matter
          ["BulkAddRecord", "_grist_Attachments", [1, 2, 3], {}],
        ],
        [
          {id: 1, Attached: ['L', 1]},
          {id: 2, Attached: ['L', 2, 3]},
        ],
        [
          {id: 1, deleted: false},
          {id: 2, deleted: false},
          {id: 3, deleted: false},
        ],
      );

      // Remove the record containing ['L', 2, 3], so the metadata for 2 and 3 now says deleted
      await check(
        [["RemoveRecord", "Table1", 2]],
        [
          {id: 1, Attached: ['L', 1]},
        ],
        [
          {id: 1, deleted: false},
          {id: 2, deleted: true},  // deleted here
          {id: 3, deleted: true},  // deleted here
        ],
      );

      // Add back a reference to attacument 2 to test 'undeletion', plus some junk values
      await check(
        [["BulkAddRecord", "Table1", [3, 4, 5], {Attached: [null, "foo", ['L', 2, 2, 4, 4, 5]]}]],
        [
          {id: 1, Attached: ['L', 1]},
          {id: 3, Attached: null},
          {id: 4, Attached: "foo"},
          {id: 5, Attached: ['L', 2, 2, 4, 4, 5]},
        ],
        [
          {id: 1, deleted: false},
          {id: 2, deleted: false},  // undeleted here
          {id: 3, deleted: true},
        ],
      );

      // Remove the whole column to test what happens when there's no Attachment columns
      await check(
        [["RemoveColumn", "Table1", "Attached"]],
        [
          {id: 1, Attached: undefined},
          {id: 3, Attached: undefined},
          {id: 4, Attached: undefined},
          {id: 5, Attached: undefined},
        ],
        [
          {id: 1, deleted: true},  // deleted here
          {id: 2, deleted: true},  // deleted here
          {id: 3, deleted: true},
        ],
      );

      // Test performance with a large number of records and attachments.
      // The maximum value of numRecords that doesn't return a 413 error is about 18,000.
      // In that case it took about 5.7 seconds to apply the initial user actions (i.e. add the records),
      // 0.3 seconds to call updateUsed once, and 0.1 seconds to call it again immediately after.
      // That last time roughly measures the time taken to do the SQL query
      // without having to apply any user actions after to update timeDeleted.
      // 10,000 records is a compromise so that tests aren't too slow.
      const numRecords = 10000;
      const attachmentsPerRecord = 4;
      const totalUsedAttachments = numRecords * attachmentsPerRecord;  // 40,000 attachments referenced in user data
      const totalAttachments = totalUsedAttachments * 1.1;  // 44,000 attachment IDs listed in metadata
      const attachedValues = _.chunk(_.range(1, totalUsedAttachments + 1), attachmentsPerRecord)
        .map(arr => ['L', ...arr]);
      await check(
        [
          // Reset the state: add back the removed column and delete the previously added data
          ["AddColumn", "Table1", "Attached", {type: "Attachments"}],
          ["BulkRemoveRecord", "Table1", [1, 3, 4, 5]],
          ["BulkRemoveRecord", "_grist_Attachments", [1, 2, 3]],
          ["BulkAddRecord", "Table1", arrayRepeat(numRecords, null), {Attached: attachedValues}],
          ["BulkAddRecord", "_grist_Attachments", arrayRepeat(totalAttachments, null), {}],
        ],
        attachedValues.map((Attached, index) => ({id: index + 1, Attached})),
        _.range(totalAttachments).map(index => ({id: index + 1, deleted: index >= totalUsedAttachments})),
      );
    });

    it("POST /docs/{did}/attachments/removeUnused removes unused attachments", async function() {
      const wid = await getWorkspaceId(userApi, 'Private');
      const docId = await userApi.newDoc({name: 'TestDoc3'}, wid);
      const docUrl = `${serverUrl}/api/docs/${docId}`;

      const formData = new FormData();
      formData.append('upload', 'foobar', "hello.doc");
      formData.append('upload', '123456', "world.jpg");
      formData.append('upload', 'foobar', "hello2.doc");
      let resp = await axios.post(`${docUrl}/attachments`, formData,
        defaultsDeep({headers: formData.getHeaders()}, chimpy));
      assert.equal(resp.status, 200);
      assert.deepEqual(resp.data, [1, 2, 3]);

      async function checkAttachmentIds(ids: number[]) {
        resp = await axios.get(
          `${docUrl}/tables/_grist_Attachments/records`,
          chimpy,
        );
        assert.equal(resp.status, 200);
        assert.deepEqual(resp.data.records.map((r: any) => r.id), ids);
      }

      resp = await axios.patch(
        `${docUrl}/tables/_grist_Attachments/records`,
        {
          records: [
            {id: 1, fields: {timeDeleted: Date.now() / 1000 - 8 * 24 * 60 * 60}},  // 8 days ago, i.e. expired
            {id: 2, fields: {timeDeleted: Date.now() / 1000 - 6 * 24 * 60 * 60}},  // 6 days ago, i.e. not expired
          ]
        },
        chimpy,
      );
      assert.equal(resp.status, 200);
      await checkAttachmentIds([1, 2, 3]);

      // Remove the expired attachment (1) by force-reloading, so it removes it during shutdown.
      // It has a duplicate (3) that hasn't expired and thus isn't removed,
      // although they share the same fileIdent and row in _gristsys_Files.
      // So for now only the metadata is removed.
      resp = await axios.post(`${docUrl}/force-reload`, null, chimpy);
      assert.equal(resp.status, 200);
      await checkAttachmentIds([2, 3]);
      resp = await axios.post(`${docUrl}/attachments/verifyFiles`, null, chimpy);
      assert.equal(resp.status, 200);

      // Remove the not expired attachments (2 and 3).
      // We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachments.
      resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
      assert.equal(resp.status, 200);
      await checkAttachmentIds([]);
    });

  });

  it("GET /docs/{did}/download serves document", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, chimpy);
    assert.equal(resp.status, 200);
    assert.match(resp.data, /grist_Tables_column/);
  });

  it("GET /docs/{did}/download respects permissions", async function() {
    // kiwi has no access to TestDoc
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, kiwi);
    assert.equal(resp.status, 403);
    assert.notMatch(resp.data, /grist_Tables_column/);
  });

  it("GET /docs/{did}/download/csv serves CSV-encoded document", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
    assert.equal(resp.status, 200);
    assert.equal(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');

    const resp2 = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Foo`, chimpy);
    assert.equal(resp2.status, 200);
    assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
  });

  it("GET /docs/{did}/download/csv respects permissions", async function() {
    // kiwi has no access to TestDoc
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);
    assert.equal(resp.status, 403);
    assert.notEqual(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
  });

  it("GET /docs/{did}/download/csv returns 404 if tableId is invalid", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=MissingTableId`, chimpy);
    assert.equal(resp.status, 404);
    assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' });
  });

  it("GET /docs/{did}/download/csv returns 404 if viewSectionId is invalid", async function() {
    const resp = await axios.get(
      `${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1&viewSection=9999`, chimpy);
    assert.equal(resp.status, 404);
    assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' });
  });

  it("GET /docs/{did}/download/csv returns 400 if tableId is missing", async function() {
    const resp = await axios.get(
      `${serverUrl}/api/docs/${docIds.TestDoc}/download/csv`, chimpy);
    assert.equal(resp.status, 400);
    assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
  });

  it('POST /workspaces/{wid}/import handles empty filenames', async function() {
    if (!process.env.TEST_REDIS_URL) { this.skip(); }
    const worker1 = await userApi.getWorkerAPI('import');
    const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
    const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
    const uploadId1 = await worker1.upload(fakeData1, '.grist');
    const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
                                configForUser('Chimpy'));
    assert.equal(resp.status, 200);
    assert.equal(resp.data.title, 'Untitled upload');
    assert.equal(typeof resp.data.id, 'string');
    assert.notEqual(resp.data.id, '');
  });

  it("document is protected during upload-and-import sequence", async function() {
    if (!process.env.TEST_REDIS_URL) { this.skip(); }
    // Prepare an API for a different user.
    const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
      headers: {Authorization: 'Bearer api_key_for_kiwi'},
      fetch : fetch as any,
      newFormData: () => new FormData() as any,
      logger: log
    });
    // upload something for Chimpy and something else for Kiwi.
    const worker1 = await userApi.getWorkerAPI('import');
    const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
    const uploadId1 = await worker1.upload(fakeData1, 'upload.grist');
    const worker2 = await kiwiApi.getWorkerAPI('import');
    const fakeData2 = await testUtils.readFixtureDoc('Favorite_Films.grist');
    const uploadId2 = await worker2.upload(fakeData2, 'upload2.grist');

    // Check that kiwi only has access to their own upload.
    let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id;
    let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
                                configForUser('Kiwi'));
    assert.equal(resp.status, 403);
    assert.deepEqual(resp.data, {error: "access denied"});

    resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2},
                            configForUser('Kiwi'));
    assert.equal(resp.status, 200);

    // Check that chimpy has access to their own upload.
    wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
    resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
                            configForUser('Chimpy'));
    assert.equal(resp.status, 200);
  });

  it('limits parallel requests', async function() {
    // Launch 30 requests in parallel and see how many are honored and how many
    // return 429s.  The timing of this test is a bit delicate.  We close the doc
    // to increase the odds that results won't start coming back before all the
    // requests have passed authorization.  May need to do something more sophisticated
    // if this proves unreliable.
    await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
    const reqs = [...Array(30).keys()].map(
      i => axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy));
    const responses = await Promise.all(reqs);
    assert.lengthOf(responses.filter(r => r.status === 200), 10);
    assert.lengthOf(responses.filter(r => r.status === 429), 20);
  });

  it('allows forced reloads', async function() {
    let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
    assert.equal(resp.status, 200);
    // Check that support cannot force a reload.
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, support);
    assert.equal(resp.status, 403);
    if (hasHomeApi) {
      // Check that support can force a reload through housekeeping api.
      resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, support);
      assert.equal(resp.status, 200);
      // Check that regular user cannot force a reload through housekeeping api.
      resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
      assert.equal(resp.status, 403);
    }
  });

  it('allows assignments', async function() {
    let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, chimpy);
    assert.equal(resp.status, 200);
    // Check that support cannot force an assignment.
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, support);
    assert.equal(resp.status, 403);
    if (hasHomeApi) {
      // Check that support can force an assignment through housekeeping api.
      resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, support);
      assert.equal(resp.status, 200);
      // Check that regular user cannot force an assignment through housekeeping api.
      resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, chimpy);
      assert.equal(resp.status, 403);
    }
  });

  it('honors urlIds', async function() {
    // Make a document with a urlId
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1);
    try {
      // Make sure an edit made by docId is visible when accessed via docId or urlId
      let resp = await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
        A: ['Apple'], B: [99]
      }, chimpy);
      resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[0], 'Apple');
      resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[0], 'Apple');
      // Make sure an edit made by urlId is visible when accessed via docId or urlId
      resp = await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
        A: ['Orange'], B: [42]
      }, chimpy);
      resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[1], 'Orange');
      resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[1], 'Orange');
    } finally {
      await userApi.deleteDoc(doc1);
    }
  });

  it('filters urlIds by org', async function() {
    // Make two documents with same urlId
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
      headers: {Authorization: 'Bearer api_key_for_chimpy'},
      fetch : fetch as any,
      newFormData: () => new FormData() as any,
      logger: log
    });
    const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
    const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2);
    try {
      // Place a value in "docs" doc
      await axios.post(`${serverUrl}/o/docs/api/docs/urlid/tables/Table1/data`, {
        A: ['Apple'], B: [99]
      }, chimpy);
      // Place a value in "nasa" doc
      await axios.post(`${serverUrl}/o/nasa/api/docs/urlid/tables/Table1/data`, {
        A: ['Orange'], B: [99]
      }, chimpy);
      // Check the values made it to the right places
      let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[0], 'Apple');
      resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[0], 'Orange');
    } finally {
      await userApi.deleteDoc(doc1);
      await nasaApi.deleteDoc(doc2);
    }
  });

  it('allows docId access to any document from merged org', async function() {
    // Make two documents
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
      headers: {Authorization: 'Bearer api_key_for_chimpy'},
      fetch : fetch as any,
      newFormData: () => new FormData() as any,
      logger: log
    });
    const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
    const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2);
    try {
      // Should fail to write to a document in "docs" from "nasa" url
      let resp = await axios.post(`${serverUrl}/o/nasa/api/docs/${doc1}/tables/Table1/data`, {
        A: ['Apple'], B: [99]
      }, chimpy);
      assert.equal(resp.status, 404);
      // Should successfully write to a document in "nasa" from "docs" url
      resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/tables/Table1/data`, {
        A: ['Orange'], B: [99]
      }, chimpy);
      assert.equal(resp.status, 200);
      // Should fail to write to a document in "nasa" from "pr" url
      resp = await axios.post(`${serverUrl}/o/pr/api/docs/${doc2}/tables/Table1/data`, {
        A: ['Orange'], B: [99]
      }, chimpy);
      assert.equal(resp.status, 404);
    } finally {
      await userApi.deleteDoc(doc1);
      await nasaApi.deleteDoc(doc2);
    }
  });

  it("GET /docs/{did}/replace replaces one document with another", async function() {
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
    const doc2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
    const doc3 = await userApi.newDoc({name: 'testdoc2'}, ws1);
    await userApi.updateDocPermissions(doc2, {users: {'kiwi@getgrist.com': 'editors'}});
    await userApi.updateDocPermissions(doc3, {users: {'kiwi@getgrist.com': 'viewers'}});
    try {
      // Put some material in doc3
      let resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc3}/tables/Table1/data`, {
        A: ['Orange']
      }, chimpy);
      assert.equal(resp.status, 200);

      // Kiwi can replace doc2 with doc3
      resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
        sourceDocId: doc3
      }, kiwi);
      assert.equal(resp.status, 200);
      resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
      assert.equal(resp.data.A[0], 'Orange');

      // Kiwi can't replace doc1 with doc3, no write access to doc1
      resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc1}/replace`, {
        sourceDocId: doc3
      }, kiwi);
      assert.equal(resp.status, 403);

      // Kiwi can't replace doc2 with doc1, no read access to doc1
      resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
        sourceDocId: doc1
      }, kiwi);
      assert.equal(resp.status, 403);
    } finally {
      await userApi.deleteDoc(doc1);
      await userApi.deleteDoc(doc2);
    }
  });

  it("GET /docs/{did}/snapshots retrieves a list of snapshots", async function() {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/snapshots`, chimpy);
    assert.equal(resp.status, 200);
    assert.isAtLeast(resp.data.snapshots.length, 1);
    assert.hasAllKeys(resp.data.snapshots[0], ['docId', 'lastModified', 'snapshotId']);
  });

  it("POST /docs/{did}/states/remove removes old states", async function() {
    // Check doc has plenty of states.
    let resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
    assert.equal(resp.status, 200);
    const states: DocState[] = resp.data.states;
    assert.isAbove(states.length, 5);

    // Remove all but 3.
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 3}, chimpy);
    assert.equal(resp.status, 200);
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
    assert.equal(resp.status, 200);
    assert.lengthOf(resp.data.states, 3);
    assert.equal(resp.data.states[0].h, states[0].h);
    assert.equal(resp.data.states[1].h, states[1].h);
    assert.equal(resp.data.states[2].h, states[2].h);

    // Remove all but 1.
    resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 1}, chimpy);
    assert.equal(resp.status, 200);
    resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
    assert.equal(resp.status, 200);
    assert.lengthOf(resp.data.states, 1);
    assert.equal(resp.data.states[0].h, states[0].h);
  });

  it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function() {
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
    const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
    const doc1 = userApi.getDocAPI(docId1);
    const doc2 = userApi.getDocAPI(docId2);

    // Stick some content in column A so it has a defined type
    // so diffs are smaller and simpler.
    await doc2.addRows('Table1', {A: [0]});

    let comp = await doc1.compareDoc(docId2);
    assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary']);
    assert.equal(comp.summary, 'unrelated');
    assert.equal(comp.parent, null);
    assert.hasAllKeys(comp.left, ['n', 'h']);
    assert.hasAllKeys(comp.right, ['n', 'h']);
    assert.equal(comp.left.n, 1);
    assert.equal(comp.right.n, 2);

    await doc1.replace({sourceDocId: docId2});

    comp = await doc1.compareDoc(docId2);
    assert.equal(comp.summary, 'same');
    assert.equal(comp.left.n, 2);
    assert.deepEqual(comp.left, comp.right);
    assert.deepEqual(comp.left, comp.parent);
    assert.equal(comp.details, undefined);

    comp = await doc1.compareDoc(docId2, { detail: true });
    assert.deepEqual(comp.details, {
      leftChanges: { tableRenames: [], tableDeltas: {} },
      rightChanges: { tableRenames: [], tableDeltas: {} }
    });

    await doc1.addRows('Table1', {A: [1]});
    comp = await doc1.compareDoc(docId2);
    assert.equal(comp.summary, 'left');
    assert.equal(comp.left.n, 3);
    assert.equal(comp.right.n, 2);
    assert.deepEqual(comp.right, comp.parent);
    assert.equal(comp.details, undefined);

    comp = await doc1.compareDoc(docId2, { detail: true });
    assert.deepEqual(comp.details!.rightChanges,
                     { tableRenames: [], tableDeltas: {} });
    const addA1: ActionSummary = {
      tableRenames: [],
      tableDeltas: { Table1: {
        updateRows: [],
        removeRows: [],
        addRows: [ 2 ],
        columnDeltas: {
          A: { [2]: [null, [1]] },
          manualSort: { [2]: [null, [2]] },
        },
        columnRenames: [],
      } }
    };
    assert.deepEqual(comp.details!.leftChanges, addA1);

    await doc2.addRows('Table1', {A: [1]});
    comp = await doc1.compareDoc(docId2);
    assert.equal(comp.summary, 'both');
    assert.equal(comp.left.n, 3);
    assert.equal(comp.right.n, 3);
    assert.equal(comp.parent!.n, 2);
    assert.equal(comp.details, undefined);

    comp = await doc1.compareDoc(docId2, { detail: true });
    assert.deepEqual(comp.details!.leftChanges, addA1);
    assert.deepEqual(comp.details!.rightChanges, addA1);

    await doc1.replace({sourceDocId: docId2});

    comp = await doc1.compareDoc(docId2);
    assert.equal(comp.summary, 'same');
    assert.equal(comp.left.n, 3);
    assert.deepEqual(comp.left, comp.right);
    assert.deepEqual(comp.left, comp.parent);
    assert.equal(comp.details, undefined);

    comp = await doc1.compareDoc(docId2, { detail: true });
    assert.deepEqual(comp.details, {
      leftChanges: { tableRenames: [], tableDeltas: {} },
      rightChanges: { tableRenames: [], tableDeltas: {} }
    });

    await doc2.addRows('Table1', {A: [2]});
    comp = await doc1.compareDoc(docId2);
    assert.equal(comp.summary, 'right');
    assert.equal(comp.left.n, 3);
    assert.equal(comp.right.n, 4);
    assert.deepEqual(comp.left, comp.parent);
    assert.equal(comp.details, undefined);

    comp = await doc1.compareDoc(docId2, { detail: true });
    assert.deepEqual(comp.details!.leftChanges,
                     { tableRenames: [], tableDeltas: {} });
    const addA2: ActionSummary = {
      tableRenames: [],
      tableDeltas: { Table1: {
        updateRows: [],
        removeRows: [],
        addRows: [ 3 ],
        columnDeltas: {
          A: { [3]: [null, [2]] },
          manualSort: { [3]: [null, [3]] },
        },
        columnRenames: [],
      } }
    };
    assert.deepEqual(comp.details!.rightChanges, addA2);
  });

  it("GET /docs/{did}/compare tracks changes within a doc", async function() {
    // Create a test document.
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
    const doc = userApi.getDocAPI(docId);

    // Give the document some history.
    await doc.addRows('Table1', {A: ['a1'], B: ['b1']});
    await doc.addRows('Table1', {A: ['a2'], B: ['b2']});
    await doc.updateRows('Table1', {id: [1], A: ['A1']});

    // Examine the most recent change, from HEAD~ to HEAD.
    let comp = await doc.compareVersion('HEAD~', 'HEAD');
    assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
    assert.equal(comp.summary, 'right');
    assert.deepEqual(comp.parent, comp.left);
    assert.notDeepEqual(comp.parent, comp.right);
    assert.hasAllKeys(comp.left, ['n', 'h']);
    assert.hasAllKeys(comp.right, ['n', 'h']);
    assert.equal(comp.left.n, 3);
    assert.equal(comp.right.n, 4);
    assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
    assert.deepEqual(comp.details!.rightChanges, {
      tableRenames: [],
      tableDeltas: {
        Table1: {
          updateRows: [1],
          removeRows: [],
          addRows: [],
          columnDeltas: {
            A: { [1]: [['a1'], ['A1']] }
          },
          columnRenames: [],
        }
      }
    });

    // Check we get the same result with actual hashes.
    assert.notMatch(comp.left.h, /HEAD/);
    assert.notMatch(comp.right.h, /HEAD/);
    const comp2 = await doc.compareVersion(comp.left.h, comp.right.h);
    assert.deepEqual(comp, comp2);

    // Check that comparing the HEAD with itself shows no changes.
    comp = await doc.compareVersion('HEAD', 'HEAD');
    assert.equal(comp.summary, 'same');
    assert.deepEqual(comp.parent, comp.left);
    assert.deepEqual(comp.parent, comp.right);
    assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
    assert.deepEqual(comp.details!.rightChanges, { tableRenames: [], tableDeltas: {} });

    // Examine the combination of the last two changes.
    comp = await doc.compareVersion('HEAD~~', 'HEAD');
    assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
    assert.equal(comp.summary, 'right');
    assert.deepEqual(comp.parent, comp.left);
    assert.notDeepEqual(comp.parent, comp.right);
    assert.hasAllKeys(comp.left, ['n', 'h']);
    assert.hasAllKeys(comp.right, ['n', 'h']);
    assert.equal(comp.left.n, 2);
    assert.equal(comp.right.n, 4);
    assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
    assert.deepEqual(comp.details!.rightChanges, {
      tableRenames: [],
      tableDeltas: {
        Table1: {
          updateRows: [1],
          removeRows: [],
          addRows: [2],
          columnDeltas: {
            A: { [1]: [['a1'], ['A1']],
                 [2]: [null, ['a2']] },
            B: { [2]: [null, ['b2']] },
            manualSort: { [2]: [null, [2]] },
          },
          columnRenames: [],
        }
      }
    });
  });

  it('doc worker endpoints ignore any /dw/.../ prefix', async function() {
    const docWorkerUrl = docs.serverUrl;
    let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 200);
    assert.containsAllKeys(resp.data, ['A', 'B', 'C']);

    resp = await axios.get(`${docWorkerUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 200);
    assert.containsAllKeys(resp.data, ['A', 'B', 'C']);

    if (docWorkerUrl !== homeUrl) {
      resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
      assert.equal(resp.status, 200);
      assert.containsAllKeys(resp.data, ['A', 'B', 'C']);

      resp = await axios.get(`${homeUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
      assert.equal(resp.status, 404);
    }
  });

  it("POST /docs/{did}/tables/{tid}/_subscribe validates inputs", async function () {
    async function check(requestBody: any, status: number, error: string) {
      const resp = await axios.post(
        `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
        requestBody, chimpy
      );
      assert.equal(resp.status, status);
      assert.deepEqual(resp.data, {error});
    }

    await check({}, 400, "eventTypes must be a non-empty array");
    await check({eventTypes: 0}, 400, "eventTypes must be a non-empty array");
    await check({eventTypes: []}, 400, "eventTypes must be a non-empty array");
    await check({eventTypes: ["foo"]}, 400, "Allowed values in eventTypes are: add,update");
    await check({eventTypes: ["add"]}, 400, "Bad request: url required");
    await check({eventTypes: ["add"], url: "https://evil.com"}, 403, "Provided url is forbidden");
    await check({eventTypes: ["add"], url: "http://example.com"}, 403, "Provided url is forbidden");  // not https
    await check({eventTypes: ["add"], url: "https://example.com", isReadyColumn: "bar"}, 404, `Column not found "bar"`);
  });

  it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs", async function() {
    const subscribeResponse = await axios.post(
      `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
      {eventTypes: ["add"], url: "https://example.com"}, chimpy
    );
    assert.equal(subscribeResponse.status, 200);
    const {triggerId, unsubscribeKey, webhookId} = subscribeResponse.data;

    async function check(requestBody: any, status: number, responseBody: any) {
      const resp = await axios.post(
        `${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_unsubscribe`,
        requestBody, chimpy
      );
      assert.equal(resp.status, status);
      if (status !== 200) {
        responseBody = {error: responseBody};
      }
      assert.deepEqual(resp.data, responseBody);
    }

    await check({triggerId: 999}, 404, `Trigger not found "999"`);
    await check({triggerId, webhookId: "foo"}, 404, `Webhook not found "foo"`);
    await check({triggerId, webhookId}, 400, 'Bad request: id and unsubscribeKey both required');
    await check({triggerId, webhookId, unsubscribeKey: "foo"}, 401, 'Wrong unsubscribeKey');

    // Actually unsubscribe
    await check({triggerId, webhookId, unsubscribeKey}, 200, {success: true});

    // Trigger is now deleted!
    await check({triggerId, webhookId, unsubscribeKey}, 404, `Trigger not found "${triggerId}"`);
  });

  describe("Daily API Limit", () => {
    let redisClient: RedisClient;

    before(async function() {
      if (!process.env.TEST_REDIS_URL) { this.skip(); }
      redisClient = createClient(process.env.TEST_REDIS_URL);
    });

    it("limits daily API usage", async function() {
      // Make a new document in a test product with a low daily limit
      const api = makeUserApi('testdailyapilimit');
      const workspaceId = await getWorkspaceId(api, 'TestDailyApiLimitWs');
      const docId = await api.newDoc({name: 'TestDoc1'}, workspaceId);
      const max = testDailyApiLimitFeatures.baseMaxApiUnitsPerDocumentPerDay;

      for (let i = 1; i <= max + 2; i++) {
        let success = true;
        try {
          // Make some doc request so that it fails or succeeds
          await api.getTable(docId, "Table1");
        } catch (e) {
          success = false;
        }

        // Usually the first `max` requests should succeed and the rest should fail having exceeded the daily limit.
        // If a new minute starts in the middle of the requests, an extra request will be allowed for that minute.
        // If a new day starts in the middle of the requests, this test will fail.
        if (success) {
          assert.isAtMost(i, max + 1);
        } else {
          assert.isAtLeast(i, max + 1);
        }
      }
    });

    it("limits daily API usage and sets the correct keys in redis", async function() {
      // Make a new document in a free team site, currently the only real product which limits daily API usage.
      const freeTeamApi = makeUserApi('freeteam');
      const workspaceId = await getWorkspaceId(freeTeamApi, 'FreeTeamWs');
      const docId = await freeTeamApi.newDoc({name: 'TestDoc2'}, workspaceId);
      // Rather than making 5000 requests, set high counts directly for the current and next daily and hourly keys
      const used = 999999;
      let m = moment.utc();
      const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
      const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
      const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
      const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
      await redisClient.multi()
        .set(currentDay, String(used))
        .set(currentHour, String(used))
        .set(nextDay, String(used))
        .set(nextHour, String(used))
        .execAsync();

      // Make 9 requests. The first 4 should succeed by fitting into the allocation for the minute.
      // (Free team plans get 5000 requests per day, and 5000/24/60 ~= 3.47 which is rounded up to 4)
      // The last request should fail. Don't check the middle 4 in case we're on the boundary of a minute.
      for (let i = 1; i <= 9; i++) {
        const last = i === 9;
        m = moment.utc();  // get this before delaying to calculate accurate keys below
        const response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy);
        // Allow time for redis to be updated.
        await delay(100);
        if (i <= 4) {
          assert.equal(response.status, 200);
          // Keys of the periods we expect to be incremented.
          // For the first request, the server's usage cache is empty and it hasn't seen the redis values.
          // So it thinks there hasn't been any usage and increments the current day/hour.
          // After that it increments the next day/hour.
          // We're only checking this for the first 4 requests
          // because once the limit is exceeded the counts aren't incremented.
          const first = i === 1;
          const day = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[0], m);
          const hour = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[1], m);
          const minute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);

          if (!first) {
            // The first request takes longer to serve because the document gets loaded,
            // so only check the TTL (which gets set before request processing starts) on subsequent requests.
            assert.deepEqual(
              await redisClient.multi()
                .ttl(minute)
                .ttl(hour)
                .ttl(day)
                .execAsync(),
              [
                2 * 60,
                2 * 60 * 60,
                2 * 60 * 60 * 24,
              ],
            );
          }

          assert.deepEqual(
            await redisClient.multi()
              .get(minute)
              .get(hour)
              .get(day)
              .execAsync(),
            [
              String(i),
              String(used + (first ? 1 : i - 1)),
              String(used + (first ? 1 : i - 1)),
            ],
          );
        }

        if (last) {
          assert.equal(response.status, 429);
          assert.deepEqual(response.data, {error: `Exceeded daily limit for document ${docId}`});
        }
      }
    });

    it("correctly allocates API requests based on the day, hour, and minute", async function() {
      const m = moment.utc("1999-12-31T23:59:59Z");
      const docId = "myDocId";
      const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
      const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
      const currentMinute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
      const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
      const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
      assert.equal(currentDay, `doc-myDocId-periodicApiUsage-1999-12-31`);
      assert.equal(currentHour, `doc-myDocId-periodicApiUsage-1999-12-31T23`);
      assert.equal(currentMinute, `doc-myDocId-periodicApiUsage-1999-12-31T23:59`);
      assert.equal(nextDay, `doc-myDocId-periodicApiUsage-2000-01-01`);
      assert.equal(nextHour, `doc-myDocId-periodicApiUsage-2000-01-01T00`);

      const usage = new LRUCache<string, number>({max: 1024});
      function check(expected: string[] | undefined) {
        assert.deepEqual(getDocApiUsageKeysToIncr(docId, usage, dailyMax, m), expected);
      }

      const dailyMax = 5000;
      const hourlyMax = 209;  // 5000/24    ~= 208.33
      const minuteMax = 4;    // 5000/24/60 ~= 3.47
      check([currentDay, currentHour, currentMinute]);
      usage.set(currentDay, dailyMax - 1);
      check([currentDay, currentHour, currentMinute]);
      usage.set(currentDay, dailyMax);
      check([nextDay, currentHour, currentMinute]);  // used up daily allocation
      usage.set(currentHour, hourlyMax - 1);
      check([nextDay, currentHour, currentMinute]);
      usage.set(currentHour, hourlyMax);
      check([nextDay, nextHour, currentMinute]);  // used up hourly allocation
      usage.set(currentMinute, minuteMax - 1);
      check([nextDay, nextHour, currentMinute]);
      usage.set(currentMinute, minuteMax);
      check(undefined);  // used up minutely allocation
      usage.set(currentDay, 0);
      check([currentDay, currentHour, currentMinute]);
      usage.set(currentDay, dailyMax);
      usage.set(currentHour, 0);
      check([nextDay, currentHour, currentMinute]);
    });

    after(async function() {
      if (!process.env.TEST_REDIS_URL) { this.skip(); }
      await redisClient.quitAsync();
    });
  });

  describe("Webhooks", () => {
    let serving: Serving;  // manages the test webhook server

    let requests: WebhookRequests;

    let receivedLastEvent: Promise<void>;

    // Requests corresponding to adding 200 rows, sent in two batches of 100
    const expected200AddEvents = [
      _.range(100).map(i => ({
        id: 9 + i, manualSort: 9 + i, A3: 200 + i, B3: true,
      })),
      _.range(100).map(i => ({
        id: 109 + i, manualSort: 109 + i, A3: 300 + i, B3: true,
      })),
    ];

    // Every event is sent to three webhook URLs which differ by the subscribed eventTypes
    // Each request is an array of one or more events.
    // Multiple events caused by the same action bundle get batched into a single request.
    const expectedRequests: WebhookRequests = {
      "add": [
        [{id: 1, A: 1, B: true, C: null, manualSort: 1}],
        [{id: 2, A: 4, B: true, C: null, manualSort: 2}],

        // After isReady (B) went to false and then true again
        // we treat this as creation even though it's really an update
        [{id: 2, A: 7, B: true, C: null, manualSort: 2}],

        // From the big applies
        [{id: 3, A3: 13, B3: true, manualSort: 3},
         {id: 5, A3: 15, B3: true, manualSort: 5}],
        [{id: 7, A3: 18, B3: true, manualSort: 7}],

        ...expected200AddEvents,
      ],
      "update": [
        [{id: 2, A: 8, B: true, C: null, manualSort: 2}],

        // From the big applies
        [{id: 1, A3: 101, B3: true, manualSort: 1}],
      ],
      "add,update": [
        // add
        [{id: 1, A: 1, B: true, C: null, manualSort: 1}],
        [{id: 2, A: 4, B: true, C: null, manualSort: 2}],
        [{id: 2, A: 7, B: true, C: null, manualSort: 2}],

        // update
        [{id: 2, A: 8, B: true, C: null, manualSort: 2}],

        // from the big applies
        [{id: 1, A3: 101, B3: true, manualSort: 1},  // update
         {id: 3, A3: 13, B3: true, manualSort: 3},   // add
         {id: 5, A3: 15, B3: true, manualSort: 5}],  // add

        [{id: 7, A3: 18, B3: true, manualSort: 7}],  // add

        ...expected200AddEvents,
      ]
    };

    let redisMonitor: any;
    let redisCalls: any[];

    before(async function() {
      if (!process.env.TEST_REDIS_URL) { this.skip(); }
      requests = {
        "add,update": [],
        "add": [],
        "update": [],
      };

      let resolveReceivedLastEvent: () => void;
      receivedLastEvent = new Promise<void>(r => {
        resolveReceivedLastEvent = r;
      });

      // TODO test retries on failure and slowness in a new test
      serving = await serveSomething(app => {
        app.use(bodyParser.json());
        app.post('/:eventTypes', async ({body, params: {eventTypes}}, res) => {
          requests[eventTypes as keyof WebhookRequests].push(body);
          res.sendStatus(200);
          if (
            _.flattenDeep(_.values(requests)).length >=
            _.flattenDeep(_.values(expectedRequests)).length
          ) {
            resolveReceivedLastEvent();
          }
        });
      }, webhooksTestPort);

      redisCalls = [];
      redisMonitor = createClient(process.env.TEST_REDIS_URL);
      redisMonitor.monitor();
      redisMonitor.on("monitor", (_time: any, args: any, _rawReply: any) => {
        redisCalls.push(args);
      });
    });

    after(async function() {
      if (!process.env.TEST_REDIS_URL) { this.skip(); }
      serving.shutdown();
      await redisMonitor.quitAsync();
    });

    it("delivers expected payloads from combinations of changes, with retrying and batching", async function() {
      // Create a test document.
      const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
      const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
      const doc = userApi.getDocAPI(docId);

      // For some reason B is turned into Numeric even when given bools
      await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
        ['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
      ], chimpy);

      // Make a webhook for every combination of event types
      const subscribeResponses = [];
      const webhookIds: Record<string, string> = {};
      for (const eventTypes of [
        ["add"],
        ["update"],
        ["add", "update"],
      ]) {
        const {data, status} = await axios.post(
          `${serverUrl}/api/docs/${docId}/tables/Table1/_subscribe`,
          {eventTypes, url: `${serving.url}/${eventTypes}`, isReadyColumn: "B"}, chimpy
        );
        assert.equal(status, 200);
        subscribeResponses.push(data);
        webhookIds[data.webhookId] = String(eventTypes);
      }

      // Add and update some rows, trigger some events
      // Values of A where B is true and thus the record is ready are [1, 4, 7, 8]
      // So those are the values seen in expectedEvents
      await doc.addRows("Table1", {
        A: [1, 2],
        B: [true, false], // 1  is ready, 2 is not ready yet
      });
      await doc.updateRows("Table1", {id: [2], A: [3]});  // still not ready
      await doc.updateRows("Table1", {id: [2], A: [4], B: [true]});  // ready!
      await doc.updateRows("Table1", {id: [2], A: [5], B: [false]});  // not ready again
      await doc.updateRows("Table1", {id: [2], A: [6]});  // still not ready
      await doc.updateRows("Table1", {id: [2], A: [7], B: [true]});  // ready!
      await doc.updateRows("Table1", {id: [2], A: [8]});  // still ready!

      // The end result here is additions for column A (now A3) with values [13, 15, 18]
      // and an update for 101
      await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
        ['BulkAddRecord', 'Table1', [3, 4, 5, 6], {A: [9, 10, 11, 12], B: [true, true, false, false]}],
        ['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], {
          A: [101, 102, 13, 14, 15, 16],
          B: [true, false, true, false, true, false],
        }],

        ['RenameColumn', 'Table1', 'A', 'A3'],
        ['RenameColumn', 'Table1', 'B', 'B3'],

        ['RenameTable', 'Table1', 'Table12'],

        // FIXME a double rename A->A2->A3 doesn't seem to get summarised correctly
        // ['RenameColumn', 'Table12', 'A2', 'A3'],
        // ['RenameColumn', 'Table12', 'B2', 'B3'],

        ['RemoveColumn', 'Table12', 'C'],
      ], chimpy);

      // FIXME record changes after a RenameTable in the same bundle
      //  don't appear in the action summary
      await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
        ['AddRecord', 'Table12', 7, {A3: 17, B3: false}],
        ['UpdateRecord', 'Table12', 7, {A3: 18, B3: true}],

        ['AddRecord', 'Table12', 8, {A3: 19, B3: true}],
        ['UpdateRecord', 'Table12', 8, {A3: 20, B3: false}],

        ['AddRecord', 'Table12', 9, {A3: 20, B3: true}],
        ['RemoveRecord', 'Table12', 9],
      ], chimpy);

      // Add 200 rows. These become the `expected200AddEvents`
      await doc.addRows("Table12", {
        A3: _.range(200, 400),
        B3: arrayRepeat(200, true),
      });

      await receivedLastEvent;

      // Unsubscribe
      await Promise.all(subscribeResponses.map(async subscribeResponse => {
        const unsubscribeResponse = await axios.post(
          `${serverUrl}/api/docs/${docId}/tables/Table12/_unsubscribe`,
          subscribeResponse, chimpy
        );
        assert.equal(unsubscribeResponse.status, 200);
        assert.deepEqual(unsubscribeResponse.data, {success: true});
      }));

      // Further changes should generate no events because the triggers are gone
      await doc.addRows("Table12", {
        A3: [88, 99],
        B3: [true, false],
      });

      assert.deepEqual(requests, expectedRequests);

      // Check that the events were all pushed to the redis queue
      const queueRedisCalls = redisCalls.filter(args => args[1] === "webhook-queue-" + docId);
      const redisPushes = _.chain(queueRedisCalls)
        .filter(args => args[0] === "rpush")          // Array<["rpush", key, ...events: string[]]>
        .flatMap(args => args.slice(2))               // events: string[]
        .map(JSON.parse)                              // events: WebhookEvent[]
        .groupBy('id')                                // {[webHookId: string]: WebhookEvent[]}
        .mapKeys((_value, key) => webhookIds[key])    // {[eventTypes: 'add'|'update'|'add,update']: WebhookEvent[]}
        .mapValues(group => _.map(group, 'payload'))  // {[eventTypes: 'add'|'update'|'add,update']: RowRecord[]}
        .value();
      const expectedPushes = _.mapValues(expectedRequests, value => _.flatten(value));
      assert.deepEqual(redisPushes, expectedPushes);

      // Check that the events were all removed from the redis queue
      const redisTrims = queueRedisCalls.filter(args => args[0] === "ltrim")
        .map(([,, start, end]) => {
          assert.equal(end, '-1');
          start = Number(start);
          assert.isTrue(start > 0);
          return start;
      });
      const expectedTrims = Object.values(redisPushes).map(value => value.length);
      assert.equal(
        _.sum(redisTrims),
        _.sum(expectedTrims),
      );

    });
  });

  // PLEASE ADD MORE TESTS HERE
}

interface WebhookRequests {
  add: object[][];
  update: object[][];
  "add,update": object[][];
}

function setup(name: string, cb: () => Promise<void>) {
  let api: UserAPIImpl;

  before(async function() {
    suitename = name;
    dataDir = path.join(tmpDir, `${suitename}-data`);
    await fse.mkdirs(dataDir);
    await setupDataDir(dataDir);
    await cb();

    // create TestDoc as an empty doc into Private workspace
    userApi = api = makeUserApi('docs-1');
    const wid = await getWorkspaceId(api, 'Private');
    docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid);
  });

  after(async function() {
    // remove TestDoc
    await api.deleteDoc(docIds.TestDoc);
    delete docIds.TestDoc;

    // stop all servers
    await home.stop();
    await docs.stop();
  });
}

function makeUserApi(org: string) {
  return new UserAPIImpl(`${home.serverUrl}/o/${org}`, {
    headers: {Authorization: 'Bearer api_key_for_chimpy'},
    fetch: fetch as any,
    newFormData: () => new FormData() as any,
    logger: log
  });
}

async function getWorkspaceId(api: UserAPIImpl, name: string) {
  const workspaces = await api.getOrgWorkspaces('current');
  return workspaces.find((w) => w.name === name)!.id;
}

async function startServer(serverTypes: string, _homeUrl?: string): Promise<TestServer> {
  const server = new TestServer(serverTypes);
  await server.start(_homeUrl);
  return server;
}

const webhooksTestPort = 34365;

class TestServer {
  public testingSocket: string;
  public testingHooks: TestingHooksClient;
  public serverUrl: string;
  public stopped = false;

  private _server: ChildProcess;
  private _exitPromise: Promise<number|string>;

  constructor(private _serverTypes: string) {}

  public async start(_homeUrl?: string) {

    // put node logs into files with meaningful name that relate to the suite name and server type
    const fixedName = this._serverTypes.replace(/,/, '_');
    const nodeLogPath = path.join(tmpDir, `${suitename}-${fixedName}-node.log`);
    const nodeLogFd = await fse.open(nodeLogPath, 'a');
    const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd;

    // use a path for socket that relates to suite name and server types
    this.testingSocket = path.join(tmpDir, `${suitename}-${fixedName}.socket`);

    // env
    const env = {
      GRIST_DATA_DIR: dataDir,
      GRIST_INST_DIR: tmpDir,
      GRIST_SERVERS: this._serverTypes,
      // with port '0' no need to hard code a port number (we can use testing hooks to find out what
      // port server is listening on).
      GRIST_PORT: '0',
      GRIST_TESTING_SOCKET: this.testingSocket,
      GRIST_DISABLE_S3: 'true',
      REDIS_URL: process.env.TEST_REDIS_URL,
      APP_HOME_URL: _homeUrl,
      ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
      ...process.env
    };

    const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
    this._server = spawn('node', [main, '--testingHooks'], {
      env,
      stdio: ['inherit', serverLog, serverLog]
    });

    this._exitPromise = exitPromise(this._server);

    // Try to be more helpful when server exits by printing out the tail of its log.
    this._exitPromise.then((code) => {
        if (this._server.killed) { return; }
        log.error("Server died unexpectedly, with code", code);
        const output = execFileSync('tail', ['-30', nodeLogPath]);
        log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`);
      })
      .catch(() => undefined);

    await this._waitServerReady(30000);
    log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
  }

  public async stop() {
    if (this.stopped) { return; }
    log.info("Stopping node server: " + this._serverTypes);
    this.stopped = true;
    this._server.kill();
    this.testingHooks.close();
    await this._exitPromise;
  }

  public async isServerReady(): Promise<boolean> {
    // Let's wait for the testingSocket to be created, then get the port the server is listening on,
    // and then do an api check. This approach allow us to start server with GRIST_PORT set to '0',
    // which will listen on first available port, removing the need to hard code a port number.
    try {

      // wait for testing socket
      while (!(await fse.pathExists(this.testingSocket))) {
        await delay(200);
      }

      // create testing hooks and get own port
      this.testingHooks = await connectTestingHooks(this.testingSocket);
      const port: number = await this.testingHooks.getOwnPort();
      this.serverUrl = `http://localhost:${port}`;

      // wait for check
      return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
    } catch (err) {
      return false;
    }
  }


  private async _waitServerReady(ms: number) {
    // It's important to clear the timeout, because it can prevent node from exiting otherwise,
    // which is annoying when running only this test for debugging.
    let timeout: any;
    const maxDelay = new Promise((resolve) => {
      timeout = setTimeout(resolve, 30000);
    });
    try {
      await Promise.race([
        this.isServerReady(),
        this._exitPromise.then(() => { throw new Error("Server exited while waiting for it"); }),
        maxDelay,
      ]);
    } finally {
      clearTimeout(timeout);
    }
  }
}

async function setupDataDir(dir: string) {
  // we'll be serving Hello.grist content for various document ids, so let's make copies of it in
  // tmpDir
  await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Timesheets + '.grist'));
  await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Bananas + '.grist'));
  await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Antartic + '.grist'));

  await testUtils.copyFixtureDoc(
    'ApiDataRecordsTest.grist',
    path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
}