Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity
a91d493ffc
gristlabs_grist-core/test/server/lib/DocApi.ts
Dmitry S a91d493ffc (core) Fix issue with 'UNEXPECTED ORDER OF CALLBACKS' in Client.ts. 
Summary:
- Substantial refactoring of the logic when the server fails to send some
  messages to a client.
- Add seqId numbers to server messages to ensure reliable order.
- Add a needReload flag in clientConnect for a clear indication whent the
  browser client needs to reload the app.
- Reproduce some potential failure scenarios in a test case (some of which
  previously could have led to incorrectly ordered messages).
- Convert other Comm tests to typescript.
- Tweak logging of Comm and Client to be slightly more concise (in particular,
  avoid logging sessionId)

Note that despite the big refactoring, this only addresses a fairly rare
situation, with websocket failures while server is trying to send to the
client. It includes no improvements for failures while the client is sending to
the server.

(I looked for an existing library that would take care of these issues. A relevant article I found is https://docs.microsoft.com/en-us/azure/azure-web-pubsub/howto-develop-reliable-clients, but it doesn't include a library for both ends, and is still in review. Other libraries with similar purposes did not inspire enough confidence.)

Test Plan: New test cases, which reproduce some previously problematic scenarios.

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3470
2022-06-16 23:51:14 -04:00

2915 lines
111 KiB
TypeScript
Raw Blame History

import {ActionSummary} from 'app/common/ActionSummary';
import {BulkColValues, UserAction} from 'app/common/DocActions';
import {arrayRepeat} from 'app/common/gutil';
import {DocState, UserAPIImpl} from 'app/common/UserAPI';
import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
import {CellValue, GristObjCode} from 'app/plugin/GristData';
import {
applyQueryParameters,
docApiUsagePeriods,
docPeriodicApiUsageKey,
getDocApiUsageKeysToIncr
} from 'app/server/lib/DocApi';
import * as log from 'app/server/lib/log';
import {exitPromise} from 'app/server/lib/serverUtils';
import {connectTestingHooks, TestingHooksClient} from 'app/server/lib/TestingHooks';
import axios, {AxiosResponse} from 'axios';
import {delay} from 'bluebird';
import * as bodyParser from 'body-parser';
import {assert} from 'chai';
import {ChildProcess, execFileSync, spawn} from 'child_process';
import * as FormData from 'form-data';
import * as fse from 'fs-extra';
import * as _ from 'lodash';
import * as LRUCache from 'lru-cache';
import * as moment from 'moment';
import fetch from 'node-fetch';
import {tmpdir} from 'os';
import * as path from 'path';
import {createClient, RedisClient} from 'redis';
import {configForUser} from 'test/gen-server/testUtils';
import {serveSomething, Serving} from 'test/server/customUtil';
import * as testUtils from 'test/server/testUtils';
import clone = require('lodash/clone');
import defaultsDeep = require('lodash/defaultsDeep');
const chimpy = configForUser('Chimpy');
const kiwi = configForUser('Kiwi');
const charon = configForUser('Charon');
const nobody = configForUser('Anonymous');
const support = configForUser('support');
// some doc ids
const docIds: {[name: string]: string} = {
ApiDataRecordsTest: 'sample_7',
Timesheets: 'sample_13',
Bananas: 'sample_6',
Antartic: 'sample_11'
};
// A testDir of the form grist_test_{USER}_{SERVER_NAME}
const username = process.env.USER || "nobody";
const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi`);
let dataDir: string;
let suitename: string;
let serverUrl: string;
let homeUrl: string;
let hasHomeApi: boolean;
let home: TestServer;
let docs: TestServer;
let userApi: UserAPIImpl;
describe('DocApi', function() {
this.timeout(20000);
testUtils.setTmpLogLevel('error');
const oldEnv = clone(process.env);
before(async function() {
// Clear redis test database if redis is in use.
if (process.env.TEST_REDIS_URL) {
const cli = createClient(process.env.TEST_REDIS_URL);
await cli.flushdbAsync();
await cli.quitAsync();
}
// Create the tmp dir removing any previous one
await fse.remove(tmpDir);
await fse.mkdirs(tmpDir);
log.warn(`Test logs and data are at: ${tmpDir}/`);
// Let's create a sqlite db that we can share with servers that run in other processes, hence
// not an in-memory db. Running seed.ts directly might not take in account the most recent value
// for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
// configuration (in-memory for instance). Spawning a process is one way to make sure that the
// latest value prevail.
process.env.TYPEORM_DATABASE = path.join(tmpDir, 'landing.db');
const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
execFileSync('node', [seed, 'init'], {
env: process.env,
stdio: 'inherit'
});
});
after(() => {
Object.assign(process.env, oldEnv);
});
/**
* Doc api tests are run against three different setup:
* - a merged server: a single server serving both as a home and doc worker
* - two separated servers: requests are sent to a home server which then forward them to a doc worker
* - a doc worker: request are sent directly to the doc worker (note that even though it is not
* used for testing we starts anyway a home server, needed for setting up the test cases)
*
* Future tests must be added within the testDocApi() function.
*/
describe("should work with a merged server", async () => {
setup('merged', async () => {
home = docs = await startServer('home,docs');
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
// the way these tests are written, non-merged server requires redis.
if (process.env.TEST_REDIS_URL) {
describe("should work with a home server and a docworker", async () => {
setup('separated', async () => {
home = await startServer('home');
docs = await startServer('docs', home.serverUrl);
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
describe("should work directly with a docworker", async () => {
setup('docs', async () => {
home = await startServer('home');
docs = await startServer('docs', home.serverUrl);
homeUrl = home.serverUrl;
serverUrl = docs.serverUrl;
hasHomeApi = false;
});
testDocApi();
});
}
describe("QueryParameters", async () => {
function makeExample() {
return {
id: [ 1, 2, 3, 7, 8, 9 ],
color: ['red', 'yellow', 'white', 'blue', 'black', 'purple'],
spin: [ 'up', 'up', 'down', 'down', 'up', 'up'],
};
}
it("supports ascending sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['color']}, null), {
id: [8, 7, 9, 1, 3, 2],
color: ['black', 'blue', 'purple', 'red', 'white', 'yellow'],
spin: ['up', 'down', 'up', 'up', 'down', 'up']
});
});
it("supports descending sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-id']}, null), {
id: [9, 8, 7, 3, 2, 1],
color: ['purple', 'black', 'blue', 'white', 'yellow', 'red'],
spin: ['up', 'up', 'down', 'down', 'up', 'up'],
});
});
it("supports multi-key sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-spin', 'color']}, null), {
id: [8, 9, 1, 2, 7, 3],
color: ['black', 'purple', 'red', 'yellow', 'blue', 'white'],
spin: ['up', 'up', 'up', 'up', 'down', 'down'],
});
});
it("does not freak out sorting mixed data", async function() {
const example = {
id: [ 1, 2, 3, 4, 5, 6, 7, 8, 9],
mixed: ['red', 'green', 'white', 2.5, 1, null, ['zing', 3] as any, 5, 'blue']
};
assert.deepEqual(applyQueryParameters(example, {sort: ['mixed']}, null), {
mixed: [1, 2.5, 5, null, ['zing', 3] as any, 'blue', 'green', 'red', 'white'],
id: [5, 4, 8, 6, 7, 9, 2, 1, 3],
});
});
it("supports limit", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {limit: 1}),
{ id: [1], color: ['red'], spin: ['up'] });
});
it("supports sort and limit", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-color'], limit: 2}, null),
{ id: [2, 3], color: ['yellow', 'white'], spin: ['up', 'down'] });
});
});
});
// Contains the tests. This is where you want to add more test.
function testDocApi() {
it("guesses types of new columns", async () => {
const userActions = [
['AddTable', 'GuessTypes', []],
// Make 5 blank columns of type Any
['AddColumn', 'GuessTypes', 'Date', {}],
['AddColumn', 'GuessTypes', 'DateTime', {}],
['AddColumn', 'GuessTypes', 'Bool', {}],
['AddColumn', 'GuessTypes', 'Numeric', {}],
['AddColumn', 'GuessTypes', 'Text', {}],
// Add string values from which the initial type will be guessed
['AddRecord', 'GuessTypes', null, {
Date: "1970-01-02",
DateTime: "1970-01-02 12:00",
Bool: "true",
Numeric: "1.2",
Text: "hello",
}],
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
// Check that the strings were parsed to typed values
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/records`, chimpy)).data,
{
records: [
{
id: 1,
fields: {
Date: 24 * 60 * 60,
DateTime: 36 * 60 * 60,
Bool: true,
Numeric: 1.2,
Text: "hello",
},
},
],
},
);
// Check the column types
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/columns`, chimpy))
.data.columns.map((col: any) => col.fields.type),
["Date", "DateTime:UTC", "Bool", "Numeric", "Text"],
);
});
for (const mode of ['logged in', 'anonymous']) {
for (const content of ['with content', 'without content']) {
it(`POST /api/docs ${content} creates an unsaved doc when ${mode}`, async function() {
const user = (mode === 'logged in') ? chimpy : nobody;
const formData = new FormData();
formData.append('upload', 'A,B\n1,2\n3,4\n', 'table1.csv');
const config = defaultsDeep({headers: formData.getHeaders()}, user);
let resp = await axios.post(`${serverUrl}/api/docs`,
...(content === 'with content' ? [formData, config] : [null, user]));
assert.equal(resp.status, 200);
const urlId = resp.data;
if (mode === 'logged in') {
assert.match(urlId, /^new~[^~]*~[0-9]+$/);
} else {
assert.match(urlId, /^new~[^~]*$/);
}
// Access information about that document should be sane for current user
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, user);
assert.equal(resp.status, 200);
assert.equal(resp.data.name, 'Untitled');
assert.equal(resp.data.workspace.name, 'Examples & Templates');
assert.equal(resp.data.access, 'owners');
if (mode === 'anonymous') {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, chimpy);
assert.equal(resp.data.access, 'owners');
} else {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, charon);
assert.equal(resp.status, 403);
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, nobody);
assert.equal(resp.status, 403);
}
// content was successfully stored
resp = await axios.get(`${serverUrl}/api/docs/${urlId}/tables/Table1/data`, user);
if (content === 'with content') {
assert.deepEqual(resp.data, { id: [ 1, 2 ], manualSort: [ 1, 2 ], A: [ 1, 3 ], B: [ 2, 4 ] });
} else {
assert.deepEqual(resp.data, { id: [], manualSort: [], A: [], B: [], C: [] });
}
});
}
}
it("GET /docs/{did}/tables/{tid}/data retrieves data in column format", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
});
it("GET /docs/{did}/tables/{tid}/records retrieves data in records format", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'hello',
B: '',
C: '',
D: null,
E: 'HELLO',
},
},
{
id: 2,
fields: {
A: '',
B: 'world',
C: '',
D: null,
E: '',
},
},
{
id: 3,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
{
id: 4,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
]
});
});
it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"records": [
{
"id": 1,
"fields": {
"A": null,
"B": "Hi",
"C": 1,
},
"errors": {
"A": "ZeroDivisionError"
}
}
]
}
);
// /data format for comparison: includes manualSort, gristHelper_Display, and ["E", "ZeroDivisionError"]
resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"id": [
1
],
"manualSort": [
1
],
"A": [
[
"E",
"ZeroDivisionError"
]
],
"B": [
"Hi"
],
"C": [
1
],
"gristHelper_Display": [
"Hi"
]
}
);
});
it("GET /docs/{did}/tables/{tid}/columns retrieves columns", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
columns: [
{
id: 'A',
fields: {
colRef: 2,
parentId: 1,
parentPos: 1,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'A',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'B',
fields: {
colRef: 3,
parentId: 1,
parentPos: 2,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'B',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'C',
fields: {
colRef: 4,
parentId: 1,
parentPos: 3,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'C',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'D',
fields: {
colRef: 5,
parentId: 1,
parentPos: 3,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '',
label: 'D',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'E',
fields: {
colRef: 6,
parentId: 1,
parentPos: 4,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '$A.upper()',
label: 'E',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
}
]
}
);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent doc", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent table", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent doc", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent table", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/data supports filters", async function() {
function makeQuery(filters: {[colId: string]: any[]}) {
const query = "filter=" + encodeURIComponent(JSON.stringify(filters));
return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy);
}
function checkResults(resp: AxiosResponse<any>, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery({B: ['world']}), {
id: [2], A: [''], B: ['world'], C: [''], D: [null], E: [''], manualSort: [2],
});
// Can query by id
checkResults(await makeQuery({id: [1]}), {
id: [1], A: ['hello'], B: [''], C: [''], D: [null], E: ['HELLO'], manualSort: [1],
});
checkResults(await makeQuery({B: [''], A: ['']}), {
id: [3, 4], A: ['', ''], B: ['', ''], C: ['', ''], D: [null, null], E: ['', ''], manualSort: [3, 4],
});
// Empty filter is equivalent to no filter and should return full data.
checkResults(await makeQuery({}), {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
// An impossible filter should succeed but return an empty set of rows.
checkResults(await makeQuery({B: ['world'], C: ['Neptune']}), {
id: [], A: [], B: [], C: [], D: [], E: [], manualSort: [],
});
// An invalid filter should return an error
{
const resp = await makeQuery({BadCol: ['']});
assert.equal(resp.status, 400);
assert.match(resp.data.error, /BadCol/);
}
{
const resp = await makeQuery({B: 'world'} as any);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /filter values must be arrays/);
}
});
for (const mode of ['url', 'header']) {
it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function() {
function makeQuery(sort: string[]|null, limit: number|null) {
const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
const config = configForUser('chimpy');
if (mode === 'url') {
if (sort) { url.searchParams.append('sort', sort.join(',')); }
if (limit) { url.searchParams.append('limit', String(limit)); }
} else {
if (sort) { config.headers['x-sort'] = sort.join(','); }
if (limit) { config.headers['x-limit'] = String(limit); }
}
return axios.get(url.href, config);
}
function checkResults(resp: AxiosResponse<any>, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery(['-id'], null), {
id: [4, 3, 2, 1],
A: ['', '', '', 'hello'],
B: ['', '', 'world', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['', '', '', 'HELLO'],
manualSort: [4, 3, 2, 1]
});
checkResults(await makeQuery(['-id'], 2), {
id: [4, 3],
A: ['', ''],
B: ['', ''],
C: ['', ''],
D: [null, null],
E: ['', ''],
manualSort: [4, 3]
});
});
}
it("GET /docs/{did}/tables/{tid}/data respects document permissions", async function() {
// as not part of any group kiwi cannot fetch Timesheets
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, kiwi);
assert.equal(resp.status, 403);
});
it("GET /docs/{did}/tables/{tid}/data returns matches /not found/ for bad table id", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/apply applies user actions", async function() {
const userActions = [
['AddTable', 'Foo', [{id: 'A'}, {id: 'B'}]],
['BulkAddRecord', 'Foo', [1, 2], {A: ["Santa", "Bob"], B: [1, 11]}]
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data,
{id: [1, 2], A: ['Santa', 'Bob'], B: ['1', '11'], manualSort: [1, 2]});
});
it("POST /docs/{did}/apply respects document permissions", async function() {
const userActions = [
['AddTable', 'FooBar', [{id: 'A'}]]
];
let resp: AxiosResponse;
// as a guest chimpy cannot edit Bananas
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Bananas}/apply`, userActions, chimpy);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Bananas}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
// as not in any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, kiwi);
assert.equal(resp.status, 403);
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/tables/{tid}/data adds records", async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3, 4]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("POST /docs/{did}/tables/{tid}/records adds records", async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{fields: {A: 'John', B: 55}},
{fields: {A: 'Jane', B: 0}},
]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
records: [
{id: 5},
{id: 6},
]
});
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'Santa',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
{
id: 5,
fields: {
A: 'John',
B: '55',
},
},
{
id: 6,
fields: {
A: 'Jane',
B: '0',
},
},
]
});
});
it("POST /docs/{did}/tables/{tid}/data/delete deletes records", async function() {
let resp = await axios.post(
`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data/delete`,
[3, 4, 5, 6],
chimpy,
);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, null);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2],
A: ['Santa', 'Bob'],
B: ["1", "11"],
manualSort: [1, 2]
});
// restore rows
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
function checkError(status: number, test: RegExp|object, resp: AxiosResponse<any>, message?: string) {
assert.equal(resp.status, status);
if (test instanceof RegExp) {
assert.match(resp.data.error, test, message);
} else {
try {
assert.deepEqual(resp.data, test, message);
} catch(err) {
console.log(JSON.stringify(resp.data));
console.log(JSON.stringify(test));
throw err;
}
}
}
it("parses strings in user actions", async () => {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const recordsUrl = `${docUrl}/tables/Table1/records`;
// Make the column numeric, delete the other columns we don't care about
await axios.post(`${docUrl}/apply`, [
['ModifyColumn', 'Table1', 'A', {type: 'Numeric'}],
['RemoveColumn', 'Table1', 'B'],
['RemoveColumn', 'Table1', 'C'],
], chimpy);
// Add/update some records without and with string parsing
// Specifically test:
// 1. /apply, with an AddRecord
// 2. POST /records (BulkAddRecord)
// 3. PATCH /records (BulkUpdateRecord)
// Send strings that look like currency which need string parsing to become numbers
for (const queryParams of ['?noparse=1', '']) {
await axios.post(`${docUrl}/apply${queryParams}`, [
['AddRecord', 'Table1', null, {'A': '$1'}],
], chimpy);
const response = await axios.post(`${recordsUrl}${queryParams}`,
{
records: [
{fields: {'A': '$2'}},
{fields: {'A': '$3'}},
]
},
chimpy);
// Update $3 -> $4
const rowId = response.data.records[1].id;
await axios.patch(`${recordsUrl}${queryParams}`,
{
records: [
{id: rowId, fields: {'A': '$4'}}
]
},
chimpy);
}
// Check the results
const resp = await axios.get(recordsUrl, chimpy);
assert.deepEqual(resp.data, {
records:
[
// Without string parsing
{id: 1, fields: {A: '$1'}},
{id: 2, fields: {A: '$2'}},
{id: 3, fields: {A: '$4'}},
// With string parsing
{id: 4, fields: {A: 1}},
{id: 5, fields: {A: 2}},
{id: 6, fields: {A: 4}},
]
}
);
});
describe("PUT /docs/{did}/tables/{tid}/records", async function() {
it("should add or update records", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
async function check(records: AddOrUpdateRecord[], expectedTableData: BulkColValues, params: any={}) {
const resp = await axios.put(url, {records}, {...chimpy, params});
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
delete table.C;
assert.deepStrictEqual(table, expectedTableData);
}
// Add 3 new records, since the table is empty so nothing matches `requires`
await check(
[
{
require: {A: 1},
},
{
// Since no record with A=2 is found, create a new record,
// but `fields` overrides `require` for the value when creating,
// so the new record has A=3
require: {A: 2},
fields: {A: 3},
},
{
require: {A: 4},
fields: {B: 5},
},
],
{id: [1, 2, 3], A: [1, 3, 4], B: [0, 0, 5]}
);
// Update all three records since they all match the `require` values here
await check(
[
{
// Does nothing
require: {A: 1},
},
{
// Changes A from 3 to 33
require: {A: 3},
fields: {A: 33},
},
{
// Changes B from 5 to 6 in the third record where A=4
require: {A: 4},
fields: {B: 6},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}
);
// This would normally add a record, but noadd suppresses that
await check([
{
require: {A: 100},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noadd: "1"},
);
// This would normally update A from 1 to 11, bot noupdate suppresses that
await check([
{
require: {A: 1},
fields: {A: 11},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noupdate: "1"},
);
// There are 2 records with B=0, update them both to B=1
// Use onmany=all to specify that they should both be updated
await check([
{
require: {B: 0},
fields: {B: 1},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [1, 1, 6]},
{onmany: "all"}
);
// In contrast to the above, the default behaviour for no value of onmany
// is to only update the first matching record,
// so only one of the records with B=1 is updated to B=2
await check([
{
require: {B: 1},
fields: {B: 2},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [2, 1, 6]},
);
// By default, strings in `require` and `fields` are parsed based on column type,
// so these dollar amounts are treated as currency
// and parsed as A=4 and A=44
await check([
{
require: {A: "$4"},
fields: {A: "$44"},
},
],
{id: [1, 2, 3], A: [1, 33, 44], B: [2, 1, 6]},
);
// Turn off the default string parsing with noparse=1
// Now we need A=44 to actually be a number to match,
// A="$44" wouldn't match and would create a new record.
// Because A="$55" isn't parsed, the raw string is stored in the table.
await check([
{
require: {A: 44},
fields: {A: "$55"},
},
],
{id: [1, 2, 3], A: [1, 33, "$55"], B: [2, 1, 6]},
{noparse: 1}
);
await check([
// First three records already exist and nothing happens
{require: {A: 1}},
{require: {A: 33}},
{require: {A: "$55"}},
// Without string parsing, A="$33" doesn't match A=33 and a new record is created
{require: {A: "$33"}},
],
{id: [1, 2, 3, 4], A: [1, 33, "$55", "$33"], B: [2, 1, 6, 0]},
{noparse: 1}
);
// Checking that updating by `id` works.
await check([
{
require: {id: 3},
fields: {A: "66"},
},
],
{id: [1, 2, 3, 4], A: [1, 33, 66, "$33"], B: [2, 1, 6, 0]},
);
// allow_empty_require option with empty `require` updates all records
await check([
{
require: {},
fields: {A: 99, B: 99},
},
],
{id: [1, 2, 3, 4], A: [99, 99, 99, 99], B: [99, 99, 99, 99]},
{allow_empty_require: "1", onmany: "all"},
);
});
it("should 404 for missing tables", async () => {
checkError(404, /Table not found "Bad_Foo_"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/records`,
{records: [{require: {id: 1}}]}, chimpy));
});
it("should 400 for missing columns", async () => {
checkError(400, /Invalid column "no_such_column"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {no_such_column: 1}}]}, chimpy));
});
it("should 400 for an incorrect onmany parameter", async function() {
checkError(400,
/onmany parameter foo should be one of first,none,all/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {id: 1}}]}, {...chimpy, params: {onmany: "foo"}}));
});
it("should 400 for an empty require without allow_empty_require", async function() {
checkError(400,
/require is empty but allow_empty_require isn't set/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {}}]}, chimpy));
});
it("should validate request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async (payload: any, error: { error: string, details: string }) => {
const resp = await axios.put(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await test({records: [{fields: {}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require is missing',
});
await test({records: [{require: {id: "1"}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require.id is not a number',
});
});
});
describe("POST /docs/{did}/tables/{tid}/records", async function() {
it("POST should have good errors", async () => {
checkError(404, /not found/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`,
{ A: ['Alice', 'Felix'], B: [2, 22] }, chimpy));
checkError(400, /Invalid column "Bad"/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{ A: ['Alice'], Bad: ['Monthy'] }, chimpy));
// Other errors should also be maximally informative.
checkError(400, /Error manipulating data/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{ A: ['Alice'], B: null }, chimpy));
});
it("validates request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async(payload: any, error: {error: string, details: string}) => {
const resp = await axios.post(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
// All column types are allowed, except Arrays (or objects) without correct code.
const testField = async (A: any) => {
await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a NewRecord; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'});
};
// test no code at all
await testField([]);
// test invalid code
await testField(['ZZ']);
});
it("allows to create a blank record", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
// Create two blank records
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const resp = await axios.post(url, {records: [{}, { fields: {}}]}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, { records : [{id: 1}, {id: 2}]});
});
it("allows to create partial records", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create partial records
const resp = await axios.post(url, {records: [{fields: { A: 1}}, { fields: {B: 2}}, {}]}, chimpy);
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
assert.deepStrictEqual(
table,
{ id: [1, 2, 3], A: [1, null, null], B: [null, 2, null], C:[null, null, null]});
});
it("allows CellValue as a field", async function() {
// create sample document
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'PostTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const testField = async(A?: CellValue, message?: string) =>{
const resp = await axios.post(url, {records: [{ fields: { A } }]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
// test allowed types for a field
await testField(1); // ints
await testField(1.2); // floats
await testField("string"); // strings
await testField(true); // true and false
await testField(false);
await testField(null); // null
// encoded values (though not all make sense)
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
it("POST /docs/{did}/tables/{tid}/data respects document permissions", async function() {
let resp: AxiosResponse;
const data = {
A: ['Alice', 'Felix'],
B: [2, 22]
};
// as a viewer charon cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that TestDoc did not change
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
describe("PATCH /docs/{did}/tables/{tid}/records", function() {
it("updates records", async function () {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{
id: 1,
fields: {
A: 'Father Christmas',
},
},
],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
records:
[
{
id: 1,
fields: {
A: 'Father Christmas',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
]
});
});
it("validates request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
async function failsWithError(payload: any, error: { error: string, details?: string }){
const resp = await axios.patch(url, payload, chimpy);
checkError(400, error, resp);
}
await failsWithError({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await failsWithError({records: []}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; body.records[0] is not an object'});
await failsWithError({records: [{}]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n '+
'body.records[0].id is missing\n '+
'body.records[0].fields is missing'});
await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n' +
' body.records[0].id is not a number\n' +
' body.records[0].fields is missing'});
await failsWithError(
{records: [{id: 1, fields: {A : 1}}, {id: 2, fields: {B: 3}}]},
{error: 'PATCH requires all records to have same fields'});
// Test invalid object codes
const fieldIsNotValid = async (A: any) => {
await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'});
};
await fieldIsNotValid([]);
await fieldIsNotValid(['ZZ']);
});
it("allows CellValue as a field", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'PatchTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create record for patching
const id = (await axios.post(url, { records: [{}] }, chimpy)).data.records[0].id;
const testField = async(A?: CellValue, message?: string) =>{
const resp = await axios.patch(url, {records: [{ id, fields: { A } }]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
await testField(1);
await testField(1.2);
await testField("string");
await testField(true);
await testField(false);
await testField(null);
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
describe("PATCH /docs/{did}/tables/{tid}/data", function() {
it("updates records", async function() {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Santa Klaus'],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid row ids", async function() {
// combination of valid and invalid ids fails
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 5/);
// only invalid ids also fails
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [10, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 10/);
// check that changes related to id 1 did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid column", async function() {
const resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Alice'],
C: ['Monthy']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid column "C"/);
});
it("respects document permissions", async function() {
let resp: AxiosResponse;
const data = {
id: [1],
A: ['Santa'],
};
// check data
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
// as a viewer charon cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that changes did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
});
describe('attachments', function() {
it("POST /docs/{did}/attachments adds attachments", async function() {
let formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2]);
// Another upload gets the next number.
formData = new FormData();
formData.append('upload', 'abcdef', "hello.png");
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3]);
});
it("GET /docs/{did}/attachments lists attachment metadata", async function() {
// Test that the usual /records query parameters like sort and filter also work
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/attachments?sort=-fileName&limit=2`;
const resp = await axios.get(url, chimpy);
assert.equal(resp.status, 200);
const {records} = resp.data;
for (const record of records) {
assert.match(record.fields.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
delete record.fields.timeUploaded;
}
assert.deepEqual(records, [
{id: 2, fields: {fileName: "world.jpg", fileSize: 6}},
{id: 3, fields: {fileName: "hello.png", fileSize: 6}},
]
);
});
it("GET /docs/{did}/attachments/{id} returns attachment metadata", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2`, chimpy);
assert.equal(resp.status, 200);
assert.include(resp.data, {fileName: "world.jpg", fileSize: 6});
assert.match(resp.data.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
});
it("GET /docs/{did}/attachments/{id}/download downloads attachment contents", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}/download works after doc shutdown", async function() {
// Check that we can download when ActiveDoc isn't currently open.
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}... returns 404 when attachment not found", async function() {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo`, chimpy);
checkError(404, /Attachment not found: moo/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22/download`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo/download`, chimpy);
checkError(404, /Attachment not found: moo/, resp);
});
it("POST /docs/{did}/attachments produces reasonable errors", async function() {
// Check that it produces reasonable errors if we try to use it with non-form-data
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, [4, 5, 6], chimpy);
assert.equal(resp.status, 415); // Wrong content-type
// Check for an error if there is no data included.
const formData = new FormData();
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 400);
// TODO The error here is "stream ended unexpectedly", which isn't really reasonable.
});
it("POST/GET /docs/{did}/attachments respect document permissions", async function() {
const formData = new FormData();
formData.append('upload', 'xyzzz', "wrong.png");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, kiwi));
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3`, kiwi);
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3/download`, kiwi);
checkError(403, /No view access/, resp);
});
it("POST /docs/{did}/attachments respects untrusted content-type only if valid", async function() {
const formData = new FormData();
formData.append('upload', 'xyz', {filename: "foo", contentType: "application/pdf"});
formData.append('upload', 'abc', {filename: "hello.png", contentType: "invalid/content-type"});
formData.append('upload', 'def', {filename: "world.doc", contentType: "text/plain\nbad-header: 1\n\nEvil"});
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [4, 5, 6]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/4/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/pdf'); // A valid content-type is respected
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="foo.pdf"');
assert.deepEqual(resp.data, 'xyz');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/5/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/png'); // Did not pay attention to invalid header
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="hello.png"');
assert.deepEqual(resp.data, 'abc');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/6/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/msword'); // Another invalid header ignored
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.doc"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.headers['bad-header'], undefined); // Attempt to hack in more headers didn't work
assert.deepEqual(resp.data, 'def');
});
it("POST /docs/{did}/attachments/updateUsed updates timeDeleted on metadata", async function() {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc2'}, wid);
// Apply the given user actions,
// POST to /attachments/updateUsed
// Check that Table1 and _grist_Attachments contain the expected rows
async function check(
actions: UserAction[],
userData: { id: number, Attached: any }[],
metaData: { id: number, deleted: boolean }[],
) {
const docUrl = `${serverUrl}/api/docs/${docId}`;
let resp = await axios.post(`${docUrl}/apply`, actions, chimpy);
assert.equal(resp.status, 200);
resp = await axios.post(`${docUrl}/attachments/updateUsed`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${docUrl}/tables/Table1/records`, chimpy);
const actualUserData = resp.data.records.map(
({id, fields: {Attached}}: ApiRecord) =>
({id, Attached})
);
assert.deepEqual(actualUserData, userData);
resp = await axios.get(`${docUrl}/tables/_grist_Attachments/records`, chimpy);
const actualMetaData = resp.data.records.map(
({id, fields: {timeDeleted}}: ApiRecord) =>
({id, deleted: Boolean(timeDeleted)})
);
assert.deepEqual(actualMetaData, metaData);
}
// Set up the document and initial data.
await check(
[
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkAddRecord", "Table1", [1, 2], {Attached: [['L', 1], ['L', 2, 3]]}],
// There's no actual attachments here but that doesn't matter
["BulkAddRecord", "_grist_Attachments", [1, 2, 3], {}],
],
[
{id: 1, Attached: ['L', 1]},
{id: 2, Attached: ['L', 2, 3]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false},
{id: 3, deleted: false},
],
);
// Remove the record containing ['L', 2, 3], so the metadata for 2 and 3 now says deleted
await check(
[["RemoveRecord", "Table1", 2]],
[
{id: 1, Attached: ['L', 1]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true}, // deleted here
],
);
// Add back a reference to attacument 2 to test 'undeletion', plus some junk values
await check(
[["BulkAddRecord", "Table1", [3, 4, 5], {Attached: [null, "foo", ['L', 2, 2, 4, 4, 5]]}]],
[
{id: 1, Attached: ['L', 1]},
{id: 3, Attached: null},
{id: 4, Attached: "foo"},
{id: 5, Attached: ['L', 2, 2, 4, 4, 5]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false}, // undeleted here
{id: 3, deleted: true},
],
);
// Remove the whole column to test what happens when there's no Attachment columns
await check(
[["RemoveColumn", "Table1", "Attached"]],
[
{id: 1, Attached: undefined},
{id: 3, Attached: undefined},
{id: 4, Attached: undefined},
{id: 5, Attached: undefined},
],
[
{id: 1, deleted: true}, // deleted here
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true},
],
);
// Test performance with a large number of records and attachments.
// The maximum value of numRecords that doesn't return a 413 error is about 18,000.
// In that case it took about 5.7 seconds to apply the initial user actions (i.e. add the records),
// 0.3 seconds to call updateUsed once, and 0.1 seconds to call it again immediately after.
// That last time roughly measures the time taken to do the SQL query
// without having to apply any user actions after to update timeDeleted.
// 10,000 records is a compromise so that tests aren't too slow.
const numRecords = 10000;
const attachmentsPerRecord = 4;
const totalUsedAttachments = numRecords * attachmentsPerRecord; // 40,000 attachments referenced in user data
const totalAttachments = totalUsedAttachments * 1.1; // 44,000 attachment IDs listed in metadata
const attachedValues = _.chunk(_.range(1, totalUsedAttachments + 1), attachmentsPerRecord)
.map(arr => ['L', ...arr]);
await check(
[
// Reset the state: add back the removed column and delete the previously added data
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkRemoveRecord", "Table1", [1, 3, 4, 5]],
["BulkRemoveRecord", "_grist_Attachments", [1, 2, 3]],
["BulkAddRecord", "Table1", arrayRepeat(numRecords, null), {Attached: attachedValues}],
["BulkAddRecord", "_grist_Attachments", arrayRepeat(totalAttachments, null), {}],
],
attachedValues.map((Attached, index) => ({id: index + 1, Attached})),
_.range(totalAttachments).map(index => ({id: index + 1, deleted: index >= totalUsedAttachments})),
);
});
it("POST /docs/{did}/attachments/removeUnused removes unused attachments", async function() {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc3'}, wid);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
formData.append('upload', 'foobar', "hello2.doc");
let resp = await axios.post(`${docUrl}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2, 3]);
async function checkAttachmentIds(ids: number[]) {
resp = await axios.get(`${docUrl}/attachments`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data.records.map((r: any) => r.id), ids);
}
resp = await axios.patch(
`${docUrl}/tables/_grist_Attachments/records`,
{
records: [
{id: 1, fields: {timeDeleted: Date.now() / 1000 - 8 * 24 * 60 * 60}}, // 8 days ago, i.e. expired
{id: 2, fields: {timeDeleted: Date.now() / 1000 - 6 * 24 * 60 * 60}}, // 6 days ago, i.e. not expired
]
},
chimpy,
);
assert.equal(resp.status, 200);
await checkAttachmentIds([1, 2, 3]);
// Remove the expired attachment (1) by force-reloading, so it removes it during shutdown.
// It has a duplicate (3) that hasn't expired and thus isn't removed,
// although they share the same fileIdent and row in _gristsys_Files.
// So for now only the metadata is removed.
resp = await axios.post(`${docUrl}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([2, 3]);
resp = await axios.post(`${docUrl}/attachments/verifyFiles`, null, chimpy);
assert.equal(resp.status, 200);
// Remove the not expired attachments (2 and 3).
// We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded.
resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([]);
});
});
it("GET /docs/{did}/download serves document", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, chimpy);
assert.equal(resp.status, 200);
assert.match(resp.data, /grist_Tables_column/);
});
it("GET /docs/{did}/download respects permissions", async function() {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, kiwi);
assert.equal(resp.status, 403);
assert.notMatch(resp.data, /grist_Tables_column/);
});
it("GET /docs/{did}/download/csv serves CSV-encoded document", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
assert.equal(resp.status, 200);
assert.equal(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
const resp2 = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Foo`, chimpy);
assert.equal(resp2.status, 200);
assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
});
it("GET /docs/{did}/download/csv respects permissions", async function() {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);
assert.equal(resp.status, 403);
assert.notEqual(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
});
it("GET /docs/{did}/download/csv returns 404 if tableId is invalid", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=MissingTableId`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' });
});
it("GET /docs/{did}/download/csv returns 404 if viewSectionId is invalid", async function() {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1&viewSection=9999`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' });
});
it("GET /docs/{did}/download/csv returns 400 if tableId is missing", async function() {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv`, chimpy);
assert.equal(resp.status, 400);
assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
});
it('POST /workspaces/{wid}/import handles empty filenames', async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
const worker1 = await userApi.getWorkerAPI('import');
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, '.grist');
const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
assert.equal(resp.data.title, 'Untitled upload');
assert.equal(typeof resp.data.id, 'string');
assert.notEqual(resp.data.id, '');
});
it("document is protected during upload-and-import sequence", async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
// Prepare an API for a different user.
const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
headers: {Authorization: 'Bearer api_key_for_kiwi'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
// upload something for Chimpy and something else for Kiwi.
const worker1 = await userApi.getWorkerAPI('import');
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, 'upload.grist');
const worker2 = await kiwiApi.getWorkerAPI('import');
const fakeData2 = await testUtils.readFixtureDoc('Favorite_Films.grist');
const uploadId2 = await worker2.upload(fakeData2, 'upload2.grist');
// Check that kiwi only has access to their own upload.
let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id;
let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Kiwi'));
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: "access denied"});
resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2},
configForUser('Kiwi'));
assert.equal(resp.status, 200);
// Check that chimpy has access to their own upload.
wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
});
it('limits parallel requests', async function() {
// Launch 30 requests in parallel and see how many are honored and how many
// return 429s. The timing of this test is a bit delicate. We close the doc
// to increase the odds that results won't start coming back before all the
// requests have passed authorization. May need to do something more sophisticated
// if this proves unreliable.
await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
const reqs = [...Array(30).keys()].map(
i => axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy));
const responses = await Promise.all(reqs);
assert.lengthOf(responses.filter(r => r.status === 200), 10);
assert.lengthOf(responses.filter(r => r.status === 429), 20);
});
it('allows forced reloads', async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force a reload.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('allows assignments', async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force an assignment.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('honors urlIds', async function() {
// Make a document with a urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1);
try {
// Make sure an edit made by docId is visible when accessed via docId or urlId
let resp = await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
// Make sure an edit made by urlId is visible when accessed via docId or urlId
resp = await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
A: ['Orange'], B: [42]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
}
});
it('filters urlIds by org', async function() {
// Make two documents with same urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2);
try {
// Place a value in "docs" doc
await axios.post(`${serverUrl}/o/docs/api/docs/urlid/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
// Place a value in "nasa" doc
await axios.post(`${serverUrl}/o/nasa/api/docs/urlid/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
// Check the values made it to the right places
let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it('allows docId access to any document from merged org', async function() {
// Make two documents
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2);
try {
// Should fail to write to a document in "docs" from "nasa" url
let resp = await axios.post(`${serverUrl}/o/nasa/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
// Should successfully write to a document in "nasa" from "docs" url
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 200);
// Should fail to write to a document in "nasa" from "pr" url
resp = await axios.post(`${serverUrl}/o/pr/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it("GET /docs/{did}/replace replaces one document with another", async function() {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const doc2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc3 = await userApi.newDoc({name: 'testdoc2'}, ws1);
await userApi.updateDocPermissions(doc2, {users: {'kiwi@getgrist.com': 'editors'}});
await userApi.updateDocPermissions(doc3, {users: {'kiwi@getgrist.com': 'viewers'}});
try {
// Put some material in doc3
let resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc3}/tables/Table1/data`, {
A: ['Orange']
}, chimpy);
assert.equal(resp.status, 200);
// Kiwi can replace doc2 with doc3
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Orange');
// Kiwi can't replace doc1 with doc3, no write access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc1}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 403);
// Kiwi can't replace doc2 with doc1, no read access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc1
}, kiwi);
assert.equal(resp.status, 403);
} finally {
await userApi.deleteDoc(doc1);
await userApi.deleteDoc(doc2);
}
});
it("GET /docs/{did}/snapshots retrieves a list of snapshots", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/snapshots`, chimpy);
assert.equal(resp.status, 200);
assert.isAtLeast(resp.data.snapshots.length, 1);
assert.hasAllKeys(resp.data.snapshots[0], ['docId', 'lastModified', 'snapshotId']);
});
it("POST /docs/{did}/states/remove removes old states", async function() {
// Check doc has plenty of states.
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
const states: DocState[] = resp.data.states;
assert.isAbove(states.length, 5);
// Remove all but 3.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 3}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 3);
assert.equal(resp.data.states[0].h, states[0].h);
assert.equal(resp.data.states[1].h, states[1].h);
assert.equal(resp.data.states[2].h, states[2].h);
// Remove all but 1.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 1}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 1);
assert.equal(resp.data.states[0].h, states[0].h);
});
it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function() {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc1 = userApi.getDocAPI(docId1);
const doc2 = userApi.getDocAPI(docId2);
// Stick some content in column A so it has a defined type
// so diffs are smaller and simpler.
await doc2.addRows('Table1', {A: [0]});
let comp = await doc1.compareDoc(docId2);
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary']);
assert.equal(comp.summary, 'unrelated');
assert.equal(comp.parent, null);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 1);
assert.equal(comp.right.n, 2);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 2);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details, {
leftChanges: { tableRenames: [], tableDeltas: {} },
rightChanges: { tableRenames: [], tableDeltas: {} }
});
await doc1.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'left');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 2);
assert.deepEqual(comp.right, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.rightChanges,
{ tableRenames: [], tableDeltas: {} });
const addA1: ActionSummary = {
tableRenames: [],
tableDeltas: { Table1: {
updateRows: [],
removeRows: [],
addRows: [ 2 ],
columnDeltas: {
A: { [2]: [null, [1]] },
manualSort: { [2]: [null, [2]] },
},
columnRenames: [],
} }
};
assert.deepEqual(comp.details!.leftChanges, addA1);
await doc2.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'both');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 3);
assert.equal(comp.parent!.n, 2);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.leftChanges, addA1);
assert.deepEqual(comp.details!.rightChanges, addA1);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 3);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details, {
leftChanges: { tableRenames: [], tableDeltas: {} },
rightChanges: { tableRenames: [], tableDeltas: {} }
});
await doc2.addRows('Table1', {A: [2]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'right');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.leftChanges,
{ tableRenames: [], tableDeltas: {} });
const addA2: ActionSummary = {
tableRenames: [],
tableDeltas: { Table1: {
updateRows: [],
removeRows: [],
addRows: [ 3 ],
columnDeltas: {
A: { [3]: [null, [2]] },
manualSort: { [3]: [null, [3]] },
},
columnRenames: [],
} }
};
assert.deepEqual(comp.details!.rightChanges, addA2);
});
it("GET /docs/{did}/compare tracks changes within a doc", async function() {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// Give the document some history.
await doc.addRows('Table1', {A: ['a1'], B: ['b1']});
await doc.addRows('Table1', {A: ['a2'], B: ['b2']});
await doc.updateRows('Table1', {id: [1], A: ['A1']});
// Examine the most recent change, from HEAD~ to HEAD.
let comp = await doc.compareVersion('HEAD~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [],
columnDeltas: {
A: { [1]: [['a1'], ['A1']] }
},
columnRenames: [],
}
}
});
// Check we get the same result with actual hashes.
assert.notMatch(comp.left.h, /HEAD/);
assert.notMatch(comp.right.h, /HEAD/);
const comp2 = await doc.compareVersion(comp.left.h, comp.right.h);
assert.deepEqual(comp, comp2);
// Check that comparing the HEAD with itself shows no changes.
comp = await doc.compareVersion('HEAD', 'HEAD');
assert.equal(comp.summary, 'same');
assert.deepEqual(comp.parent, comp.left);
assert.deepEqual(comp.parent, comp.right);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, { tableRenames: [], tableDeltas: {} });
// Examine the combination of the last two changes.
comp = await doc.compareVersion('HEAD~~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 2);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [2],
columnDeltas: {
A: { [1]: [['a1'], ['A1']],
[2]: [null, ['a2']] },
B: { [2]: [null, ['b2']] },
manualSort: { [2]: [null, [2]] },
},
columnRenames: [],
}
}
});
});
it('doc worker endpoints ignore any /dw/.../ prefix', async function() {
const docWorkerUrl = docs.serverUrl;
let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${docWorkerUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
if (docWorkerUrl !== homeUrl) {
resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${homeUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
}
});
it("POST /docs/{did}/tables/{tid}/_subscribe validates inputs", async function () {
async function check(requestBody: any, status: number, error: string) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
requestBody, chimpy
);
assert.equal(resp.status, status);
assert.deepEqual(resp.data, {error});
}
await check({}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: 0}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: []}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: ["foo"]}, 400, "Allowed values in eventTypes are: add,update");
await check({eventTypes: ["add"]}, 400, "Bad request: url required");
await check({eventTypes: ["add"], url: "https://evil.com"}, 403, "Provided url is forbidden");
await check({eventTypes: ["add"], url: "http://example.com"}, 403, "Provided url is forbidden"); // not https
await check({eventTypes: ["add"], url: "https://example.com", isReadyColumn: "bar"}, 404, `Column not found "bar"`);
});
it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs", async function() {
const subscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
{eventTypes: ["add"], url: "https://example.com"}, chimpy
);
assert.equal(subscribeResponse.status, 200);
const {triggerId, unsubscribeKey, webhookId} = subscribeResponse.data;
async function check(requestBody: any, status: number, responseBody: any) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_unsubscribe`,
requestBody, chimpy
);
assert.equal(resp.status, status);
if (status !== 200) {
responseBody = {error: responseBody};
}
assert.deepEqual(resp.data, responseBody);
}
await check({triggerId: 999}, 404, `Trigger not found "999"`);
await check({triggerId, webhookId: "foo"}, 404, `Webhook not found "foo"`);
await check({triggerId, webhookId}, 400, 'Bad request: id and unsubscribeKey both required');
await check({triggerId, webhookId, unsubscribeKey: "foo"}, 401, 'Wrong unsubscribeKey');
// Actually unsubscribe
await check({triggerId, webhookId, unsubscribeKey}, 200, {success: true});
// Trigger is now deleted!
await check({triggerId, webhookId, unsubscribeKey}, 404, `Trigger not found "${triggerId}"`);
});
describe("Daily API Limit", () => {
let redisClient: RedisClient;
before(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
redisClient = createClient(process.env.TEST_REDIS_URL);
});
it("limits daily API usage", async function() {
// Make a new document in a test product with a low daily limit
const api = makeUserApi('testdailyapilimit');
const workspaceId = await getWorkspaceId(api, 'TestDailyApiLimitWs');
const docId = await api.newDoc({name: 'TestDoc1'}, workspaceId);
const max = testDailyApiLimitFeatures.baseMaxApiUnitsPerDocumentPerDay;
for (let i = 1; i <= max + 2; i++) {
let success = true;
try {
// Make some doc request so that it fails or succeeds
await api.getTable(docId, "Table1");
} catch (e) {
success = false;
}
// Usually the first `max` requests should succeed and the rest should fail having exceeded the daily limit.
// If a new minute starts in the middle of the requests, an extra request will be allowed for that minute.
// If a new day starts in the middle of the requests, this test will fail.
if (success) {
assert.isAtMost(i, max + 1);
} else {
assert.isAtLeast(i, max + 1);
}
}
});
it("limits daily API usage and sets the correct keys in redis", async function() {
// Make a new document in a free team site, currently the only real product which limits daily API usage.
const freeTeamApi = makeUserApi('freeteam');
const workspaceId = await getWorkspaceId(freeTeamApi, 'FreeTeamWs');
const docId = await freeTeamApi.newDoc({name: 'TestDoc2'}, workspaceId);
// Rather than making 5000 requests, set high counts directly for the current and next daily and hourly keys
const used = 999999;
let m = moment.utc();
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
await redisClient.multi()
.set(currentDay, String(used))
.set(currentHour, String(used))
.set(nextDay, String(used))
.set(nextHour, String(used))
.execAsync();
// Make 9 requests. The first 4 should succeed by fitting into the allocation for the minute.
// (Free team plans get 5000 requests per day, and 5000/24/60 ~= 3.47 which is rounded up to 4)
// The last request should fail. Don't check the middle 4 in case we're on the boundary of a minute.
for (let i = 1; i <= 9; i++) {
const last = i === 9;
m = moment.utc(); // get this before delaying to calculate accurate keys below
const response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy);
// Allow time for redis to be updated.
await delay(100);
if (i <= 4) {
assert.equal(response.status, 200);
// Keys of the periods we expect to be incremented.
// For the first request, the server's usage cache is empty and it hasn't seen the redis values.
// So it thinks there hasn't been any usage and increments the current day/hour.
// After that it increments the next day/hour.
// We're only checking this for the first 4 requests
// because once the limit is exceeded the counts aren't incremented.
const first = i === 1;
const day = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[0], m);
const hour = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[1], m);
const minute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
if (!first) {
// The first request takes longer to serve because the document gets loaded,
// so only check the TTL (which gets set before request processing starts) on subsequent requests.
assert.deepEqual(
await redisClient.multi()
.ttl(minute)
.ttl(hour)
.ttl(day)
.execAsync(),
[
2 * 60,
2 * 60 * 60,
2 * 60 * 60 * 24,
],
);
}
assert.deepEqual(
await redisClient.multi()
.get(minute)
.get(hour)
.get(day)
.execAsync(),
[
String(i),
String(used + (first ? 1 : i - 1)),
String(used + (first ? 1 : i - 1)),
],
);
}
if (last) {
assert.equal(response.status, 429);
assert.deepEqual(response.data, {error: `Exceeded daily limit for document ${docId}`});
}
}
});
it("correctly allocates API requests based on the day, hour, and minute", async function() {
const m = moment.utc("1999-12-31T23:59:59Z");
const docId = "myDocId";
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const currentMinute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
assert.equal(currentDay, `doc-myDocId-periodicApiUsage-1999-12-31`);
assert.equal(currentHour, `doc-myDocId-periodicApiUsage-1999-12-31T23`);
assert.equal(currentMinute, `doc-myDocId-periodicApiUsage-1999-12-31T23:59`);
assert.equal(nextDay, `doc-myDocId-periodicApiUsage-2000-01-01`);
assert.equal(nextHour, `doc-myDocId-periodicApiUsage-2000-01-01T00`);
const usage = new LRUCache<string, number>({max: 1024});
function check(expected: string[] | undefined) {
assert.deepEqual(getDocApiUsageKeysToIncr(docId, usage, dailyMax, m), expected);
}
const dailyMax = 5000;
const hourlyMax = 209; // 5000/24 ~= 208.33
const minuteMax = 4; // 5000/24/60 ~= 3.47
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax - 1);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
check([nextDay, currentHour, currentMinute]); // used up daily allocation
usage.set(currentHour, hourlyMax - 1);
check([nextDay, currentHour, currentMinute]);
usage.set(currentHour, hourlyMax);
check([nextDay, nextHour, currentMinute]); // used up hourly allocation
usage.set(currentMinute, minuteMax - 1);
check([nextDay, nextHour, currentMinute]);
usage.set(currentMinute, minuteMax);
check(undefined); // used up minutely allocation
usage.set(currentDay, 0);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
usage.set(currentHour, 0);
check([nextDay, currentHour, currentMinute]);
});
after(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
await redisClient.quitAsync();
});
});
describe("Webhooks", () => {
let serving: Serving; // manages the test webhook server
let requests: WebhookRequests;
let receivedLastEvent: Promise<void>;
// Requests corresponding to adding 200 rows, sent in two batches of 100
const expected200AddEvents = [
_.range(100).map(i => ({
id: 9 + i, manualSort: 9 + i, A3: 200 + i, B3: true,
})),
_.range(100).map(i => ({
id: 109 + i, manualSort: 109 + i, A3: 300 + i, B3: true,
})),
];
// Every event is sent to three webhook URLs which differ by the subscribed eventTypes
// Each request is an array of one or more events.
// Multiple events caused by the same action bundle get batched into a single request.
const expectedRequests: WebhookRequests = {
"add": [
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
// After isReady (B) went to false and then true again
// we treat this as creation even though it's really an update
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 3, A3: 13, B3: true, manualSort: 3},
{id: 5, A3: 15, B3: true, manualSort: 5}],
[{id: 7, A3: 18, B3: true, manualSort: 7}],
...expected200AddEvents,
],
"update": [
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}],
],
"add,update": [
// add
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// update
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// from the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}, // update
{id: 3, A3: 13, B3: true, manualSort: 3}, // add
{id: 5, A3: 15, B3: true, manualSort: 5}], // add
[{id: 7, A3: 18, B3: true, manualSort: 7}], // add
...expected200AddEvents,
]
};
let redisMonitor: any;
let redisCalls: any[];
before(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
requests = {
"add,update": [],
"add": [],
"update": [],
};
let resolveReceivedLastEvent: () => void;
receivedLastEvent = new Promise<void>(r => {
resolveReceivedLastEvent = r;
});
// TODO test retries on failure and slowness in a new test
serving = await serveSomething(app => {
app.use(bodyParser.json());
app.post('/:eventTypes', async ({body, params: {eventTypes}}, res) => {
requests[eventTypes as keyof WebhookRequests].push(body);
res.sendStatus(200);
if (
_.flattenDeep(_.values(requests)).length >=
_.flattenDeep(_.values(expectedRequests)).length
) {
resolveReceivedLastEvent();
}
});
}, webhooksTestPort);
redisCalls = [];
redisMonitor = createClient(process.env.TEST_REDIS_URL);
redisMonitor.monitor();
redisMonitor.on("monitor", (_time: any, args: any, _rawReply: any) => {
redisCalls.push(args);
});
});
after(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
await serving.shutdown();
await redisMonitor.quitAsync();
});
it("delivers expected payloads from combinations of changes, with retrying and batching", async function() {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// For some reason B is turned into Numeric even when given bools
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
// Make a webhook for every combination of event types
const subscribeResponses = [];
const webhookIds: Record<string, string> = {};
for (const eventTypes of [
["add"],
["update"],
["add", "update"],
]) {
const {data, status} = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table1/_subscribe`,
{eventTypes, url: `${serving.url}/${eventTypes}`, isReadyColumn: "B"}, chimpy
);
assert.equal(status, 200);
subscribeResponses.push(data);
webhookIds[data.webhookId] = String(eventTypes);
}
// Add and update some rows, trigger some events
// Values of A where B is true and thus the record is ready are [1, 4, 7, 8]
// So those are the values seen in expectedEvents
await doc.addRows("Table1", {
A: [1, 2],
B: [true, false], // 1 is ready, 2 is not ready yet
});
await doc.updateRows("Table1", {id: [2], A: [3]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [4], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [5], B: [false]}); // not ready again
await doc.updateRows("Table1", {id: [2], A: [6]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [7], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [8]}); // still ready!
// The end result here is additions for column A (now A3) with values [13, 15, 18]
// and an update for 101
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['BulkAddRecord', 'Table1', [3, 4, 5, 6], {A: [9, 10, 11, 12], B: [true, true, false, false]}],
['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], {
A: [101, 102, 13, 14, 15, 16],
B: [true, false, true, false, true, false],
}],
['RenameColumn', 'Table1', 'A', 'A3'],
['RenameColumn', 'Table1', 'B', 'B3'],
['RenameTable', 'Table1', 'Table12'],
// FIXME a double rename A->A2->A3 doesn't seem to get summarised correctly
// ['RenameColumn', 'Table12', 'A2', 'A3'],
// ['RenameColumn', 'Table12', 'B2', 'B3'],
['RemoveColumn', 'Table12', 'C'],
], chimpy);
// FIXME record changes after a RenameTable in the same bundle
// don't appear in the action summary
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['AddRecord', 'Table12', 7, {A3: 17, B3: false}],
['UpdateRecord', 'Table12', 7, {A3: 18, B3: true}],
['AddRecord', 'Table12', 8, {A3: 19, B3: true}],
['UpdateRecord', 'Table12', 8, {A3: 20, B3: false}],
['AddRecord', 'Table12', 9, {A3: 20, B3: true}],
['RemoveRecord', 'Table12', 9],
], chimpy);
// Add 200 rows. These become the `expected200AddEvents`
await doc.addRows("Table12", {
A3: _.range(200, 400),
B3: arrayRepeat(200, true),
});
await receivedLastEvent;
// Unsubscribe
await Promise.all(subscribeResponses.map(async subscribeResponse => {
const unsubscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table12/_unsubscribe`,
subscribeResponse, chimpy
);
assert.equal(unsubscribeResponse.status, 200);
assert.deepEqual(unsubscribeResponse.data, {success: true});
}));
// Further changes should generate no events because the triggers are gone
await doc.addRows("Table12", {
A3: [88, 99],
B3: [true, false],
});
assert.deepEqual(requests, expectedRequests);
// Check that the events were all pushed to the redis queue
const queueRedisCalls = redisCalls.filter(args => args[1] === "webhook-queue-" + docId);
const redisPushes = _.chain(queueRedisCalls)
.filter(args => args[0] === "rpush") // Array<["rpush", key, ...events: string[]]>
.flatMap(args => args.slice(2)) // events: string[]
.map(JSON.parse) // events: WebhookEvent[]
.groupBy('id') // {[webHookId: string]: WebhookEvent[]}
.mapKeys((_value, key) => webhookIds[key]) // {[eventTypes: 'add'|'update'|'add,update']: WebhookEvent[]}
.mapValues(group => _.map(group, 'payload')) // {[eventTypes: 'add'|'update'|'add,update']: RowRecord[]}
.value();
const expectedPushes = _.mapValues(expectedRequests, value => _.flatten(value));
assert.deepEqual(redisPushes, expectedPushes);
// Check that the events were all removed from the redis queue
const redisTrims = queueRedisCalls.filter(args => args[0] === "ltrim")
.map(([,, start, end]) => {
assert.equal(end, '-1');
start = Number(start);
assert.isTrue(start > 0);
return start;
});
const expectedTrims = Object.values(redisPushes).map(value => value.length);
assert.equal(
_.sum(redisTrims),
_.sum(expectedTrims),
);
});
});
// PLEASE ADD MORE TESTS HERE
}
interface WebhookRequests {
add: object[][];
update: object[][];
"add,update": object[][];
}
function setup(name: string, cb: () => Promise<void>) {
let api: UserAPIImpl;
before(async function() {
suitename = name;
dataDir = path.join(tmpDir, `${suitename}-data`);
await fse.mkdirs(dataDir);
await setupDataDir(dataDir);
await cb();
// create TestDoc as an empty doc into Private workspace
userApi = api = makeUserApi('docs-1');
const wid = await getWorkspaceId(api, 'Private');
docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid);
});
after(async function() {
// remove TestDoc
await api.deleteDoc(docIds.TestDoc);
delete docIds.TestDoc;
// stop all servers
await home.stop();
await docs.stop();
});
}
function makeUserApi(org: string) {
return new UserAPIImpl(`${home.serverUrl}/o/${org}`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
}
async function getWorkspaceId(api: UserAPIImpl, name: string) {
const workspaces = await api.getOrgWorkspaces('current');
return workspaces.find((w) => w.name === name)!.id;
}
async function startServer(serverTypes: string, _homeUrl?: string): Promise<TestServer> {
const server = new TestServer(serverTypes);
await server.start(_homeUrl);
return server;
}
const webhooksTestPort = 34365;
class TestServer {
public testingSocket: string;
public testingHooks: TestingHooksClient;
public serverUrl: string;
public stopped = false;
private _server: ChildProcess;
private _exitPromise: Promise<number|string>;
constructor(private _serverTypes: string) {}
public async start(_homeUrl?: string) {
// put node logs into files with meaningful name that relate to the suite name and server type
const fixedName = this._serverTypes.replace(/,/, '_');
const nodeLogPath = path.join(tmpDir, `${suitename}-${fixedName}-node.log`);
const nodeLogFd = await fse.open(nodeLogPath, 'a');
const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd;
// use a path for socket that relates to suite name and server types
this.testingSocket = path.join(tmpDir, `${suitename}-${fixedName}.socket`);
// env
const env = {
GRIST_DATA_DIR: dataDir,
GRIST_INST_DIR: tmpDir,
GRIST_SERVERS: this._serverTypes,
// with port '0' no need to hard code a port number (we can use testing hooks to find out what
// port server is listening on).
GRIST_PORT: '0',
GRIST_TESTING_SOCKET: this.testingSocket,
GRIST_DISABLE_S3: 'true',
REDIS_URL: process.env.TEST_REDIS_URL,
APP_HOME_URL: _homeUrl,
ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
...process.env
};
const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
this._server = spawn('node', [main, '--testingHooks'], {
env,
stdio: ['inherit', serverLog, serverLog]
});
this._exitPromise = exitPromise(this._server);
// Try to be more helpful when server exits by printing out the tail of its log.
this._exitPromise.then((code) => {
if (this._server.killed) { return; }
log.error("Server died unexpectedly, with code", code);
const output = execFileSync('tail', ['-30', nodeLogPath]);
log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`);
})
.catch(() => undefined);
await this._waitServerReady(30000);
log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
}
public async stop() {
if (this.stopped) { return; }
log.info("Stopping node server: " + this._serverTypes);
this.stopped = true;
this._server.kill();
this.testingHooks.close();
await this._exitPromise;
}
public async isServerReady(): Promise<boolean> {
// Let's wait for the testingSocket to be created, then get the port the server is listening on,
// and then do an api check. This approach allow us to start server with GRIST_PORT set to '0',
// which will listen on first available port, removing the need to hard code a port number.
try {
// wait for testing socket
while (!(await fse.pathExists(this.testingSocket))) {
await delay(200);
}
// create testing hooks and get own port
this.testingHooks = await connectTestingHooks(this.testingSocket);
const port: number = await this.testingHooks.getOwnPort();
this.serverUrl = `http://localhost:${port}`;
// wait for check
return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
} catch (err) {
return false;
}
}
private async _waitServerReady(ms: number) {
// It's important to clear the timeout, because it can prevent node from exiting otherwise,
// which is annoying when running only this test for debugging.
let timeout: any;
const maxDelay = new Promise((resolve) => {
timeout = setTimeout(resolve, 30000);
});
try {
await Promise.race([
this.isServerReady(),
this._exitPromise.then(() => { throw new Error("Server exited while waiting for it"); }),
maxDelay,
]);
} finally {
clearTimeout(timeout);
}
}
}
async function setupDataDir(dir: string) {
// we'll be serving Hello.grist content for various document ids, so let's make copies of it in
// tmpDir
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Timesheets + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Bananas + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Antartic + '.grist'));
await testUtils.copyFixtureDoc(
'ApiDataRecordsTest.grist',
path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
}
Reference in New Issue View Git Blame Copy Permalink