Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity
8bc5c7d595
gristlabs_grist-core/test/server/lib/DocApi.ts
Alex Hall e06f0bc1d8 (core) Retry flaky daily API usage test 
Summary: This particular test fails often enough to be annoying but not often enough to be worrying. It's not clear why, but it seems like a race condition involving redis. Fixing the test 'properly' seems hard and not worth the effort. Looking at the past 20 Jenkins builds, I found this test failing once. If we assume that the probability of failing is 1/20 (it's probably less since the test actually runs 3 times with different server configurations) then the probability of failing 3 times independently is (1/20)^3 = 1/8000, so `this.retry(3)` seems like a good enough solution. See also: https://grist.slack.com/archives/C0234CPPXPA/p1652909955773049

Test Plan: Manually made the test fail randomly 90% of the time, in which case `this.retries(3)` was not usually enough to prevent failures, but `this.retries(300)` was.

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3595
2022-08-23 15:22:08 +02:00

2916 lines
111 KiB
TypeScript
Raw Blame History

import {ActionSummary} from 'app/common/ActionSummary';
import {BulkColValues, UserAction} from 'app/common/DocActions';
import {arrayRepeat} from 'app/common/gutil';
import {DocState, UserAPIImpl} from 'app/common/UserAPI';
import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
import {CellValue, GristObjCode} from 'app/plugin/GristData';
import {
applyQueryParameters,
docApiUsagePeriods,
docPeriodicApiUsageKey,
getDocApiUsageKeysToIncr
} from 'app/server/lib/DocApi';
import log from 'app/server/lib/log';
import {exitPromise} from 'app/server/lib/serverUtils';
import {connectTestingHooks, TestingHooksClient} from 'app/server/lib/TestingHooks';
import axios, {AxiosResponse} from 'axios';
import {delay} from 'bluebird';
import * as bodyParser from 'body-parser';
import {assert} from 'chai';
import {ChildProcess, execFileSync, spawn} from 'child_process';
import FormData from 'form-data';
import * as fse from 'fs-extra';
import * as _ from 'lodash';
import LRUCache from 'lru-cache';
import * as moment from 'moment';
import fetch from 'node-fetch';
import {tmpdir} from 'os';
import * as path from 'path';
import {createClient, RedisClient} from 'redis';
import {configForUser} from 'test/gen-server/testUtils';
import {serveSomething, Serving} from 'test/server/customUtil';
import * as testUtils from 'test/server/testUtils';
import clone = require('lodash/clone');
import defaultsDeep = require('lodash/defaultsDeep');
const chimpy = configForUser('Chimpy');
const kiwi = configForUser('Kiwi');
const charon = configForUser('Charon');
const nobody = configForUser('Anonymous');
const support = configForUser('support');
// some doc ids
const docIds: {[name: string]: string} = {
ApiDataRecordsTest: 'sample_7',
Timesheets: 'sample_13',
Bananas: 'sample_6',
Antartic: 'sample_11'
};
// A testDir of the form grist_test_{USER}_{SERVER_NAME}
const username = process.env.USER || "nobody";
const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi`);
let dataDir: string;
let suitename: string;
let serverUrl: string;
let homeUrl: string;
let hasHomeApi: boolean;
let home: TestServer;
let docs: TestServer;
let userApi: UserAPIImpl;
describe('DocApi', function() {
this.timeout(20000);
testUtils.setTmpLogLevel('error');
const oldEnv = clone(process.env);
before(async function() {
// Clear redis test database if redis is in use.
if (process.env.TEST_REDIS_URL) {
const cli = createClient(process.env.TEST_REDIS_URL);
await cli.flushdbAsync();
await cli.quitAsync();
}
// Create the tmp dir removing any previous one
await fse.remove(tmpDir);
await fse.mkdirs(tmpDir);
log.warn(`Test logs and data are at: ${tmpDir}/`);
// Let's create a sqlite db that we can share with servers that run in other processes, hence
// not an in-memory db. Running seed.ts directly might not take in account the most recent value
// for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
// configuration (in-memory for instance). Spawning a process is one way to make sure that the
// latest value prevail.
process.env.TYPEORM_DATABASE = path.join(tmpDir, 'landing.db');
const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
execFileSync('node', [seed, 'init'], {
env: process.env,
stdio: 'inherit'
});
});
after(() => {
Object.assign(process.env, oldEnv);
});
/**
* Doc api tests are run against three different setup:
* - a merged server: a single server serving both as a home and doc worker
* - two separated servers: requests are sent to a home server which then forward them to a doc worker
* - a doc worker: request are sent directly to the doc worker (note that even though it is not
* used for testing we starts anyway a home server, needed for setting up the test cases)
*
* Future tests must be added within the testDocApi() function.
*/
describe("should work with a merged server", async () => {
setup('merged', async () => {
home = docs = await startServer('home,docs');
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
// the way these tests are written, non-merged server requires redis.
if (process.env.TEST_REDIS_URL) {
describe("should work with a home server and a docworker", async () => {
setup('separated', async () => {
home = await startServer('home');
docs = await startServer('docs', home.serverUrl);
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
describe("should work directly with a docworker", async () => {
setup('docs', async () => {
home = await startServer('home');
docs = await startServer('docs', home.serverUrl);
homeUrl = home.serverUrl;
serverUrl = docs.serverUrl;
hasHomeApi = false;
});
testDocApi();
});
}
describe("QueryParameters", async () => {
function makeExample() {
return {
id: [ 1, 2, 3, 7, 8, 9 ],
color: ['red', 'yellow', 'white', 'blue', 'black', 'purple'],
spin: [ 'up', 'up', 'down', 'down', 'up', 'up'],
};
}
it("supports ascending sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['color']}, null), {
id: [8, 7, 9, 1, 3, 2],
color: ['black', 'blue', 'purple', 'red', 'white', 'yellow'],
spin: ['up', 'down', 'up', 'up', 'down', 'up']
});
});
it("supports descending sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-id']}, null), {
id: [9, 8, 7, 3, 2, 1],
color: ['purple', 'black', 'blue', 'white', 'yellow', 'red'],
spin: ['up', 'up', 'down', 'down', 'up', 'up'],
});
});
it("supports multi-key sort", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-spin', 'color']}, null), {
id: [8, 9, 1, 2, 7, 3],
color: ['black', 'purple', 'red', 'yellow', 'blue', 'white'],
spin: ['up', 'up', 'up', 'up', 'down', 'down'],
});
});
it("does not freak out sorting mixed data", async function() {
const example = {
id: [ 1, 2, 3, 4, 5, 6, 7, 8, 9],
mixed: ['red', 'green', 'white', 2.5, 1, null, ['zing', 3] as any, 5, 'blue']
};
assert.deepEqual(applyQueryParameters(example, {sort: ['mixed']}, null), {
mixed: [1, 2.5, 5, null, ['zing', 3] as any, 'blue', 'green', 'red', 'white'],
id: [5, 4, 8, 6, 7, 9, 2, 1, 3],
});
});
it("supports limit", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {limit: 1}),
{ id: [1], color: ['red'], spin: ['up'] });
});
it("supports sort and limit", async function() {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-color'], limit: 2}, null),
{ id: [2, 3], color: ['yellow', 'white'], spin: ['up', 'down'] });
});
});
});
// Contains the tests. This is where you want to add more test.
function testDocApi() {
it("guesses types of new columns", async () => {
const userActions = [
['AddTable', 'GuessTypes', []],
// Make 5 blank columns of type Any
['AddColumn', 'GuessTypes', 'Date', {}],
['AddColumn', 'GuessTypes', 'DateTime', {}],
['AddColumn', 'GuessTypes', 'Bool', {}],
['AddColumn', 'GuessTypes', 'Numeric', {}],
['AddColumn', 'GuessTypes', 'Text', {}],
// Add string values from which the initial type will be guessed
['AddRecord', 'GuessTypes', null, {
Date: "1970-01-02",
DateTime: "1970-01-02 12:00",
Bool: "true",
Numeric: "1.2",
Text: "hello",
}],
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
// Check that the strings were parsed to typed values
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/records`, chimpy)).data,
{
records: [
{
id: 1,
fields: {
Date: 24 * 60 * 60,
DateTime: 36 * 60 * 60,
Bool: true,
Numeric: 1.2,
Text: "hello",
},
},
],
},
);
// Check the column types
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/columns`, chimpy))
.data.columns.map((col: any) => col.fields.type),
["Date", "DateTime:UTC", "Bool", "Numeric", "Text"],
);
});
for (const mode of ['logged in', 'anonymous']) {
for (const content of ['with content', 'without content']) {
it(`POST /api/docs ${content} creates an unsaved doc when ${mode}`, async function() {
const user = (mode === 'logged in') ? chimpy : nobody;
const formData = new FormData();
formData.append('upload', 'A,B\n1,2\n3,4\n', 'table1.csv');
const config = defaultsDeep({headers: formData.getHeaders()}, user);
let resp = await axios.post(`${serverUrl}/api/docs`,
...(content === 'with content' ? [formData, config] : [null, user]));
assert.equal(resp.status, 200);
const urlId = resp.data;
if (mode === 'logged in') {
assert.match(urlId, /^new~[^~]*~[0-9]+$/);
} else {
assert.match(urlId, /^new~[^~]*$/);
}
// Access information about that document should be sane for current user
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, user);
assert.equal(resp.status, 200);
assert.equal(resp.data.name, 'Untitled');
assert.equal(resp.data.workspace.name, 'Examples & Templates');
assert.equal(resp.data.access, 'owners');
if (mode === 'anonymous') {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, chimpy);
assert.equal(resp.data.access, 'owners');
} else {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, charon);
assert.equal(resp.status, 403);
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, nobody);
assert.equal(resp.status, 403);
}
// content was successfully stored
resp = await axios.get(`${serverUrl}/api/docs/${urlId}/tables/Table1/data`, user);
if (content === 'with content') {
assert.deepEqual(resp.data, { id: [ 1, 2 ], manualSort: [ 1, 2 ], A: [ 1, 3 ], B: [ 2, 4 ] });
} else {
assert.deepEqual(resp.data, { id: [], manualSort: [], A: [], B: [], C: [] });
}
});
}
}
it("GET /docs/{did}/tables/{tid}/data retrieves data in column format", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
});
it("GET /docs/{did}/tables/{tid}/records retrieves data in records format", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'hello',
B: '',
C: '',
D: null,
E: 'HELLO',
},
},
{
id: 2,
fields: {
A: '',
B: 'world',
C: '',
D: null,
E: '',
},
},
{
id: 3,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
{
id: 4,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
]
});
});
it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"records": [
{
"id": 1,
"fields": {
"A": null,
"B": "Hi",
"C": 1,
},
"errors": {
"A": "ZeroDivisionError"
}
}
]
}
);
// /data format for comparison: includes manualSort, gristHelper_Display, and ["E", "ZeroDivisionError"]
resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"id": [
1
],
"manualSort": [
1
],
"A": [
[
"E",
"ZeroDivisionError"
]
],
"B": [
"Hi"
],
"C": [
1
],
"gristHelper_Display": [
"Hi"
]
}
);
});
it("GET /docs/{did}/tables/{tid}/columns retrieves columns", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
columns: [
{
id: 'A',
fields: {
colRef: 2,
parentId: 1,
parentPos: 1,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'A',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'B',
fields: {
colRef: 3,
parentId: 1,
parentPos: 2,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'B',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'C',
fields: {
colRef: 4,
parentId: 1,
parentPos: 3,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'C',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'D',
fields: {
colRef: 5,
parentId: 1,
parentPos: 3,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '',
label: 'D',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'E',
fields: {
colRef: 6,
parentId: 1,
parentPos: 4,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '$A.upper()',
label: 'E',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
}
]
}
);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent doc", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent table", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent doc", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent table", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/data supports filters", async function() {
function makeQuery(filters: {[colId: string]: any[]}) {
const query = "filter=" + encodeURIComponent(JSON.stringify(filters));
return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy);
}
function checkResults(resp: AxiosResponse<any>, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery({B: ['world']}), {
id: [2], A: [''], B: ['world'], C: [''], D: [null], E: [''], manualSort: [2],
});
// Can query by id
checkResults(await makeQuery({id: [1]}), {
id: [1], A: ['hello'], B: [''], C: [''], D: [null], E: ['HELLO'], manualSort: [1],
});
checkResults(await makeQuery({B: [''], A: ['']}), {
id: [3, 4], A: ['', ''], B: ['', ''], C: ['', ''], D: [null, null], E: ['', ''], manualSort: [3, 4],
});
// Empty filter is equivalent to no filter and should return full data.
checkResults(await makeQuery({}), {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
// An impossible filter should succeed but return an empty set of rows.
checkResults(await makeQuery({B: ['world'], C: ['Neptune']}), {
id: [], A: [], B: [], C: [], D: [], E: [], manualSort: [],
});
// An invalid filter should return an error
{
const resp = await makeQuery({BadCol: ['']});
assert.equal(resp.status, 400);
assert.match(resp.data.error, /BadCol/);
}
{
const resp = await makeQuery({B: 'world'} as any);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /filter values must be arrays/);
}
});
for (const mode of ['url', 'header']) {
it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function() {
function makeQuery(sort: string[]|null, limit: number|null) {
const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
const config = configForUser('chimpy');
if (mode === 'url') {
if (sort) { url.searchParams.append('sort', sort.join(',')); }
if (limit) { url.searchParams.append('limit', String(limit)); }
} else {
if (sort) { config.headers['x-sort'] = sort.join(','); }
if (limit) { config.headers['x-limit'] = String(limit); }
}
return axios.get(url.href, config);
}
function checkResults(resp: AxiosResponse<any>, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery(['-id'], null), {
id: [4, 3, 2, 1],
A: ['', '', '', 'hello'],
B: ['', '', 'world', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['', '', '', 'HELLO'],
manualSort: [4, 3, 2, 1]
});
checkResults(await makeQuery(['-id'], 2), {
id: [4, 3],
A: ['', ''],
B: ['', ''],
C: ['', ''],
D: [null, null],
E: ['', ''],
manualSort: [4, 3]
});
});
}
it("GET /docs/{did}/tables/{tid}/data respects document permissions", async function() {
// as not part of any group kiwi cannot fetch Timesheets
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, kiwi);
assert.equal(resp.status, 403);
});
it("GET /docs/{did}/tables/{tid}/data returns matches /not found/ for bad table id", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/apply applies user actions", async function() {
const userActions = [
['AddTable', 'Foo', [{id: 'A'}, {id: 'B'}]],
['BulkAddRecord', 'Foo', [1, 2], {A: ["Santa", "Bob"], B: [1, 11]}]
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data,
{id: [1, 2], A: ['Santa', 'Bob'], B: ['1', '11'], manualSort: [1, 2]});
});
it("POST /docs/{did}/apply respects document permissions", async function() {
const userActions = [
['AddTable', 'FooBar', [{id: 'A'}]]
];
let resp: AxiosResponse;
// as a guest chimpy cannot edit Bananas
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Bananas}/apply`, userActions, chimpy);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Bananas}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
// as not in any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, kiwi);
assert.equal(resp.status, 403);
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/tables/{tid}/data adds records", async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3, 4]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("POST /docs/{did}/tables/{tid}/records adds records", async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{fields: {A: 'John', B: 55}},
{fields: {A: 'Jane', B: 0}},
]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
records: [
{id: 5},
{id: 6},
]
});
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'Santa',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
{
id: 5,
fields: {
A: 'John',
B: '55',
},
},
{
id: 6,
fields: {
A: 'Jane',
B: '0',
},
},
]
});
});
it("POST /docs/{did}/tables/{tid}/data/delete deletes records", async function() {
let resp = await axios.post(
`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data/delete`,
[3, 4, 5, 6],
chimpy,
);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, null);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2],
A: ['Santa', 'Bob'],
B: ["1", "11"],
manualSort: [1, 2]
});
// restore rows
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
function checkError(status: number, test: RegExp|object, resp: AxiosResponse<any>, message?: string) {
assert.equal(resp.status, status);
if (test instanceof RegExp) {
assert.match(resp.data.error, test, message);
} else {
try {
assert.deepEqual(resp.data, test, message);
} catch(err) {
console.log(JSON.stringify(resp.data));
console.log(JSON.stringify(test));
throw err;
}
}
}
it("parses strings in user actions", async () => {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const recordsUrl = `${docUrl}/tables/Table1/records`;
// Make the column numeric, delete the other columns we don't care about
await axios.post(`${docUrl}/apply`, [
['ModifyColumn', 'Table1', 'A', {type: 'Numeric'}],
['RemoveColumn', 'Table1', 'B'],
['RemoveColumn', 'Table1', 'C'],
], chimpy);
// Add/update some records without and with string parsing
// Specifically test:
// 1. /apply, with an AddRecord
// 2. POST /records (BulkAddRecord)
// 3. PATCH /records (BulkUpdateRecord)
// Send strings that look like currency which need string parsing to become numbers
for (const queryParams of ['?noparse=1', '']) {
await axios.post(`${docUrl}/apply${queryParams}`, [
['AddRecord', 'Table1', null, {'A': '$1'}],
], chimpy);
const response = await axios.post(`${recordsUrl}${queryParams}`,
{
records: [
{fields: {'A': '$2'}},
{fields: {'A': '$3'}},
]
},
chimpy);
// Update $3 -> $4
const rowId = response.data.records[1].id;
await axios.patch(`${recordsUrl}${queryParams}`,
{
records: [
{id: rowId, fields: {'A': '$4'}}
]
},
chimpy);
}
// Check the results
const resp = await axios.get(recordsUrl, chimpy);
assert.deepEqual(resp.data, {
records:
[
// Without string parsing
{id: 1, fields: {A: '$1'}},
{id: 2, fields: {A: '$2'}},
{id: 3, fields: {A: '$4'}},
// With string parsing
{id: 4, fields: {A: 1}},
{id: 5, fields: {A: 2}},
{id: 6, fields: {A: 4}},
]
}
);
});
describe("PUT /docs/{did}/tables/{tid}/records", async function() {
it("should add or update records", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
async function check(records: AddOrUpdateRecord[], expectedTableData: BulkColValues, params: any={}) {
const resp = await axios.put(url, {records}, {...chimpy, params});
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
delete table.C;
assert.deepStrictEqual(table, expectedTableData);
}
// Add 3 new records, since the table is empty so nothing matches `requires`
await check(
[
{
require: {A: 1},
},
{
// Since no record with A=2 is found, create a new record,
// but `fields` overrides `require` for the value when creating,
// so the new record has A=3
require: {A: 2},
fields: {A: 3},
},
{
require: {A: 4},
fields: {B: 5},
},
],
{id: [1, 2, 3], A: [1, 3, 4], B: [0, 0, 5]}
);
// Update all three records since they all match the `require` values here
await check(
[
{
// Does nothing
require: {A: 1},
},
{
// Changes A from 3 to 33
require: {A: 3},
fields: {A: 33},
},
{
// Changes B from 5 to 6 in the third record where A=4
require: {A: 4},
fields: {B: 6},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}
);
// This would normally add a record, but noadd suppresses that
await check([
{
require: {A: 100},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noadd: "1"},
);
// This would normally update A from 1 to 11, bot noupdate suppresses that
await check([
{
require: {A: 1},
fields: {A: 11},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noupdate: "1"},
);
// There are 2 records with B=0, update them both to B=1
// Use onmany=all to specify that they should both be updated
await check([
{
require: {B: 0},
fields: {B: 1},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [1, 1, 6]},
{onmany: "all"}
);
// In contrast to the above, the default behaviour for no value of onmany
// is to only update the first matching record,
// so only one of the records with B=1 is updated to B=2
await check([
{
require: {B: 1},
fields: {B: 2},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [2, 1, 6]},
);
// By default, strings in `require` and `fields` are parsed based on column type,
// so these dollar amounts are treated as currency
// and parsed as A=4 and A=44
await check([
{
require: {A: "$4"},
fields: {A: "$44"},
},
],
{id: [1, 2, 3], A: [1, 33, 44], B: [2, 1, 6]},
);
// Turn off the default string parsing with noparse=1
// Now we need A=44 to actually be a number to match,
// A="$44" wouldn't match and would create a new record.
// Because A="$55" isn't parsed, the raw string is stored in the table.
await check([
{
require: {A: 44},
fields: {A: "$55"},
},
],
{id: [1, 2, 3], A: [1, 33, "$55"], B: [2, 1, 6]},
{noparse: 1}
);
await check([
// First three records already exist and nothing happens
{require: {A: 1}},
{require: {A: 33}},
{require: {A: "$55"}},
// Without string parsing, A="$33" doesn't match A=33 and a new record is created
{require: {A: "$33"}},
],
{id: [1, 2, 3, 4], A: [1, 33, "$55", "$33"], B: [2, 1, 6, 0]},
{noparse: 1}
);
// Checking that updating by `id` works.
await check([
{
require: {id: 3},
fields: {A: "66"},
},
],
{id: [1, 2, 3, 4], A: [1, 33, 66, "$33"], B: [2, 1, 6, 0]},
);
// allow_empty_require option with empty `require` updates all records
await check([
{
require: {},
fields: {A: 99, B: 99},
},
],
{id: [1, 2, 3, 4], A: [99, 99, 99, 99], B: [99, 99, 99, 99]},
{allow_empty_require: "1", onmany: "all"},
);
});
it("should 404 for missing tables", async () => {
checkError(404, /Table not found "Bad_Foo_"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/records`,
{records: [{require: {id: 1}}]}, chimpy));
});
it("should 400 for missing columns", async () => {
checkError(400, /Invalid column "no_such_column"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {no_such_column: 1}}]}, chimpy));
});
it("should 400 for an incorrect onmany parameter", async function() {
checkError(400,
/onmany parameter foo should be one of first,none,all/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {id: 1}}]}, {...chimpy, params: {onmany: "foo"}}));
});
it("should 400 for an empty require without allow_empty_require", async function() {
checkError(400,
/require is empty but allow_empty_require isn't set/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {}}]}, chimpy));
});
it("should validate request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async (payload: any, error: { error: string, details: string }) => {
const resp = await axios.put(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await test({records: [{fields: {}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require is missing',
});
await test({records: [{require: {id: "1"}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require.id is not a number',
});
});
});
describe("POST /docs/{did}/tables/{tid}/records", async function() {
it("POST should have good errors", async () => {
checkError(404, /not found/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`,
{ A: ['Alice', 'Felix'], B: [2, 22] }, chimpy));
checkError(400, /Invalid column "Bad"/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{ A: ['Alice'], Bad: ['Monthy'] }, chimpy));
// Other errors should also be maximally informative.
checkError(400, /Error manipulating data/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{ A: ['Alice'], B: null }, chimpy));
});
it("validates request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async(payload: any, error: {error: string, details: string}) => {
const resp = await axios.post(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
// All column types are allowed, except Arrays (or objects) without correct code.
const testField = async (A: any) => {
await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a NewRecord; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'});
};
// test no code at all
await testField([]);
// test invalid code
await testField(['ZZ']);
});
it("allows to create a blank record", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
// Create two blank records
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const resp = await axios.post(url, {records: [{}, { fields: {}}]}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, { records : [{id: 1}, {id: 2}]});
});
it("allows to create partial records", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create partial records
const resp = await axios.post(url, {records: [{fields: { A: 1}}, { fields: {B: 2}}, {}]}, chimpy);
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
assert.deepStrictEqual(
table,
{ id: [1, 2, 3], A: [1, null, null], B: [null, 2, null], C:[null, null, null]});
});
it("allows CellValue as a field", async function() {
// create sample document
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'PostTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const testField = async(A?: CellValue, message?: string) =>{
const resp = await axios.post(url, {records: [{ fields: { A } }]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
// test allowed types for a field
await testField(1); // ints
await testField(1.2); // floats
await testField("string"); // strings
await testField(true); // true and false
await testField(false);
await testField(null); // null
// encoded values (though not all make sense)
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
it("POST /docs/{did}/tables/{tid}/data respects document permissions", async function() {
let resp: AxiosResponse;
const data = {
A: ['Alice', 'Felix'],
B: [2, 22]
};
// as a viewer charon cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that TestDoc did not change
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
describe("PATCH /docs/{did}/tables/{tid}/records", function() {
it("updates records", async function () {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{
id: 1,
fields: {
A: 'Father Christmas',
},
},
],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
records:
[
{
id: 1,
fields: {
A: 'Father Christmas',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
]
});
});
it("validates request schema", async function() {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
async function failsWithError(payload: any, error: { error: string, details?: string }){
const resp = await axios.patch(url, payload, chimpy);
checkError(400, error, resp);
}
await failsWithError({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await failsWithError({records: []}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; body.records[0] is not an object'});
await failsWithError({records: [{}]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n '+
'body.records[0].id is missing\n '+
'body.records[0].fields is missing'});
await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n' +
' body.records[0].id is not a number\n' +
' body.records[0].fields is missing'});
await failsWithError(
{records: [{id: 1, fields: {A : 1}}, {id: 2, fields: {B: 3}}]},
{error: 'PATCH requires all records to have same fields'});
// Test invalid object codes
const fieldIsNotValid = async (A: any) => {
await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'});
};
await fieldIsNotValid([]);
await fieldIsNotValid(['ZZ']);
});
it("allows CellValue as a field", async function() {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({ name : 'PatchTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create record for patching
const id = (await axios.post(url, { records: [{}] }, chimpy)).data.records[0].id;
const testField = async(A?: CellValue, message?: string) =>{
const resp = await axios.patch(url, {records: [{ id, fields: { A } }]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
await testField(1);
await testField(1.2);
await testField("string");
await testField(true);
await testField(false);
await testField(null);
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
describe("PATCH /docs/{did}/tables/{tid}/data", function() {
it("updates records", async function() {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Santa Klaus'],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid row ids", async function() {
// combination of valid and invalid ids fails
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 5/);
// only invalid ids also fails
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [10, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 10/);
// check that changes related to id 1 did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid column", async function() {
const resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Alice'],
C: ['Monthy']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid column "C"/);
});
it("respects document permissions", async function() {
let resp: AxiosResponse;
const data = {
id: [1],
A: ['Santa'],
};
// check data
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
// as a viewer charon cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that changes did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
});
describe('attachments', function() {
it("POST /docs/{did}/attachments adds attachments", async function() {
let formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2]);
// Another upload gets the next number.
formData = new FormData();
formData.append('upload', 'abcdef', "hello.png");
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3]);
});
it("GET /docs/{did}/attachments lists attachment metadata", async function() {
// Test that the usual /records query parameters like sort and filter also work
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/attachments?sort=-fileName&limit=2`;
const resp = await axios.get(url, chimpy);
assert.equal(resp.status, 200);
const {records} = resp.data;
for (const record of records) {
assert.match(record.fields.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
delete record.fields.timeUploaded;
}
assert.deepEqual(records, [
{id: 2, fields: {fileName: "world.jpg", fileSize: 6}},
{id: 3, fields: {fileName: "hello.png", fileSize: 6}},
]
);
});
it("GET /docs/{did}/attachments/{id} returns attachment metadata", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2`, chimpy);
assert.equal(resp.status, 200);
assert.include(resp.data, {fileName: "world.jpg", fileSize: 6});
assert.match(resp.data.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
});
it("GET /docs/{did}/attachments/{id}/download downloads attachment contents", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}/download works after doc shutdown", async function() {
// Check that we can download when ActiveDoc isn't currently open.
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}... returns 404 when attachment not found", async function() {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo`, chimpy);
checkError(400, /parameter cannot be understood as an integer: moo/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22/download`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo/download`, chimpy);
checkError(400, /parameter cannot be understood as an integer: moo/, resp);
});
it("POST /docs/{did}/attachments produces reasonable errors", async function() {
// Check that it produces reasonable errors if we try to use it with non-form-data
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, [4, 5, 6], chimpy);
assert.equal(resp.status, 415); // Wrong content-type
// Check for an error if there is no data included.
const formData = new FormData();
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 400);
// TODO The error here is "stream ended unexpectedly", which isn't really reasonable.
});
it("POST/GET /docs/{did}/attachments respect document permissions", async function() {
const formData = new FormData();
formData.append('upload', 'xyzzz', "wrong.png");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, kiwi));
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3`, kiwi);
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3/download`, kiwi);
checkError(403, /No view access/, resp);
});
it("POST /docs/{did}/attachments respects untrusted content-type only if valid", async function() {
const formData = new FormData();
formData.append('upload', 'xyz', {filename: "foo", contentType: "application/pdf"});
formData.append('upload', 'abc', {filename: "hello.png", contentType: "invalid/content-type"});
formData.append('upload', 'def', {filename: "world.doc", contentType: "text/plain\nbad-header: 1\n\nEvil"});
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [4, 5, 6]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/4/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/pdf'); // A valid content-type is respected
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="foo.pdf"');
assert.deepEqual(resp.data, 'xyz');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/5/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/png'); // Did not pay attention to invalid header
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="hello.png"');
assert.deepEqual(resp.data, 'abc');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/6/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/msword'); // Another invalid header ignored
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.doc"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.headers['bad-header'], undefined); // Attempt to hack in more headers didn't work
assert.deepEqual(resp.data, 'def');
});
it("POST /docs/{did}/attachments/updateUsed updates timeDeleted on metadata", async function() {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc2'}, wid);
// Apply the given user actions,
// POST to /attachments/updateUsed
// Check that Table1 and _grist_Attachments contain the expected rows
async function check(
actions: UserAction[],
userData: { id: number, Attached: any }[],
metaData: { id: number, deleted: boolean }[],
) {
const docUrl = `${serverUrl}/api/docs/${docId}`;
let resp = await axios.post(`${docUrl}/apply`, actions, chimpy);
assert.equal(resp.status, 200);
resp = await axios.post(`${docUrl}/attachments/updateUsed`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${docUrl}/tables/Table1/records`, chimpy);
const actualUserData = resp.data.records.map(
({id, fields: {Attached}}: ApiRecord) =>
({id, Attached})
);
assert.deepEqual(actualUserData, userData);
resp = await axios.get(`${docUrl}/tables/_grist_Attachments/records`, chimpy);
const actualMetaData = resp.data.records.map(
({id, fields: {timeDeleted}}: ApiRecord) =>
({id, deleted: Boolean(timeDeleted)})
);
assert.deepEqual(actualMetaData, metaData);
}
// Set up the document and initial data.
await check(
[
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkAddRecord", "Table1", [1, 2], {Attached: [['L', 1], ['L', 2, 3]]}],
// There's no actual attachments here but that doesn't matter
["BulkAddRecord", "_grist_Attachments", [1, 2, 3], {}],
],
[
{id: 1, Attached: ['L', 1]},
{id: 2, Attached: ['L', 2, 3]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false},
{id: 3, deleted: false},
],
);
// Remove the record containing ['L', 2, 3], so the metadata for 2 and 3 now says deleted
await check(
[["RemoveRecord", "Table1", 2]],
[
{id: 1, Attached: ['L', 1]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true}, // deleted here
],
);
// Add back a reference to attacument 2 to test 'undeletion', plus some junk values
await check(
[["BulkAddRecord", "Table1", [3, 4, 5], {Attached: [null, "foo", ['L', 2, 2, 4, 4, 5]]}]],
[
{id: 1, Attached: ['L', 1]},
{id: 3, Attached: null},
{id: 4, Attached: "foo"},
{id: 5, Attached: ['L', 2, 2, 4, 4, 5]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false}, // undeleted here
{id: 3, deleted: true},
],
);
// Remove the whole column to test what happens when there's no Attachment columns
await check(
[["RemoveColumn", "Table1", "Attached"]],
[
{id: 1, Attached: undefined},
{id: 3, Attached: undefined},
{id: 4, Attached: undefined},
{id: 5, Attached: undefined},
],
[
{id: 1, deleted: true}, // deleted here
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true},
],
);
// Test performance with a large number of records and attachments.
// The maximum value of numRecords that doesn't return a 413 error is about 18,000.
// In that case it took about 5.7 seconds to apply the initial user actions (i.e. add the records),
// 0.3 seconds to call updateUsed once, and 0.1 seconds to call it again immediately after.
// That last time roughly measures the time taken to do the SQL query
// without having to apply any user actions after to update timeDeleted.
// 10,000 records is a compromise so that tests aren't too slow.
const numRecords = 10000;
const attachmentsPerRecord = 4;
const totalUsedAttachments = numRecords * attachmentsPerRecord; // 40,000 attachments referenced in user data
const totalAttachments = totalUsedAttachments * 1.1; // 44,000 attachment IDs listed in metadata
const attachedValues = _.chunk(_.range(1, totalUsedAttachments + 1), attachmentsPerRecord)
.map(arr => ['L', ...arr]);
await check(
[
// Reset the state: add back the removed column and delete the previously added data
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkRemoveRecord", "Table1", [1, 3, 4, 5]],
["BulkRemoveRecord", "_grist_Attachments", [1, 2, 3]],
["BulkAddRecord", "Table1", arrayRepeat(numRecords, null), {Attached: attachedValues}],
["BulkAddRecord", "_grist_Attachments", arrayRepeat(totalAttachments, null), {}],
],
attachedValues.map((Attached, index) => ({id: index + 1, Attached})),
_.range(totalAttachments).map(index => ({id: index + 1, deleted: index >= totalUsedAttachments})),
);
});
it("POST /docs/{did}/attachments/removeUnused removes unused attachments", async function() {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc3'}, wid);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
formData.append('upload', 'foobar', "hello2.doc");
let resp = await axios.post(`${docUrl}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2, 3]);
async function checkAttachmentIds(ids: number[]) {
resp = await axios.get(`${docUrl}/attachments`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data.records.map((r: any) => r.id), ids);
}
resp = await axios.patch(
`${docUrl}/tables/_grist_Attachments/records`,
{
records: [
{id: 1, fields: {timeDeleted: Date.now() / 1000 - 8 * 24 * 60 * 60}}, // 8 days ago, i.e. expired
{id: 2, fields: {timeDeleted: Date.now() / 1000 - 6 * 24 * 60 * 60}}, // 6 days ago, i.e. not expired
]
},
chimpy,
);
assert.equal(resp.status, 200);
await checkAttachmentIds([1, 2, 3]);
// Remove the expired attachment (1) by force-reloading, so it removes it during shutdown.
// It has a duplicate (3) that hasn't expired and thus isn't removed,
// although they share the same fileIdent and row in _gristsys_Files.
// So for now only the metadata is removed.
resp = await axios.post(`${docUrl}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([2, 3]);
resp = await axios.post(`${docUrl}/attachments/verifyFiles`, null, chimpy);
assert.equal(resp.status, 200);
// Remove the not expired attachments (2 and 3).
// We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded.
resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([]);
});
});
it("GET /docs/{did}/download serves document", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, chimpy);
assert.equal(resp.status, 200);
assert.match(resp.data, /grist_Tables_column/);
});
it("GET /docs/{did}/download respects permissions", async function() {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, kiwi);
assert.equal(resp.status, 403);
assert.notMatch(resp.data, /grist_Tables_column/);
});
it("GET /docs/{did}/download/csv serves CSV-encoded document", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
assert.equal(resp.status, 200);
assert.equal(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
const resp2 = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Foo`, chimpy);
assert.equal(resp2.status, 200);
assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
});
it("GET /docs/{did}/download/csv respects permissions", async function() {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);
assert.equal(resp.status, 403);
assert.notEqual(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
});
it("GET /docs/{did}/download/csv returns 404 if tableId is invalid", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=MissingTableId`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, { error: 'Table MissingTableId not found.' });
});
it("GET /docs/{did}/download/csv returns 404 if viewSectionId is invalid", async function() {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1&viewSection=9999`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, { error: 'No record 9999 in table _grist_Views_section' });
});
it("GET /docs/{did}/download/csv returns 400 if tableId is missing", async function() {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv`, chimpy);
assert.equal(resp.status, 400);
assert.deepEqual(resp.data, { error: 'tableId parameter should be a string: undefined' });
});
it('POST /workspaces/{wid}/import handles empty filenames', async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
const worker1 = await userApi.getWorkerAPI('import');
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, '.grist');
const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
assert.equal(resp.data.title, 'Untitled upload');
assert.equal(typeof resp.data.id, 'string');
assert.notEqual(resp.data.id, '');
});
it("document is protected during upload-and-import sequence", async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
// Prepare an API for a different user.
const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
headers: {Authorization: 'Bearer api_key_for_kiwi'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
// upload something for Chimpy and something else for Kiwi.
const worker1 = await userApi.getWorkerAPI('import');
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, 'upload.grist');
const worker2 = await kiwiApi.getWorkerAPI('import');
const fakeData2 = await testUtils.readFixtureDoc('Favorite_Films.grist');
const uploadId2 = await worker2.upload(fakeData2, 'upload2.grist');
// Check that kiwi only has access to their own upload.
let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id;
let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Kiwi'));
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: "access denied"});
resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2},
configForUser('Kiwi'));
assert.equal(resp.status, 200);
// Check that chimpy has access to their own upload.
wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
});
it('limits parallel requests', async function() {
// Launch 30 requests in parallel and see how many are honored and how many
// return 429s. The timing of this test is a bit delicate. We close the doc
// to increase the odds that results won't start coming back before all the
// requests have passed authorization. May need to do something more sophisticated
// if this proves unreliable.
await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
const reqs = [...Array(30).keys()].map(
i => axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy));
const responses = await Promise.all(reqs);
assert.lengthOf(responses.filter(r => r.status === 200), 10);
assert.lengthOf(responses.filter(r => r.status === 429), 20);
});
it('allows forced reloads', async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force a reload.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('allows assignments', async function() {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force an assignment.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('honors urlIds', async function() {
// Make a document with a urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1);
try {
// Make sure an edit made by docId is visible when accessed via docId or urlId
let resp = await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
// Make sure an edit made by urlId is visible when accessed via docId or urlId
resp = await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
A: ['Orange'], B: [42]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
}
});
it('filters urlIds by org', async function() {
// Make two documents with same urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2);
try {
// Place a value in "docs" doc
await axios.post(`${serverUrl}/o/docs/api/docs/urlid/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
// Place a value in "nasa" doc
await axios.post(`${serverUrl}/o/nasa/api/docs/urlid/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
// Check the values made it to the right places
let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it('allows docId access to any document from merged org', async function() {
// Make two documents
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch : fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2);
try {
// Should fail to write to a document in "docs" from "nasa" url
let resp = await axios.post(`${serverUrl}/o/nasa/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
// Should successfully write to a document in "nasa" from "docs" url
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 200);
// Should fail to write to a document in "nasa" from "pr" url
resp = await axios.post(`${serverUrl}/o/pr/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it("GET /docs/{did}/replace replaces one document with another", async function() {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const doc2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc3 = await userApi.newDoc({name: 'testdoc2'}, ws1);
await userApi.updateDocPermissions(doc2, {users: {'kiwi@getgrist.com': 'editors'}});
await userApi.updateDocPermissions(doc3, {users: {'kiwi@getgrist.com': 'viewers'}});
try {
// Put some material in doc3
let resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc3}/tables/Table1/data`, {
A: ['Orange']
}, chimpy);
assert.equal(resp.status, 200);
// Kiwi can replace doc2 with doc3
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Orange');
// Kiwi can't replace doc1 with doc3, no write access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc1}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 403);
// Kiwi can't replace doc2 with doc1, no read access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc1
}, kiwi);
assert.equal(resp.status, 403);
} finally {
await userApi.deleteDoc(doc1);
await userApi.deleteDoc(doc2);
}
});
it("GET /docs/{did}/snapshots retrieves a list of snapshots", async function() {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/snapshots`, chimpy);
assert.equal(resp.status, 200);
assert.isAtLeast(resp.data.snapshots.length, 1);
assert.hasAllKeys(resp.data.snapshots[0], ['docId', 'lastModified', 'snapshotId']);
});
it("POST /docs/{did}/states/remove removes old states", async function() {
// Check doc has plenty of states.
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
const states: DocState[] = resp.data.states;
assert.isAbove(states.length, 5);
// Remove all but 3.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 3}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 3);
assert.equal(resp.data.states[0].h, states[0].h);
assert.equal(resp.data.states[1].h, states[1].h);
assert.equal(resp.data.states[2].h, states[2].h);
// Remove all but 1.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 1}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 1);
assert.equal(resp.data.states[0].h, states[0].h);
});
it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function() {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc1 = userApi.getDocAPI(docId1);
const doc2 = userApi.getDocAPI(docId2);
// Stick some content in column A so it has a defined type
// so diffs are smaller and simpler.
await doc2.addRows('Table1', {A: [0]});
let comp = await doc1.compareDoc(docId2);
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary']);
assert.equal(comp.summary, 'unrelated');
assert.equal(comp.parent, null);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 1);
assert.equal(comp.right.n, 2);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 2);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details, {
leftChanges: { tableRenames: [], tableDeltas: {} },
rightChanges: { tableRenames: [], tableDeltas: {} }
});
await doc1.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'left');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 2);
assert.deepEqual(comp.right, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.rightChanges,
{ tableRenames: [], tableDeltas: {} });
const addA1: ActionSummary = {
tableRenames: [],
tableDeltas: { Table1: {
updateRows: [],
removeRows: [],
addRows: [ 2 ],
columnDeltas: {
A: { [2]: [null, [1]] },
manualSort: { [2]: [null, [2]] },
},
columnRenames: [],
} }
};
assert.deepEqual(comp.details!.leftChanges, addA1);
await doc2.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'both');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 3);
assert.equal(comp.parent!.n, 2);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.leftChanges, addA1);
assert.deepEqual(comp.details!.rightChanges, addA1);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 3);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details, {
leftChanges: { tableRenames: [], tableDeltas: {} },
rightChanges: { tableRenames: [], tableDeltas: {} }
});
await doc2.addRows('Table1', {A: [2]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'right');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, { detail: true });
assert.deepEqual(comp.details!.leftChanges,
{ tableRenames: [], tableDeltas: {} });
const addA2: ActionSummary = {
tableRenames: [],
tableDeltas: { Table1: {
updateRows: [],
removeRows: [],
addRows: [ 3 ],
columnDeltas: {
A: { [3]: [null, [2]] },
manualSort: { [3]: [null, [3]] },
},
columnRenames: [],
} }
};
assert.deepEqual(comp.details!.rightChanges, addA2);
});
it("GET /docs/{did}/compare tracks changes within a doc", async function() {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// Give the document some history.
await doc.addRows('Table1', {A: ['a1'], B: ['b1']});
await doc.addRows('Table1', {A: ['a2'], B: ['b2']});
await doc.updateRows('Table1', {id: [1], A: ['A1']});
// Examine the most recent change, from HEAD~ to HEAD.
let comp = await doc.compareVersion('HEAD~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [],
columnDeltas: {
A: { [1]: [['a1'], ['A1']] }
},
columnRenames: [],
}
}
});
// Check we get the same result with actual hashes.
assert.notMatch(comp.left.h, /HEAD/);
assert.notMatch(comp.right.h, /HEAD/);
const comp2 = await doc.compareVersion(comp.left.h, comp.right.h);
assert.deepEqual(comp, comp2);
// Check that comparing the HEAD with itself shows no changes.
comp = await doc.compareVersion('HEAD', 'HEAD');
assert.equal(comp.summary, 'same');
assert.deepEqual(comp.parent, comp.left);
assert.deepEqual(comp.parent, comp.right);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, { tableRenames: [], tableDeltas: {} });
// Examine the combination of the last two changes.
comp = await doc.compareVersion('HEAD~~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 2);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, { tableRenames: [], tableDeltas: {} });
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [2],
columnDeltas: {
A: { [1]: [['a1'], ['A1']],
[2]: [null, ['a2']] },
B: { [2]: [null, ['b2']] },
manualSort: { [2]: [null, [2]] },
},
columnRenames: [],
}
}
});
});
it('doc worker endpoints ignore any /dw/.../ prefix', async function() {
const docWorkerUrl = docs.serverUrl;
let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${docWorkerUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
if (docWorkerUrl !== homeUrl) {
resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${homeUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
}
});
it("POST /docs/{did}/tables/{tid}/_subscribe validates inputs", async function () {
async function check(requestBody: any, status: number, error: string) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
requestBody, chimpy
);
assert.equal(resp.status, status);
assert.deepEqual(resp.data, {error});
}
await check({}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: 0}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: []}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: ["foo"]}, 400, "Allowed values in eventTypes are: add,update");
await check({eventTypes: ["add"]}, 400, "Bad request: url required");
await check({eventTypes: ["add"], url: "https://evil.com"}, 403, "Provided url is forbidden");
await check({eventTypes: ["add"], url: "http://example.com"}, 403, "Provided url is forbidden"); // not https
await check({eventTypes: ["add"], url: "https://example.com", isReadyColumn: "bar"}, 404, `Column not found "bar"`);
});
it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs", async function() {
const subscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
{eventTypes: ["add"], url: "https://example.com"}, chimpy
);
assert.equal(subscribeResponse.status, 200);
const {triggerId, unsubscribeKey, webhookId} = subscribeResponse.data;
async function check(requestBody: any, status: number, responseBody: any) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_unsubscribe`,
requestBody, chimpy
);
assert.equal(resp.status, status);
if (status !== 200) {
responseBody = {error: responseBody};
}
assert.deepEqual(resp.data, responseBody);
}
await check({triggerId: 999}, 404, `Trigger not found "999"`);
await check({triggerId, webhookId: "foo"}, 404, `Webhook not found "foo"`);
await check({triggerId, webhookId}, 400, 'Bad request: id and unsubscribeKey both required');
await check({triggerId, webhookId, unsubscribeKey: "foo"}, 401, 'Wrong unsubscribeKey');
// Actually unsubscribe
await check({triggerId, webhookId, unsubscribeKey}, 200, {success: true});
// Trigger is now deleted!
await check({triggerId, webhookId, unsubscribeKey}, 404, `Trigger not found "${triggerId}"`);
});
describe("Daily API Limit", () => {
let redisClient: RedisClient;
before(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
redisClient = createClient(process.env.TEST_REDIS_URL);
});
it("limits daily API usage", async function() {
// Make a new document in a test product with a low daily limit
const api = makeUserApi('testdailyapilimit');
const workspaceId = await getWorkspaceId(api, 'TestDailyApiLimitWs');
const docId = await api.newDoc({name: 'TestDoc1'}, workspaceId);
const max = testDailyApiLimitFeatures.baseMaxApiUnitsPerDocumentPerDay;
for (let i = 1; i <= max + 2; i++) {
let success = true;
try {
// Make some doc request so that it fails or succeeds
await api.getTable(docId, "Table1");
} catch (e) {
success = false;
}
// Usually the first `max` requests should succeed and the rest should fail having exceeded the daily limit.
// If a new minute starts in the middle of the requests, an extra request will be allowed for that minute.
// If a new day starts in the middle of the requests, this test will fail.
if (success) {
assert.isAtMost(i, max + 1);
} else {
assert.isAtLeast(i, max + 1);
}
}
});
it("limits daily API usage and sets the correct keys in redis", async function() {
this.retries(3);
// Make a new document in a free team site, currently the only real product which limits daily API usage.
const freeTeamApi = makeUserApi('freeteam');
const workspaceId = await getWorkspaceId(freeTeamApi, 'FreeTeamWs');
const docId = await freeTeamApi.newDoc({name: 'TestDoc2'}, workspaceId);
// Rather than making 5000 requests, set high counts directly for the current and next daily and hourly keys
const used = 999999;
let m = moment.utc();
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
await redisClient.multi()
.set(currentDay, String(used))
.set(currentHour, String(used))
.set(nextDay, String(used))
.set(nextHour, String(used))
.execAsync();
// Make 9 requests. The first 4 should succeed by fitting into the allocation for the minute.
// (Free team plans get 5000 requests per day, and 5000/24/60 ~= 3.47 which is rounded up to 4)
// The last request should fail. Don't check the middle 4 in case we're on the boundary of a minute.
for (let i = 1; i <= 9; i++) {
const last = i === 9;
m = moment.utc(); // get this before delaying to calculate accurate keys below
const response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`, chimpy);
// Allow time for redis to be updated.
await delay(100);
if (i <= 4) {
assert.equal(response.status, 200);
// Keys of the periods we expect to be incremented.
// For the first request, the server's usage cache is empty and it hasn't seen the redis values.
// So it thinks there hasn't been any usage and increments the current day/hour.
// After that it increments the next day/hour.
// We're only checking this for the first 4 requests
// because once the limit is exceeded the counts aren't incremented.
const first = i === 1;
const day = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[0], m);
const hour = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[1], m);
const minute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
if (!first) {
// The first request takes longer to serve because the document gets loaded,
// so only check the TTL (which gets set before request processing starts) on subsequent requests.
assert.deepEqual(
await redisClient.multi()
.ttl(minute)
.ttl(hour)
.ttl(day)
.execAsync(),
[
2 * 60,
2 * 60 * 60,
2 * 60 * 60 * 24,
],
);
}
assert.deepEqual(
await redisClient.multi()
.get(minute)
.get(hour)
.get(day)
.execAsync(),
[
String(i),
String(used + (first ? 1 : i - 1)),
String(used + (first ? 1 : i - 1)),
],
);
}
if (last) {
assert.equal(response.status, 429);
assert.deepEqual(response.data, {error: `Exceeded daily limit for document ${docId}`});
}
}
});
it("correctly allocates API requests based on the day, hour, and minute", async function() {
const m = moment.utc("1999-12-31T23:59:59Z");
const docId = "myDocId";
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const currentMinute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
assert.equal(currentDay, `doc-myDocId-periodicApiUsage-1999-12-31`);
assert.equal(currentHour, `doc-myDocId-periodicApiUsage-1999-12-31T23`);
assert.equal(currentMinute, `doc-myDocId-periodicApiUsage-1999-12-31T23:59`);
assert.equal(nextDay, `doc-myDocId-periodicApiUsage-2000-01-01`);
assert.equal(nextHour, `doc-myDocId-periodicApiUsage-2000-01-01T00`);
const usage = new LRUCache<string, number>({max: 1024});
function check(expected: string[] | undefined) {
assert.deepEqual(getDocApiUsageKeysToIncr(docId, usage, dailyMax, m), expected);
}
const dailyMax = 5000;
const hourlyMax = 209; // 5000/24 ~= 208.33
const minuteMax = 4; // 5000/24/60 ~= 3.47
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax - 1);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
check([nextDay, currentHour, currentMinute]); // used up daily allocation
usage.set(currentHour, hourlyMax - 1);
check([nextDay, currentHour, currentMinute]);
usage.set(currentHour, hourlyMax);
check([nextDay, nextHour, currentMinute]); // used up hourly allocation
usage.set(currentMinute, minuteMax - 1);
check([nextDay, nextHour, currentMinute]);
usage.set(currentMinute, minuteMax);
check(undefined); // used up minutely allocation
usage.set(currentDay, 0);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
usage.set(currentHour, 0);
check([nextDay, currentHour, currentMinute]);
});
after(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
await redisClient.quitAsync();
});
});
describe("Webhooks", () => {
let serving: Serving; // manages the test webhook server
let requests: WebhookRequests;
let receivedLastEvent: Promise<void>;
// Requests corresponding to adding 200 rows, sent in two batches of 100
const expected200AddEvents = [
_.range(100).map(i => ({
id: 9 + i, manualSort: 9 + i, A3: 200 + i, B3: true,
})),
_.range(100).map(i => ({
id: 109 + i, manualSort: 109 + i, A3: 300 + i, B3: true,
})),
];
// Every event is sent to three webhook URLs which differ by the subscribed eventTypes
// Each request is an array of one or more events.
// Multiple events caused by the same action bundle get batched into a single request.
const expectedRequests: WebhookRequests = {
"add": [
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
// After isReady (B) went to false and then true again
// we treat this as creation even though it's really an update
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 3, A3: 13, B3: true, manualSort: 3},
{id: 5, A3: 15, B3: true, manualSort: 5}],
[{id: 7, A3: 18, B3: true, manualSort: 7}],
...expected200AddEvents,
],
"update": [
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}],
],
"add,update": [
// add
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// update
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// from the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}, // update
{id: 3, A3: 13, B3: true, manualSort: 3}, // add
{id: 5, A3: 15, B3: true, manualSort: 5}], // add
[{id: 7, A3: 18, B3: true, manualSort: 7}], // add
...expected200AddEvents,
]
};
let redisMonitor: any;
let redisCalls: any[];
before(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
requests = {
"add,update": [],
"add": [],
"update": [],
};
let resolveReceivedLastEvent: () => void;
receivedLastEvent = new Promise<void>(r => {
resolveReceivedLastEvent = r;
});
// TODO test retries on failure and slowness in a new test
serving = await serveSomething(app => {
app.use(bodyParser.json());
app.post('/:eventTypes', async ({body, params: {eventTypes}}, res) => {
requests[eventTypes as keyof WebhookRequests].push(body);
res.sendStatus(200);
if (
_.flattenDeep(_.values(requests)).length >=
_.flattenDeep(_.values(expectedRequests)).length
) {
resolveReceivedLastEvent();
}
});
}, webhooksTestPort);
redisCalls = [];
redisMonitor = createClient(process.env.TEST_REDIS_URL);
redisMonitor.monitor();
redisMonitor.on("monitor", (_time: any, args: any, _rawReply: any) => {
redisCalls.push(args);
});
});
after(async function() {
if (!process.env.TEST_REDIS_URL) { this.skip(); }
await serving.shutdown();
await redisMonitor.quitAsync();
});
it("delivers expected payloads from combinations of changes, with retrying and batching", async function() {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// For some reason B is turned into Numeric even when given bools
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
// Make a webhook for every combination of event types
const subscribeResponses = [];
const webhookIds: Record<string, string> = {};
for (const eventTypes of [
["add"],
["update"],
["add", "update"],
]) {
const {data, status} = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table1/_subscribe`,
{eventTypes, url: `${serving.url}/${eventTypes}`, isReadyColumn: "B"}, chimpy
);
assert.equal(status, 200);
subscribeResponses.push(data);
webhookIds[data.webhookId] = String(eventTypes);
}
// Add and update some rows, trigger some events
// Values of A where B is true and thus the record is ready are [1, 4, 7, 8]
// So those are the values seen in expectedEvents
await doc.addRows("Table1", {
A: [1, 2],
B: [true, false], // 1 is ready, 2 is not ready yet
});
await doc.updateRows("Table1", {id: [2], A: [3]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [4], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [5], B: [false]}); // not ready again
await doc.updateRows("Table1", {id: [2], A: [6]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [7], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [8]}); // still ready!
// The end result here is additions for column A (now A3) with values [13, 15, 18]
// and an update for 101
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['BulkAddRecord', 'Table1', [3, 4, 5, 6], {A: [9, 10, 11, 12], B: [true, true, false, false]}],
['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], {
A: [101, 102, 13, 14, 15, 16],
B: [true, false, true, false, true, false],
}],
['RenameColumn', 'Table1', 'A', 'A3'],
['RenameColumn', 'Table1', 'B', 'B3'],
['RenameTable', 'Table1', 'Table12'],
// FIXME a double rename A->A2->A3 doesn't seem to get summarised correctly
// ['RenameColumn', 'Table12', 'A2', 'A3'],
// ['RenameColumn', 'Table12', 'B2', 'B3'],
['RemoveColumn', 'Table12', 'C'],
], chimpy);
// FIXME record changes after a RenameTable in the same bundle
// don't appear in the action summary
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['AddRecord', 'Table12', 7, {A3: 17, B3: false}],
['UpdateRecord', 'Table12', 7, {A3: 18, B3: true}],
['AddRecord', 'Table12', 8, {A3: 19, B3: true}],
['UpdateRecord', 'Table12', 8, {A3: 20, B3: false}],
['AddRecord', 'Table12', 9, {A3: 20, B3: true}],
['RemoveRecord', 'Table12', 9],
], chimpy);
// Add 200 rows. These become the `expected200AddEvents`
await doc.addRows("Table12", {
A3: _.range(200, 400),
B3: arrayRepeat(200, true),
});
await receivedLastEvent;
// Unsubscribe
await Promise.all(subscribeResponses.map(async subscribeResponse => {
const unsubscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table12/_unsubscribe`,
subscribeResponse, chimpy
);
assert.equal(unsubscribeResponse.status, 200);
assert.deepEqual(unsubscribeResponse.data, {success: true});
}));
// Further changes should generate no events because the triggers are gone
await doc.addRows("Table12", {
A3: [88, 99],
B3: [true, false],
});
assert.deepEqual(requests, expectedRequests);
// Check that the events were all pushed to the redis queue
const queueRedisCalls = redisCalls.filter(args => args[1] === "webhook-queue-" + docId);
const redisPushes = _.chain(queueRedisCalls)
.filter(args => args[0] === "rpush") // Array<["rpush", key, ...events: string[]]>
.flatMap(args => args.slice(2)) // events: string[]
.map(JSON.parse) // events: WebhookEvent[]
.groupBy('id') // {[webHookId: string]: WebhookEvent[]}
.mapKeys((_value, key) => webhookIds[key]) // {[eventTypes: 'add'|'update'|'add,update']: WebhookEvent[]}
.mapValues(group => _.map(group, 'payload')) // {[eventTypes: 'add'|'update'|'add,update']: RowRecord[]}
.value();
const expectedPushes = _.mapValues(expectedRequests, value => _.flatten(value));
assert.deepEqual(redisPushes, expectedPushes);
// Check that the events were all removed from the redis queue
const redisTrims = queueRedisCalls.filter(args => args[0] === "ltrim")
.map(([,, start, end]) => {
assert.equal(end, '-1');
start = Number(start);
assert.isTrue(start > 0);
return start;
});
const expectedTrims = Object.values(redisPushes).map(value => value.length);
assert.equal(
_.sum(redisTrims),
_.sum(expectedTrims),
);
});
});
// PLEASE ADD MORE TESTS HERE
}
interface WebhookRequests {
add: object[][];
update: object[][];
"add,update": object[][];
}
function setup(name: string, cb: () => Promise<void>) {
let api: UserAPIImpl;
before(async function() {
suitename = name;
dataDir = path.join(tmpDir, `${suitename}-data`);
await fse.mkdirs(dataDir);
await setupDataDir(dataDir);
await cb();
// create TestDoc as an empty doc into Private workspace
userApi = api = makeUserApi('docs-1');
const wid = await getWorkspaceId(api, 'Private');
docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid);
});
after(async function() {
// remove TestDoc
await api.deleteDoc(docIds.TestDoc);
delete docIds.TestDoc;
// stop all servers
await home.stop();
await docs.stop();
});
}
function makeUserApi(org: string) {
return new UserAPIImpl(`${home.serverUrl}/o/${org}`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
}
async function getWorkspaceId(api: UserAPIImpl, name: string) {
const workspaces = await api.getOrgWorkspaces('current');
return workspaces.find((w) => w.name === name)!.id;
}
async function startServer(serverTypes: string, _homeUrl?: string): Promise<TestServer> {
const server = new TestServer(serverTypes);
await server.start(_homeUrl);
return server;
}
const webhooksTestPort = 34365;
class TestServer {
public testingSocket: string;
public testingHooks: TestingHooksClient;
public serverUrl: string;
public stopped = false;
private _server: ChildProcess;
private _exitPromise: Promise<number|string>;
constructor(private _serverTypes: string) {}
public async start(_homeUrl?: string) {
// put node logs into files with meaningful name that relate to the suite name and server type
const fixedName = this._serverTypes.replace(/,/, '_');
const nodeLogPath = path.join(tmpDir, `${suitename}-${fixedName}-node.log`);
const nodeLogFd = await fse.open(nodeLogPath, 'a');
const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd;
// use a path for socket that relates to suite name and server types
this.testingSocket = path.join(tmpDir, `${suitename}-${fixedName}.socket`);
// env
const env = {
GRIST_DATA_DIR: dataDir,
GRIST_INST_DIR: tmpDir,
GRIST_SERVERS: this._serverTypes,
// with port '0' no need to hard code a port number (we can use testing hooks to find out what
// port server is listening on).
GRIST_PORT: '0',
GRIST_TESTING_SOCKET: this.testingSocket,
GRIST_DISABLE_S3: 'true',
REDIS_URL: process.env.TEST_REDIS_URL,
APP_HOME_URL: _homeUrl,
ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
...process.env
};
const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
this._server = spawn('node', [main, '--testingHooks'], {
env,
stdio: ['inherit', serverLog, serverLog]
});
this._exitPromise = exitPromise(this._server);
// Try to be more helpful when server exits by printing out the tail of its log.
this._exitPromise.then((code) => {
if (this._server.killed) { return; }
log.error("Server died unexpectedly, with code", code);
const output = execFileSync('tail', ['-30', nodeLogPath]);
log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`);
})
.catch(() => undefined);
await this._waitServerReady(30000);
log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
}
public async stop() {
if (this.stopped) { return; }
log.info("Stopping node server: " + this._serverTypes);
this.stopped = true;
this._server.kill();
this.testingHooks.close();
await this._exitPromise;
}
public async isServerReady(): Promise<boolean> {
// Let's wait for the testingSocket to be created, then get the port the server is listening on,
// and then do an api check. This approach allow us to start server with GRIST_PORT set to '0',
// which will listen on first available port, removing the need to hard code a port number.
try {
// wait for testing socket
while (!(await fse.pathExists(this.testingSocket))) {
await delay(200);
}
// create testing hooks and get own port
this.testingHooks = await connectTestingHooks(this.testingSocket);
const port: number = await this.testingHooks.getOwnPort();
this.serverUrl = `http://localhost:${port}`;
// wait for check
return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
} catch (err) {
return false;
}
}
private async _waitServerReady(ms: number) {
// It's important to clear the timeout, because it can prevent node from exiting otherwise,
// which is annoying when running only this test for debugging.
let timeout: any;
const maxDelay = new Promise((resolve) => {
timeout = setTimeout(resolve, 30000);
});
try {
await Promise.race([
this.isServerReady(),
this._exitPromise.then(() => { throw new Error("Server exited while waiting for it"); }),
maxDelay,
]);
} finally {
clearTimeout(timeout);
}
}
}
async function setupDataDir(dir: string) {
// we'll be serving Hello.grist content for various document ids, so let's make copies of it in
// tmpDir
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Timesheets + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Bananas + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Antartic + '.grist'));
await testUtils.copyFixtureDoc(
'ApiDataRecordsTest.grist',
path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
}
Reference in New Issue View Git Blame Copy Permalink