/* eslint-disable @typescript-eslint/no-shadow */
import {ActionSummary} from 'app/common/ActionSummary';
import {BulkColValues, UserAction} from 'app/common/DocActions';
import {arrayRepeat} from 'app/common/gutil';
import {WebhookSummary} from 'app/common/Triggers';
import {DocAPI, DocState, UserAPIImpl} from 'app/common/UserAPI';
import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
import {CellValue, GristObjCode} from 'app/plugin/GristData';
import {
applyQueryParameters,
docApiUsagePeriods,
docPeriodicApiUsageKey,
getDocApiUsageKeysToIncr,
WebhookSubscription
} from 'app/server/lib/DocApi';
import log from 'app/server/lib/log';
import {delayAbort} from 'app/server/lib/serverUtils';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
import {delay} from 'bluebird';
import * as bodyParser from 'body-parser';
import {assert} from 'chai';
import FormData from 'form-data';
import * as fse from 'fs-extra';
import * as _ from 'lodash';
import LRUCache from 'lru-cache';
import * as moment from 'moment';
import {AbortController} from 'node-abort-controller';
import fetch from 'node-fetch';
import {tmpdir} from 'os';
import * as path from 'path';
import {createClient, RedisClient} from 'redis';
import {configForUser} from 'test/gen-server/testUtils';
import {serveSomething, Serving} from 'test/server/customUtil';
import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
import {prepareFilesystemDirectoryForTests} from 'test/server/lib/helpers/PrepareFilesystemDirectoryForTests';
import {signal} from 'test/server/lib/helpers/Signal';
import {TestServer} from 'test/server/lib/helpers/TestServer';
import * as testUtils from 'test/server/testUtils';
import {waitForIt} from 'test/server/wait';
import clone = require('lodash/clone');
import defaultsDeep = require('lodash/defaultsDeep');
import pick = require('lodash/pick');
const chimpy = configForUser('Chimpy');
const kiwi = configForUser('Kiwi');
const charon = configForUser('Charon');
const nobody = configForUser('Anonymous');
const support = configForUser('support');
// some doc ids
const docIds: { [name: string]: string } = {
ApiDataRecordsTest: 'sample_7',
Timesheets: 'sample_13',
Bananas: 'sample_6',
Antartic: 'sample_11'
};
// A testDir of the form grist_test_{USER}_{SERVER_NAME}
const username = process.env.USER || "nobody";
const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi`);
let dataDir: string;
let suitename: string;
let serverUrl: string;
let homeUrl: string;
let hasHomeApi: boolean;
let home: TestServer;
let docs: TestServer;
let userApi: UserAPIImpl;
describe('DocApi', function () {
this.timeout(30000);
testUtils.setTmpLogLevel('error');
const oldEnv = clone(process.env);
before(async function () {
// Clear redis test database if redis is in use.
if (process.env.TEST_REDIS_URL) {
const cli = createClient(process.env.TEST_REDIS_URL);
await cli.flushdbAsync();
await cli.quitAsync();
}
// Create the tmp dir removing any previous one
await prepareFilesystemDirectoryForTests(tmpDir);
// Let's create a sqlite db that we can share with servers that run in other processes, hence
// not an in-memory db. Running seed.ts directly might not take in account the most recent value
// for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
// configuration (in-memory for instance). Spawning a process is one way to make sure that the
// latest value prevail.
await prepareDatabase(tmpDir);
});
after(() => {
Object.assign(process.env, oldEnv);
});
/**
* Doc api tests are run against three different setup:
* - a merged server: a single server serving both as a home and doc worker
* - two separated servers: requests are sent to a home server which then forward them to a doc worker
* - a doc worker: request are sent directly to the doc worker (note that even though it is not
* used for testing we starts anyway a home server, needed for setting up the test cases)
*
* Future tests must be added within the testDocApi() function.
*/
describe("should work with a merged server", async () => {
setup('merged', async () => {
const additionalEnvConfiguration = {
ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
GRIST_DATA_DIR: dataDir
};
home = docs = await TestServer.startServer('home,docs', tmpDir, suitename, additionalEnvConfiguration);
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
// the way these tests are written, non-merged server requires redis.
if (process.env.TEST_REDIS_URL) {
describe("should work with a home server and a docworker", async () => {
setup('separated', async () => {
const additionalEnvConfiguration = {
ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
GRIST_DATA_DIR: dataDir
};
home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
docs = await TestServer.startServer('docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl);
homeUrl = serverUrl = home.serverUrl;
hasHomeApi = true;
});
testDocApi();
});
describe("should work directly with a docworker", async () => {
setup('docs', async () => {
const additionalEnvConfiguration = {
ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
GRIST_DATA_DIR: dataDir
};
home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
docs = await TestServer.startServer('docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl);
homeUrl = home.serverUrl;
serverUrl = docs.serverUrl;
hasHomeApi = false;
});
testDocApi();
});
}
describe("QueryParameters", async () => {
function makeExample() {
return {
id: [1, 2, 3, 7, 8, 9],
color: ['red', 'yellow', 'white', 'blue', 'black', 'purple'],
spin: ['up', 'up', 'down', 'down', 'up', 'up'],
};
}
it("supports ascending sort", async function () {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['color']}, null), {
id: [8, 7, 9, 1, 3, 2],
color: ['black', 'blue', 'purple', 'red', 'white', 'yellow'],
spin: ['up', 'down', 'up', 'up', 'down', 'up']
});
});
it("supports descending sort", async function () {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-id']}, null), {
id: [9, 8, 7, 3, 2, 1],
color: ['purple', 'black', 'blue', 'white', 'yellow', 'red'],
spin: ['up', 'up', 'down', 'down', 'up', 'up'],
});
});
it("supports multi-key sort", async function () {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-spin', 'color']}, null), {
id: [8, 9, 1, 2, 7, 3],
color: ['black', 'purple', 'red', 'yellow', 'blue', 'white'],
spin: ['up', 'up', 'up', 'up', 'down', 'down'],
});
});
it("does not freak out sorting mixed data", async function () {
const example = {
id: [1, 2, 3, 4, 5, 6, 7, 8, 9],
mixed: ['red', 'green', 'white', 2.5, 1, null, ['zing', 3] as any, 5, 'blue']
};
assert.deepEqual(applyQueryParameters(example, {sort: ['mixed']}, null), {
mixed: [1, 2.5, 5, null, ['zing', 3] as any, 'blue', 'green', 'red', 'white'],
id: [5, 4, 8, 6, 7, 9, 2, 1, 3],
});
});
it("supports limit", async function () {
assert.deepEqual(applyQueryParameters(makeExample(), {limit: 1}),
{id: [1], color: ['red'], spin: ['up']});
});
it("supports sort and limit", async function () {
assert.deepEqual(applyQueryParameters(makeExample(), {sort: ['-color'], limit: 2}, null),
{id: [2, 3], color: ['yellow', 'white'], spin: ['up', 'down']});
});
});
});
// Contains the tests. This is where you want to add more test.
function testDocApi() {
it("creator should be owner of a created ws", async () => {
const kiwiEmail = 'kiwi@getgrist.com';
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
// Make sure kiwi isn't allowed here.
await userApi.updateOrgPermissions(ORG_NAME, {users: {[kiwiEmail]: null}});
const kiwiApi = makeUserApi(ORG_NAME, 'kiwi');
await assert.isRejected(kiwiApi.getWorkspaceAccess(ws1), /Forbidden/);
// Add kiwi as an editor for the org.
await assert.isRejected(kiwiApi.getOrgAccess(ORG_NAME), /Forbidden/);
await userApi.updateOrgPermissions(ORG_NAME, {users: {[kiwiEmail]: 'editors'}});
// Make a workspace as Kiwi, he should be owner of it.
const kiwiWs = await kiwiApi.newWorkspace({name: 'kiwiWs'}, ORG_NAME);
const kiwiWsAccess = await kiwiApi.getWorkspaceAccess(kiwiWs);
assert.equal(kiwiWsAccess.users.find(u => u.email === kiwiEmail)?.access, 'owners');
// Delete workspace.
await kiwiApi.deleteWorkspace(kiwiWs);
// Remove kiwi from the org.
await userApi.updateOrgPermissions(ORG_NAME, {users: {[kiwiEmail]: null}});
});
it("creator should be owner of a created doc", async () => {
const kiwiEmail = 'kiwi@getgrist.com';
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
await userApi.updateOrgPermissions(ORG_NAME, {users: {[kiwiEmail]: null}});
// Make sure kiwi isn't allowed here.
const kiwiApi = makeUserApi(ORG_NAME, 'kiwi');
await assert.isRejected(kiwiApi.getWorkspaceAccess(ws1), /Forbidden/);
// Add kiwi as an editor of this workspace.
await userApi.updateWorkspacePermissions(ws1, {users: {[kiwiEmail]: 'editors'}});
await assert.isFulfilled(kiwiApi.getWorkspaceAccess(ws1));
// Create a document as kiwi.
const kiwiDoc = await kiwiApi.newDoc({name: 'kiwiDoc'}, ws1);
// Make sure kiwi is an owner of the document.
const kiwiDocAccess = await kiwiApi.getDocAccess(kiwiDoc);
assert.equal(kiwiDocAccess.users.find(u => u.email === kiwiEmail)?.access, 'owners');
await kiwiApi.deleteDoc(kiwiDoc);
// Remove kiwi from the workspace.
await userApi.updateWorkspacePermissions(ws1, {users: {[kiwiEmail]: null}});
await assert.isRejected(kiwiApi.getWorkspaceAccess(ws1), /Forbidden/);
});
it("should allow only owners to remove a document", async () => {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdeleteme1'}, ws1);
const kiwiApi = makeUserApi(ORG_NAME, 'kiwi');
// Kiwi is editor of the document, so he can't delete it.
await userApi.updateDocPermissions(doc1, {users: {'kiwi@getgrist.com': 'editors'}});
await assert.isRejected(kiwiApi.softDeleteDoc(doc1), /Forbidden/);
await assert.isRejected(kiwiApi.deleteDoc(doc1), /Forbidden/);
// Kiwi is owner of the document - now he can delete it.
await userApi.updateDocPermissions(doc1, {users: {'kiwi@getgrist.com': 'owners'}});
await assert.isFulfilled(kiwiApi.softDeleteDoc(doc1));
await assert.isFulfilled(kiwiApi.deleteDoc(doc1));
});
it("should allow only owners to rename a document", async () => {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testrenameme1'}, ws1);
const kiwiApi = makeUserApi(ORG_NAME, 'kiwi');
// Kiwi is editor of the document, so he can't rename it.
await userApi.updateDocPermissions(doc1, {users: {'kiwi@getgrist.com': 'editors'}});
await assert.isRejected(kiwiApi.renameDoc(doc1, "testrenameme2"), /Forbidden/);
// Kiwi is owner of the document - now he can rename it.
await userApi.updateDocPermissions(doc1, {users: {'kiwi@getgrist.com': 'owners'}});
await assert.isFulfilled(kiwiApi.renameDoc(doc1, "testrenameme2"));
await userApi.deleteDoc(doc1);
});
it("guesses types of new columns", async () => {
const userActions = [
['AddTable', 'GuessTypes', []],
// Make 5 blank columns of type Any
['AddColumn', 'GuessTypes', 'Date', {}],
['AddColumn', 'GuessTypes', 'DateTime', {}],
['AddColumn', 'GuessTypes', 'Bool', {}],
['AddColumn', 'GuessTypes', 'Numeric', {}],
['AddColumn', 'GuessTypes', 'Text', {}],
// Add string values from which the initial type will be guessed
['AddRecord', 'GuessTypes', null, {
Date: "1970-01-02",
DateTime: "1970-01-02 12:00",
Bool: "true",
Numeric: "1.2",
Text: "hello",
}],
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
// Check that the strings were parsed to typed values
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/records`, chimpy)).data,
{
records: [
{
id: 1,
fields: {
Date: 24 * 60 * 60,
DateTime: 36 * 60 * 60,
Bool: true,
Numeric: 1.2,
Text: "hello",
},
},
],
},
);
// Check the column types
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/GuessTypes/columns`, chimpy))
.data.columns.map((col: any) => col.fields.type),
["Date", "DateTime:UTC", "Bool", "Numeric", "Text"],
);
});
for (const mode of ['logged in', 'anonymous']) {
for (const content of ['with content', 'without content']) {
it(`POST /api/docs ${content} creates an unsaved doc when ${mode}`, async function () {
const user = (mode === 'logged in') ? chimpy : nobody;
const formData = new FormData();
formData.append('upload', 'A,B\n1,2\n3,4\n', 'table1.csv');
const config = defaultsDeep({headers: formData.getHeaders()}, user);
let resp = await axios.post(`${serverUrl}/api/docs`,
...(content === 'with content' ? [formData, config] : [null, user]));
assert.equal(resp.status, 200);
const urlId = resp.data;
if (mode === 'logged in') {
assert.match(urlId, /^new~[^~]*~[0-9]+$/);
} else {
assert.match(urlId, /^new~[^~]*$/);
}
// Access information about that document should be sane for current user
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, user);
assert.equal(resp.status, 200);
assert.equal(resp.data.name, 'Untitled');
assert.equal(resp.data.workspace.name, 'Examples & Templates');
assert.equal(resp.data.access, 'owners');
if (mode === 'anonymous') {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, chimpy);
assert.equal(resp.data.access, 'owners');
} else {
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, charon);
assert.equal(resp.status, 403);
resp = await axios.get(`${homeUrl}/api/docs/${urlId}`, nobody);
assert.equal(resp.status, 403);
}
// content was successfully stored
resp = await axios.get(`${serverUrl}/api/docs/${urlId}/tables/Table1/data`, user);
if (content === 'with content') {
assert.deepEqual(resp.data, {id: [1, 2], manualSort: [1, 2], A: [1, 3], B: [2, 4]});
} else {
assert.deepEqual(resp.data, {id: [], manualSort: [], A: [], B: [], C: []});
}
});
}
}
it("GET /docs/{did}/tables/{tid}/data retrieves data in column format", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
});
it("GET /docs/{did}/tables/{tid}/records retrieves data in records format", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'hello',
B: '',
C: '',
D: null,
E: 'HELLO',
},
},
{
id: 2,
fields: {
A: '',
B: 'world',
C: '',
D: null,
E: '',
},
},
{
id: 3,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
{
id: 4,
fields: {
A: '',
B: '',
C: '',
D: null,
E: '',
},
},
]
});
});
it("GET /docs/{did}/tables/{tid}/records handles errors and hidden columns", async function () {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"records": [
{
"id": 1,
"fields": {
"A": null,
"B": "Hi",
"C": 1,
},
"errors": {
"A": "ZeroDivisionError"
}
}
]
}
);
// /data format for comparison: includes manualSort, gristHelper_Display, and ["E", "ZeroDivisionError"]
resp = await axios.get(`${serverUrl}/api/docs/${docIds.ApiDataRecordsTest}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"id": [
1
],
"manualSort": [
1
],
"A": [
[
"E",
"ZeroDivisionError"
]
],
"B": [
"Hi"
],
"C": [
1
],
"gristHelper_Display": [
"Hi"
]
}
);
});
it("GET /docs/{did}/tables/{tid}/columns retrieves columns", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/columns`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
columns: [
{
id: 'A',
fields: {
colRef: 2,
parentId: 1,
parentPos: 1,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'A',
description: '',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'B',
fields: {
colRef: 3,
parentId: 1,
parentPos: 2,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'B',
description: '',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'C',
fields: {
colRef: 4,
parentId: 1,
parentPos: 3,
type: 'Text',
widgetOptions: '',
isFormula: false,
formula: '',
label: 'C',
description: '',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'D',
fields: {
colRef: 5,
parentId: 1,
parentPos: 3,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '',
label: 'D',
description: '',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
},
{
id: 'E',
fields: {
colRef: 6,
parentId: 1,
parentPos: 4,
type: 'Any',
widgetOptions: '',
isFormula: true,
formula: '$A.upper()',
label: 'E',
description: '',
untieColIdFromLabel: false,
summarySourceCol: 0,
displayCol: 0,
visibleCol: 0,
rules: null,
recalcWhen: 0,
recalcDeps: null
}
}
]
}
);
});
it("GET/POST/PATCH /docs/{did}/tables and /columns", async function () {
// POST /tables: Create new tables
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables`, {
tables: [
{columns: [{}]}, // The minimal allowed request
{id: "", columns: [{id: ""}]},
{id: "NewTable1", columns: [{id: "NewCol1", fields: {}}]},
{
id: "NewTable2",
columns: [
{id: "NewCol2", fields: {label: "Label2"}},
{id: "NewCol3", fields: {label: "Label3"}},
{id: "NewCol3", fields: {label: "Label3"}}, // Duplicate column id
]
},
{
id: "NewTable2", // Create a table with duplicate tableId
columns: [
{id: "NewCol2", fields: {label: "Label2"}},
{id: "NewCol3", fields: {label: "Label3"}},
]
},
]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
tables: [
{id: "Table2"},
{id: "Table3"},
{id: "NewTable1"},
{id: "NewTable2"},
{id: "NewTable2_2"}, // duplicated tableId ends with _2
]
});
// POST /columns: Create new columns
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/NewTable2/columns`, {
columns: [
{},
{id: ""},
{id: "NewCol4", fields: {}},
{id: "NewCol4", fields: {}}, // Create a column with duplicate colId
{id: "NewCol5", fields: {label: "Label5"}},
],
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
columns: [
{id: "A"},
{id: "B"},
{id: "NewCol4"},
{id: "NewCol4_2"}, // duplicated colId ends with _2
{id: "NewCol5"},
]
});
// POST /columns to invalid table ID
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/NoSuchTable/columns`,
{columns: [{}]}, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table not found "NoSuchTable"'});
// PATCH /tables: Modify a table. This is pretty much only good for renaming tables.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables`, {
tables: [
{id: "Table3", fields: {tableId: "Table3_Renamed"}},
]
}, chimpy);
assert.equal(resp.status, 200);
// Repeat the same operation to check that it gives 404 if the table doesn't exist.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables`, {
tables: [
{id: "Table3", fields: {tableId: "Table3_Renamed"}},
]
}, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table not found "Table3"'});
// PATCH /columns: Modify a column.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table2/columns`, {
columns: [
{id: "A", fields: {colId: "A_Renamed"}},
]
}, chimpy);
assert.equal(resp.status, 200);
// Repeat the same operation to check that it gives 404 if the column doesn't exist.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table2/columns`, {
columns: [
{id: "A", fields: {colId: "A_Renamed"}},
]
}, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Column not found "A"'});
// Repeat the same operation to check that it gives 404 if the table doesn't exist.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table222/columns`, {
columns: [
{id: "A", fields: {colId: "A_Renamed"}},
]
}, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table not found "Table222"'});
// Rename NewTable2.A -> B to test the name conflict resolution.
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/NewTable2/columns`, {
columns: [
{id: "A", fields: {colId: "B"}},
]
}, chimpy);
assert.equal(resp.status, 200);
// Hide NewTable2.NewCol5 and NewTable2_2 with ACL
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/apply`, [
['AddRecord', '_grist_ACLResources', -1, {tableId: 'NewTable2', colIds: 'NewCol5'}],
['AddRecord', '_grist_ACLResources', -2, {tableId: 'NewTable2_2', colIds: '*'}],
['AddRecord', '_grist_ACLRules', null, {
resource: -1, aclFormula: '', permissionsText: '-R',
}],
['AddRecord', '_grist_ACLRules', null, {
// Don't use permissionsText: 'none' here because we need S permission to delete the table at the end.
resource: -2, aclFormula: '', permissionsText: '-R',
}],
], chimpy);
assert.equal(resp.status, 200);
// GET /tables: Check that the tables were created and renamed.
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
"tables": [
{
"id": "Table1",
"fields": {
"rawViewSectionRef": 2,
"primaryViewId": 1,
"onDemand": false,
"summarySourceTable": 0,
"tableRef": 1
}
},
// New tables start here
{
"id": "Table2",
"fields": {
"rawViewSectionRef": 4,
"primaryViewId": 2,
"onDemand": false,
"summarySourceTable": 0,
"tableRef": 2
}
},
{
"id": "Table3_Renamed",
"fields": {
"rawViewSectionRef": 6,
"primaryViewId": 3,
"onDemand": false,
"summarySourceTable": 0,
"tableRef": 3
}
},
{
"id": "NewTable1",
"fields": {
"rawViewSectionRef": 8,
"primaryViewId": 4,
"onDemand": false,
"summarySourceTable": 0,
"tableRef": 4
}
},
{
"id": "NewTable2",
"fields": {
"rawViewSectionRef": 10,
"primaryViewId": 5,
"onDemand": false,
"summarySourceTable": 0,
"tableRef": 5
}
},
// NewTable2_2 is hidden by ACL
]
}
);
// Check the created columns.
// TODO these columns should probably be included in the GET /tables response.
async function checkColumns(tableId: string, expected: { colId: string, label: string }[]) {
const colsResp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/${tableId}/columns`, chimpy);
assert.equal(colsResp.status, 200);
const actual = colsResp.data.columns.map((c: any) => ({
colId: c.id,
label: c.fields.label,
}));
assert.deepEqual(actual, expected);
}
await checkColumns("Table2", [
{colId: "A_Renamed", label: 'A'},
]);
await checkColumns("Table3_Renamed", [
{colId: "A", label: 'A'},
]);
await checkColumns("NewTable1", [
{colId: "NewCol1", label: 'NewCol1'},
]);
await checkColumns("NewTable2", [
{colId: "NewCol2", label: 'Label2'},
{colId: "NewCol3", label: 'Label3'},
{colId: "NewCol3_2", label: 'Label3'},
{colId: "B2", label: 'A'}, // Result of renaming A -> B
{colId: "B", label: 'B'},
{colId: "NewCol4", label: 'NewCol4'},
{colId: "NewCol4_2", label: 'NewCol4_2'},
// NewCol5 is hidden by ACL
]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/NewTable2_2/columns`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table not found "NewTable2_2"'}); // hidden by ACL
// Clean up the created tables for other tests
// TODO add a DELETE endpoint for /tables and /columns. Probably best to do alongside DELETE /records.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/_grist_Tables/data/delete`,
[2, 3, 4, 5, 6], chimpy);
assert.equal(resp.status, 200);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent doc", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/data returns 404 for non-existent table", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent doc", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/typotypotypo/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /document not found/i);
});
it("GET /docs/{did}/tables/{tid}/columns returns 404 for non-existent table", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Typo1/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /table not found/i);
});
it("GET /docs/{did}/tables/{tid}/data supports filters", async function () {
function makeQuery(filters: { [colId: string]: any[] }) {
const query = "filter=" + encodeURIComponent(JSON.stringify(filters));
return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy);
}
function checkResults(resp: AxiosResponse, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery({B: ['world']}), {
id: [2], A: [''], B: ['world'], C: [''], D: [null], E: [''], manualSort: [2],
});
// Can query by id
checkResults(await makeQuery({id: [1]}), {
id: [1], A: ['hello'], B: [''], C: [''], D: [null], E: ['HELLO'], manualSort: [1],
});
checkResults(await makeQuery({B: [''], A: ['']}), {
id: [3, 4], A: ['', ''], B: ['', ''], C: ['', ''], D: [null, null], E: ['', ''], manualSort: [3, 4],
});
// Empty filter is equivalent to no filter and should return full data.
checkResults(await makeQuery({}), {
id: [1, 2, 3, 4],
A: ['hello', '', '', ''],
B: ['', 'world', '', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['HELLO', '', '', ''],
manualSort: [1, 2, 3, 4]
});
// An impossible filter should succeed but return an empty set of rows.
checkResults(await makeQuery({B: ['world'], C: ['Neptune']}), {
id: [], A: [], B: [], C: [], D: [], E: [], manualSort: [],
});
// An invalid filter should return an error
{
const resp = await makeQuery({BadCol: ['']});
assert.equal(resp.status, 400);
assert.match(resp.data.error, /BadCol/);
}
{
const resp = await makeQuery({B: 'world'} as any);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /filter values must be arrays/);
}
});
for (const mode of ['url', 'header']) {
it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function () {
function makeQuery(sort: string[] | null, limit: number | null) {
const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
const config = configForUser('chimpy');
if (mode === 'url') {
if (sort) {
url.searchParams.append('sort', sort.join(','));
}
if (limit) {
url.searchParams.append('limit', String(limit));
}
} else {
if (sort) {
config.headers['x-sort'] = sort.join(',');
}
if (limit) {
config.headers['x-limit'] = String(limit);
}
}
return axios.get(url.href, config);
}
function checkResults(resp: AxiosResponse, expectedData: any) {
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, expectedData);
}
checkResults(await makeQuery(['-id'], null), {
id: [4, 3, 2, 1],
A: ['', '', '', 'hello'],
B: ['', '', 'world', ''],
C: ['', '', '', ''],
D: [null, null, null, null],
E: ['', '', '', 'HELLO'],
manualSort: [4, 3, 2, 1]
});
checkResults(await makeQuery(['-id'], 2), {
id: [4, 3],
A: ['', ''],
B: ['', ''],
C: ['', ''],
D: [null, null],
E: ['', ''],
manualSort: [4, 3]
});
});
}
it("GET /docs/{did}/tables/{tid}/data respects document permissions", async function () {
// as not part of any group kiwi cannot fetch Timesheets
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, kiwi);
assert.equal(resp.status, 403);
});
it("GET /docs/{did}/tables/{tid}/data returns matches /not found/ for bad table id", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/apply applies user actions", async function () {
const userActions = [
['AddTable', 'Foo', [{id: 'A'}, {id: 'B'}]],
['BulkAddRecord', 'Foo', [1, 2], {A: ["Santa", "Bob"], B: [1, 11]}]
];
const resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(
(await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data,
{id: [1, 2], A: ['Santa', 'Bob'], B: ['1', '11'], manualSort: [1, 2]});
});
it("POST /docs/{did}/apply respects document permissions", async function () {
const userActions = [
['AddTable', 'FooBar', [{id: 'A'}]]
];
let resp: AxiosResponse;
// as a guest chimpy cannot edit Bananas
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Bananas}/apply`, userActions, chimpy);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Bananas}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
// as not in any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/apply`, userActions, kiwi);
assert.equal(resp.status, 403);
// check that changes did not apply
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/FooBar/data`, chimpy);
assert.equal(resp.status, 404);
assert.match(resp.data.error, /not found/);
});
it("POST /docs/{did}/tables/{tid}/data adds records", async function () {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3, 4]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("POST /docs/{did}/tables/{tid}/records adds records", async function () {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{fields: {A: 'John', B: 55}},
{fields: {A: 'Jane', B: 0}},
]
}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {
records: [
{id: 5},
{id: 6},
]
});
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data,
{
records:
[
{
id: 1,
fields: {
A: 'Santa',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
{
id: 5,
fields: {
A: 'John',
B: '55',
},
},
{
id: 6,
fields: {
A: 'Jane',
B: '0',
},
},
]
});
});
it("POST /docs/{did}/tables/{tid}/data/delete deletes records", async function () {
let resp = await axios.post(
`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data/delete`,
[3, 4, 5, 6],
chimpy,
);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, null);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2],
A: ['Santa', 'Bob'],
B: ["1", "11"],
manualSort: [1, 2]
});
// restore rows
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
A: ['Alice', 'Felix'],
B: [2, 22]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
function checkError(status: number, test: RegExp | object, resp: AxiosResponse, message?: string) {
assert.equal(resp.status, status);
if (test instanceof RegExp) {
assert.match(resp.data.error, test, message);
} else {
try {
assert.deepEqual(resp.data, test, message);
} catch (err) {
console.log(JSON.stringify(resp.data));
console.log(JSON.stringify(test));
throw err;
}
}
}
it("parses strings in user actions", async () => {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const recordsUrl = `${docUrl}/tables/Table1/records`;
// Make the column numeric, delete the other columns we don't care about
await axios.post(`${docUrl}/apply`, [
['ModifyColumn', 'Table1', 'A', {type: 'Numeric'}],
['RemoveColumn', 'Table1', 'B'],
['RemoveColumn', 'Table1', 'C'],
], chimpy);
// Add/update some records without and with string parsing
// Specifically test:
// 1. /apply, with an AddRecord
// 2. POST /records (BulkAddRecord)
// 3. PATCH /records (BulkUpdateRecord)
// Send strings that look like currency which need string parsing to become numbers
for (const queryParams of ['?noparse=1', '']) {
await axios.post(`${docUrl}/apply${queryParams}`, [
['AddRecord', 'Table1', null, {'A': '$1'}],
], chimpy);
const response = await axios.post(`${recordsUrl}${queryParams}`,
{
records: [
{fields: {'A': '$2'}},
{fields: {'A': '$3'}},
]
},
chimpy);
// Update $3 -> $4
const rowId = response.data.records[1].id;
await axios.patch(`${recordsUrl}${queryParams}`,
{
records: [
{id: rowId, fields: {'A': '$4'}}
]
},
chimpy);
}
// Check the results
const resp = await axios.get(recordsUrl, chimpy);
assert.deepEqual(resp.data, {
records:
[
// Without string parsing
{id: 1, fields: {A: '$1'}},
{id: 2, fields: {A: '$2'}},
{id: 3, fields: {A: '$4'}},
// With string parsing
{id: 4, fields: {A: 1}},
{id: 5, fields: {A: 2}},
{id: 6, fields: {A: 4}},
]
}
);
});
describe("PUT /docs/{did}/tables/{tid}/records", async function () {
it("should add or update records", async function () {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
async function check(records: AddOrUpdateRecord[], expectedTableData: BulkColValues, params: any = {}) {
const resp = await axios.put(url, {records}, {...chimpy, params});
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
delete table.C;
assert.deepStrictEqual(table, expectedTableData);
}
// Add 3 new records, since the table is empty so nothing matches `requires`
await check(
[
{
require: {A: 1},
},
{
// Since no record with A=2 is found, create a new record,
// but `fields` overrides `require` for the value when creating,
// so the new record has A=3
require: {A: 2},
fields: {A: 3},
},
{
require: {A: 4},
fields: {B: 5},
},
],
{id: [1, 2, 3], A: [1, 3, 4], B: [0, 0, 5]}
);
// Update all three records since they all match the `require` values here
await check(
[
{
// Does nothing
require: {A: 1},
},
{
// Changes A from 3 to 33
require: {A: 3},
fields: {A: 33},
},
{
// Changes B from 5 to 6 in the third record where A=4
require: {A: 4},
fields: {B: 6},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]}
);
// This would normally add a record, but noadd suppresses that
await check([
{
require: {A: 100},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noadd: "1"},
);
// This would normally update A from 1 to 11, bot noupdate suppresses that
await check([
{
require: {A: 1},
fields: {A: 11},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [0, 0, 6]},
{noupdate: "1"},
);
// There are 2 records with B=0, update them both to B=1
// Use onmany=all to specify that they should both be updated
await check([
{
require: {B: 0},
fields: {B: 1},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [1, 1, 6]},
{onmany: "all"}
);
// In contrast to the above, the default behaviour for no value of onmany
// is to only update the first matching record,
// so only one of the records with B=1 is updated to B=2
await check([
{
require: {B: 1},
fields: {B: 2},
},
],
{id: [1, 2, 3], A: [1, 33, 4], B: [2, 1, 6]},
);
// By default, strings in `require` and `fields` are parsed based on column type,
// so these dollar amounts are treated as currency
// and parsed as A=4 and A=44
await check([
{
require: {A: "$4"},
fields: {A: "$44"},
},
],
{id: [1, 2, 3], A: [1, 33, 44], B: [2, 1, 6]},
);
// Turn off the default string parsing with noparse=1
// Now we need A=44 to actually be a number to match,
// A="$44" wouldn't match and would create a new record.
// Because A="$55" isn't parsed, the raw string is stored in the table.
await check([
{
require: {A: 44},
fields: {A: "$55"},
},
],
{id: [1, 2, 3], A: [1, 33, "$55"], B: [2, 1, 6]},
{noparse: 1}
);
await check([
// First three records already exist and nothing happens
{require: {A: 1}},
{require: {A: 33}},
{require: {A: "$55"}},
// Without string parsing, A="$33" doesn't match A=33 and a new record is created
{require: {A: "$33"}},
],
{id: [1, 2, 3, 4], A: [1, 33, "$55", "$33"], B: [2, 1, 6, 0]},
{noparse: 1}
);
// Checking that updating by `id` works.
await check([
{
require: {id: 3},
fields: {A: "66"},
},
],
{id: [1, 2, 3, 4], A: [1, 33, 66, "$33"], B: [2, 1, 6, 0]},
);
// Test bulk case with a mixture of record shapes
await check([
{
require: {A: 1},
fields: {A: 111},
},
{
require: {A: 33},
fields: {A: 222, B: 444},
},
{
require: {id: 3},
fields: {A: 555, B: 666},
},
],
{id: [1, 2, 3, 4], A: [111, 222, 555, "$33"], B: [2, 444, 666, 0]},
);
// allow_empty_require option with empty `require` updates all records
await check([
{
require: {},
fields: {A: 99, B: 99},
},
],
{id: [1, 2, 3, 4], A: [99, 99, 99, 99], B: [99, 99, 99, 99]},
{allow_empty_require: "1", onmany: "all"},
);
});
it("should 404 for missing tables", async () => {
checkError(404, /Table not found "Bad_Foo_"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/records`,
{records: [{require: {id: 1}}]}, chimpy));
});
it("should 400 for missing columns", async () => {
checkError(400, /Invalid column "no_such_column"/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {no_such_column: 1}}]}, chimpy));
});
it("should 400 for an incorrect onmany parameter", async function () {
checkError(400,
/onmany parameter foo should be one of first,none,all/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {id: 1}}]}, {...chimpy, params: {onmany: "foo"}}));
});
it("should 400 for an empty require without allow_empty_require", async function () {
checkError(400,
/require is empty but allow_empty_require isn't set/,
await axios.put(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`,
{records: [{require: {}}]}, chimpy));
});
it("should validate request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async (payload: any, error: { error: string, details: {userError: string} }) => {
const resp = await axios.put(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await test({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
await test({records: [{fields: {}}]},
{
error: 'Invalid payload',
details: {userError: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require is missing',
}});
await test({records: [{require: {id: "1"}}]},
{
error: 'Invalid payload',
details: {userError: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require.id is not a number',
}});
});
});
describe("POST /docs/{did}/tables/{tid}/records", async function () {
it("POST should have good errors", async () => {
checkError(404, /not found/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Bad_Foo_/data`,
{A: ['Alice', 'Felix'], B: [2, 22]}, chimpy));
checkError(400, /Invalid column "Bad"/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{A: ['Alice'], Bad: ['Monthy']}, chimpy));
// Other errors should also be maximally informative.
checkError(400, /Error manipulating data/,
await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`,
{A: ['Alice'], B: null}, chimpy));
});
it("validates request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async(payload: any, error: {error: string, details: {userError: string}}) => {
const resp = await axios.post(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await test({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
// All column types are allowed, except Arrays (or objects) without correct code.
const testField = async (A: any) => {
await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a NewRecord; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'}});
};
// test no code at all
await testField([]);
// test invalid code
await testField(['ZZ']);
});
it("allows to create a blank record", async function () {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
// Create two blank records
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const resp = await axios.post(url, {records: [{}, {fields: {}}]}, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, {records: [{id: 1}, {id: 2}]});
});
it("allows to create partial records", async function () {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'BlankTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create partial records
const resp = await axios.post(url, {records: [{fields: {A: 1}}, {fields: {B: 2}}, {}]}, chimpy);
assert.equal(resp.status, 200);
const table = await userApi.getTable(docId, "Table1");
delete table.manualSort;
assert.deepStrictEqual(
table,
{id: [1, 2, 3], A: [1, null, null], B: [null, 2, null], C: [null, null, null]});
});
it("allows CellValue as a field", async function () {
// create sample document
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'PostTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const testField = async (A?: CellValue, message?: string) => {
const resp = await axios.post(url, {records: [{fields: {A}}]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
// test allowed types for a field
await testField(1); // ints
await testField(1.2); // floats
await testField("string"); // strings
await testField(true); // true and false
await testField(false);
await testField(null); // null
// encoded values (though not all make sense)
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
it("POST /docs/{did}/tables/{tid}/data respects document permissions", async function () {
let resp: AxiosResponse;
const data = {
A: ['Alice', 'Felix'],
B: [2, 22]
};
// as a viewer charon cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot edit TestDoc
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that TestDoc did not change
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
describe("PATCH /docs/{did}/tables/{tid}/records", function () {
it("updates records", async function () {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, {
records: [
{
id: 1,
fields: {
A: 'Father Christmas',
},
},
],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
records:
[
{
id: 1,
fields: {
A: 'Father Christmas',
B: '1',
},
},
{
id: 2,
fields: {
A: 'Bob',
B: '11',
},
},
{
id: 3,
fields: {
A: 'Alice',
B: '2',
},
},
{
id: 4,
fields: {
A: 'Felix',
B: '22',
},
},
]
});
});
it("validates request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
async function failsWithError(payload: any, error: { error: string, details?: {userError: string} }){
const resp = await axios.patch(url, payload, chimpy);
checkError(400, error, resp);
}
await failsWithError({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await failsWithError({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
await failsWithError({records: []}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record; body.records[0] is not an object'}});
await failsWithError({records: [{}]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record\n '+
'body.records[0].id is missing\n '+
'body.records[0].fields is missing'}});
await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record\n' +
' body.records[0].id is not a number\n' +
' body.records[0].fields is missing'}});
await failsWithError(
{records: [{id: 1, fields: {A: 1}}, {id: 2, fields: {B: 3}}]},
{error: 'PATCH requires all records to have same fields'});
// Test invalid object codes
const fieldIsNotValid = async (A: any) => {
await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'}});
};
await fieldIsNotValid([]);
await fieldIsNotValid(['ZZ']);
});
it("allows CellValue as a field", async function () {
// create sample document for testing
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const docId = await userApi.newDoc({name: 'PatchTest'}, wid);
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
// create record for patching
const id = (await axios.post(url, {records: [{}]}, chimpy)).data.records[0].id;
const testField = async (A?: CellValue, message?: string) => {
const resp = await axios.patch(url, {records: [{id, fields: {A}}]}, chimpy);
assert.equal(resp.status, 200, message ?? `Error for code ${A}`);
};
await testField(1);
await testField(1.2);
await testField("string");
await testField(true);
await testField(false);
await testField(null);
for (const code of [
GristObjCode.List,
GristObjCode.Dict,
GristObjCode.DateTime,
GristObjCode.Date,
GristObjCode.Skip,
GristObjCode.Censored,
GristObjCode.Reference,
GristObjCode.ReferenceList,
GristObjCode.Exception,
GristObjCode.Pending,
GristObjCode.Unmarshallable,
GristObjCode.Versions,
]) {
await testField([code]);
}
});
});
describe("PATCH /docs/{did}/tables/{tid}/data", function () {
it("updates records", async function () {
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Santa Klaus'],
}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy);
// check that rest of the data is left unchanged
assert.deepEqual(resp.data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid row ids", async function () {
// combination of valid and invalid ids fails
let resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 5/);
// only invalid ids also fails
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [10, 5],
A: ['Alice', 'Felix']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid row id 10/);
// check that changes related to id 1 did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
it("throws 400 for invalid column", async function () {
const resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, {
id: [1],
A: ['Alice'],
C: ['Monthy']
}, chimpy);
assert.equal(resp.status, 400);
assert.match(resp.data.error, /Invalid column "C"/);
});
it("respects document permissions", async function () {
let resp: AxiosResponse;
const data = {
id: [1],
A: ['Santa'],
};
// check data
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
// as a viewer charon cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, charon);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No write access'});
// as not part of any group kiwi cannot patch TestDoc
resp = await axios.patch(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, data, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
// check that changes did not apply
assert.deepEqual((await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/data`, chimpy)).data, {
id: [1, 2, 3, 4],
A: ['Santa Klaus', 'Bob', 'Alice', 'Felix'],
B: ["1", "11", "2", "22"],
manualSort: [1, 2, 3, 4]
});
});
});
describe('attachments', function () {
it("POST /docs/{did}/attachments adds attachments", async function () {
let formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2]);
// Another upload gets the next number.
formData = new FormData();
formData.append('upload', 'abcdef', "hello.png");
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [3]);
});
it("GET /docs/{did}/attachments lists attachment metadata", async function () {
// Test that the usual /records query parameters like sort and filter also work
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/attachments?sort=-fileName&limit=2`;
const resp = await axios.get(url, chimpy);
assert.equal(resp.status, 200);
const {records} = resp.data;
for (const record of records) {
assert.match(record.fields.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
delete record.fields.timeUploaded;
}
assert.deepEqual(records, [
{id: 2, fields: {fileName: "world.jpg", fileSize: 6}},
{id: 3, fields: {fileName: "hello.png", fileSize: 6}},
]
);
});
it("GET /docs/{did}/attachments/{id} returns attachment metadata", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2`, chimpy);
assert.equal(resp.status, 200);
assert.include(resp.data, {fileName: "world.jpg", fileSize: 6});
assert.match(resp.data.timeUploaded, /^\d{4}-\d{2}-\d{2}T/);
});
it("GET /docs/{did}/attachments/{id}/download downloads attachment contents", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}/download works after doc shutdown", async function () {
// Check that we can download when ActiveDoc isn't currently open.
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/2/download`,
{...chimpy, responseType: 'arraybuffer'});
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/jpeg');
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.jpg"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.data, Buffer.from('123456'));
});
it("GET /docs/{did}/attachments/{id}... returns 404 when attachment not found", async function () {
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo`, chimpy);
checkError(400, /parameter cannot be understood as an integer: moo/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/22/download`, chimpy);
checkError(404, /Attachment not found: 22/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/moo/download`, chimpy);
checkError(400, /parameter cannot be understood as an integer: moo/, resp);
});
it("POST /docs/{did}/attachments produces reasonable errors", async function () {
// Check that it produces reasonable errors if we try to use it with non-form-data
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, [4, 5, 6], chimpy);
assert.equal(resp.status, 415); // Wrong content-type
// Check for an error if there is no data included.
const formData = new FormData();
resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 400);
// TODO The error here is "stream ended unexpectedly", which isn't really reasonable.
});
it("POST/GET /docs/{did}/attachments respect document permissions", async function () {
const formData = new FormData();
formData.append('upload', 'xyzzz', "wrong.png");
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, kiwi));
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3`, kiwi);
checkError(403, /No view access/, resp);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/3/download`, kiwi);
checkError(403, /No view access/, resp);
});
it("POST /docs/{did}/attachments respects untrusted content-type only if valid", async function () {
const formData = new FormData();
formData.append('upload', 'xyz', {filename: "foo", contentType: "application/pdf"});
formData.append('upload', 'abc', {filename: "hello.png", contentType: "invalid/content-type"});
formData.append('upload', 'def', {filename: "world.doc", contentType: "text/plain\nbad-header: 1\n\nEvil"});
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [4, 5, 6]);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/4/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/pdf'); // A valid content-type is respected
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="foo.pdf"');
assert.deepEqual(resp.data, 'xyz');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/5/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'image/png'); // Did not pay attention to invalid header
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="hello.png"');
assert.deepEqual(resp.data, 'abc');
resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/attachments/6/download`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.headers['content-type'], 'application/msword'); // Another invalid header ignored
assert.deepEqual(resp.headers['content-disposition'], 'attachment; filename="world.doc"');
assert.deepEqual(resp.headers['cache-control'], 'private, max-age=3600');
assert.deepEqual(resp.headers['bad-header'], undefined); // Attempt to hack in more headers didn't work
assert.deepEqual(resp.data, 'def');
});
it("POST /docs/{did}/attachments/updateUsed updates timeDeleted on metadata", async function () {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc2'}, wid);
// Apply the given user actions,
// POST to /attachments/updateUsed
// Check that Table1 and _grist_Attachments contain the expected rows
async function check(
actions: UserAction[],
userData: { id: number, Attached: any }[],
metaData: { id: number, deleted: boolean }[],
) {
const docUrl = `${serverUrl}/api/docs/${docId}`;
let resp = await axios.post(`${docUrl}/apply`, actions, chimpy);
assert.equal(resp.status, 200);
resp = await axios.post(`${docUrl}/attachments/updateUsed`, null, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${docUrl}/tables/Table1/records`, chimpy);
const actualUserData = resp.data.records.map(
({id, fields: {Attached}}: ApiRecord) =>
({id, Attached})
);
assert.deepEqual(actualUserData, userData);
resp = await axios.get(`${docUrl}/tables/_grist_Attachments/records`, chimpy);
const actualMetaData = resp.data.records.map(
({id, fields: {timeDeleted}}: ApiRecord) =>
({id, deleted: Boolean(timeDeleted)})
);
assert.deepEqual(actualMetaData, metaData);
}
// Set up the document and initial data.
await check(
[
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkAddRecord", "Table1", [1, 2], {Attached: [['L', 1], ['L', 2, 3]]}],
// There's no actual attachments here but that doesn't matter
["BulkAddRecord", "_grist_Attachments", [1, 2, 3], {}],
],
[
{id: 1, Attached: ['L', 1]},
{id: 2, Attached: ['L', 2, 3]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false},
{id: 3, deleted: false},
],
);
// Remove the record containing ['L', 2, 3], so the metadata for 2 and 3 now says deleted
await check(
[["RemoveRecord", "Table1", 2]],
[
{id: 1, Attached: ['L', 1]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true}, // deleted here
],
);
// Add back a reference to attacument 2 to test 'undeletion', plus some junk values
await check(
[["BulkAddRecord", "Table1", [3, 4, 5], {Attached: [null, "foo", ['L', 2, 2, 4, 4, 5]]}]],
[
{id: 1, Attached: ['L', 1]},
{id: 3, Attached: null},
{id: 4, Attached: "foo"},
{id: 5, Attached: ['L', 2, 2, 4, 4, 5]},
],
[
{id: 1, deleted: false},
{id: 2, deleted: false}, // undeleted here
{id: 3, deleted: true},
],
);
// Remove the whole column to test what happens when there's no Attachment columns
await check(
[["RemoveColumn", "Table1", "Attached"]],
[
{id: 1, Attached: undefined},
{id: 3, Attached: undefined},
{id: 4, Attached: undefined},
{id: 5, Attached: undefined},
],
[
{id: 1, deleted: true}, // deleted here
{id: 2, deleted: true}, // deleted here
{id: 3, deleted: true},
],
);
// Test performance with a large number of records and attachments.
// The maximum value of numRecords that doesn't return a 413 error is about 18,000.
// In that case it took about 5.7 seconds to apply the initial user actions (i.e. add the records),
// 0.3 seconds to call updateUsed once, and 0.1 seconds to call it again immediately after.
// That last time roughly measures the time taken to do the SQL query
// without having to apply any user actions after to update timeDeleted.
// 10,000 records is a compromise so that tests aren't too slow.
const numRecords = 10000;
const attachmentsPerRecord = 4;
const totalUsedAttachments = numRecords * attachmentsPerRecord; // 40,000 attachments referenced in user data
const totalAttachments = totalUsedAttachments * 1.1; // 44,000 attachment IDs listed in metadata
const attachedValues = _.chunk(_.range(1, totalUsedAttachments + 1), attachmentsPerRecord)
.map(arr => ['L', ...arr]);
await check(
[
// Reset the state: add back the removed column and delete the previously added data
["AddColumn", "Table1", "Attached", {type: "Attachments"}],
["BulkRemoveRecord", "Table1", [1, 3, 4, 5]],
["BulkRemoveRecord", "_grist_Attachments", [1, 2, 3]],
["BulkAddRecord", "Table1", arrayRepeat(numRecords, null), {Attached: attachedValues}],
["BulkAddRecord", "_grist_Attachments", arrayRepeat(totalAttachments, null), {}],
],
attachedValues.map((Attached, index) => ({id: index + 1, Attached})),
_.range(totalAttachments).map(index => ({id: index + 1, deleted: index >= totalUsedAttachments})),
);
});
it("POST /docs/{did}/attachments/removeUnused removes unused attachments", async function () {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'TestDoc3'}, wid);
const docUrl = `${serverUrl}/api/docs/${docId}`;
const formData = new FormData();
formData.append('upload', 'foobar', "hello.doc");
formData.append('upload', '123456', "world.jpg");
formData.append('upload', 'foobar', "hello2.doc");
let resp = await axios.post(`${docUrl}/attachments`, formData,
defaultsDeep({headers: formData.getHeaders()}, chimpy));
assert.equal(resp.status, 200);
assert.deepEqual(resp.data, [1, 2, 3]);
async function checkAttachmentIds(ids: number[]) {
resp = await axios.get(`${docUrl}/attachments`, chimpy);
assert.equal(resp.status, 200);
assert.deepEqual(resp.data.records.map((r: any) => r.id), ids);
}
resp = await axios.patch(
`${docUrl}/tables/_grist_Attachments/records`,
{
records: [
{id: 1, fields: {timeDeleted: Date.now() / 1000 - 8 * 24 * 60 * 60}}, // 8 days ago, i.e. expired
{id: 2, fields: {timeDeleted: Date.now() / 1000 - 6 * 24 * 60 * 60}}, // 6 days ago, i.e. not expired
]
},
chimpy,
);
assert.equal(resp.status, 200);
await checkAttachmentIds([1, 2, 3]);
// Remove the expired attachment (1) by force-reloading, so it removes it during shutdown.
// It has a duplicate (3) that hasn't expired and thus isn't removed,
// although they share the same fileIdent and row in _gristsys_Files.
// So for now only the metadata is removed.
resp = await axios.post(`${docUrl}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([2, 3]);
resp = await axios.post(`${docUrl}/attachments/verifyFiles`, null, chimpy);
assert.equal(resp.status, 200);
// Remove the not expired attachments (2 and 3).
// We didn't set a timeDeleted for 3, but it gets set automatically by updateUsedAttachmentsIfNeeded.
resp = await axios.post(`${docUrl}/attachments/removeUnused?verifyfiles=1`, null, chimpy);
assert.equal(resp.status, 200);
await checkAttachmentIds([]);
});
});
it("GET /docs/{did}/download serves document", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, chimpy);
assert.equal(resp.status, 200);
assert.match(resp.data, /grist_Tables_column/);
});
it("GET /docs/{did}/download respects permissions", async function () {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download`, kiwi);
assert.equal(resp.status, 403);
assert.notMatch(resp.data, /grist_Tables_column/);
});
// A tiny test that /copy doesn't throw.
it("POST /docs/{did}/copy succeeds", async function () {
const docId = docIds.TestDoc;
const worker1 = await userApi.getWorkerAPI(docId);
await worker1.copyDoc(docId, undefined, 'copy');
});
it("GET /docs/{did}/download/csv serves CSV-encoded document", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
assert.equal(resp.status, 200);
assert.equal(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
const resp2 = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Foo`, chimpy);
assert.equal(resp2.status, 200);
assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
});
it("GET /docs/{did}/download/csv respects permissions", async function () {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1`, kiwi);
assert.equal(resp.status, 403);
assert.notEqual(resp.data, 'A,B,C,D,E\nhello,,,,HELLO\n,world,,,\n,,,,\n,,,,\n');
});
it("GET /docs/{did}/download/csv returns 404 if tableId is invalid", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=MissingTableId`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table MissingTableId not found.'});
});
it("GET /docs/{did}/download/csv returns 404 if viewSectionId is invalid", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv?tableId=Table1&viewSection=9999`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'No record 9999 in table _grist_Views_section'});
});
it("GET /docs/{did}/download/csv returns 400 if tableId is missing", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/csv`, chimpy);
assert.equal(resp.status, 400);
assert.deepEqual(resp.data, {error: 'tableId parameter should be a string: undefined'});
});
it("GET /docs/{did}/download/table-schema serves table-schema-encoded document", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Foo`, chimpy);
assert.equal(resp.status, 200);
const expected = {
format: "csv",
mediatype: "text/csv",
encoding: "utf-8",
dialect: {
delimiter: ",",
doubleQuote: true,
},
name: 'foo',
title: 'Foo',
schema: {
fields: [{
name: 'A',
type: 'string',
format: 'default',
}, {
name: 'B',
type: 'string',
format: 'default',
}]
}
};
assert.deepInclude(resp.data, expected);
const resp2 = await axios.get(resp.data.path, chimpy);
assert.equal(resp2.status, 200);
assert.equal(resp2.data, 'A,B\nSanta,1\nBob,11\nAlice,2\nFelix,22\n');
});
it("GET /docs/{did}/download/table-schema respects permissions", async function () {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=Table1`, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {"error": "No view access"});
});
it("GET /docs/{did}/download/table-schema returns 404 if tableId is invalid", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema?tableId=MissingTableId`,
chimpy,
);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table MissingTableId not found.'});
});
it("GET /docs/{did}/download/table-schema returns 400 if tableId is missing", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/table-schema`, chimpy);
assert.equal(resp.status, 400);
assert.deepEqual(resp.data, {error: 'tableId parameter should be a string: undefined'});
});
it("GET /docs/{did}/download/xlsx serves XLSX-encoded document", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/xlsx?tableId=Table1`, chimpy);
assert.equal(resp.status, 200);
assert.notEqual(resp.data, null);
});
it("GET /docs/{did}/download/xlsx respects permissions", async function () {
// kiwi has no access to TestDoc
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1`, kiwi);
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: 'No view access'});
});
it("GET /docs/{did}/download/xlsx returns 404 if tableId is invalid", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=MissingTableId`,
chimpy
);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'Table MissingTableId not found.'});
});
it("GET /docs/{did}/download/xlsx returns 404 if viewSectionId is invalid", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx?tableId=Table1&viewSection=9999`, chimpy);
assert.equal(resp.status, 404);
assert.deepEqual(resp.data, {error: 'No record 9999 in table _grist_Views_section'});
});
it("GET /docs/{did}/download/xlsx returns 200 if tableId is missing", async function () {
const resp = await axios.get(
`${serverUrl}/api/docs/${docIds.TestDoc}/download/xlsx`, chimpy);
assert.equal(resp.status, 200);
assert.notEqual(resp.data, null);
});
it('POST /workspaces/{wid}/import handles empty filenames', async function () {
if (!process.env.TEST_REDIS_URL) {
this.skip();
}
const worker1 = await userApi.getWorkerAPI('import');
const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, '.grist');
const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
assert.equal(resp.data.title, 'Untitled upload');
assert.equal(typeof resp.data.id, 'string');
assert.notEqual(resp.data.id, '');
});
it("document is protected during upload-and-import sequence", async function () {
if (!process.env.TEST_REDIS_URL) {
this.skip();
}
// Prepare an API for a different user.
const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
headers: {Authorization: 'Bearer api_key_for_kiwi'},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
// upload something for Chimpy and something else for Kiwi.
const worker1 = await userApi.getWorkerAPI('import');
const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
const uploadId1 = await worker1.upload(fakeData1, 'upload.grist');
const worker2 = await kiwiApi.getWorkerAPI('import');
const fakeData2 = await testUtils.readFixtureDoc('Favorite_Films.grist');
const uploadId2 = await worker2.upload(fakeData2, 'upload2.grist');
// Check that kiwi only has access to their own upload.
let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id;
let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Kiwi'));
assert.equal(resp.status, 403);
assert.deepEqual(resp.data, {error: "access denied"});
resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2},
configForUser('Kiwi'));
assert.equal(resp.status, 200);
// Check that chimpy has access to their own upload.
wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
configForUser('Chimpy'));
assert.equal(resp.status, 200);
});
it('limits parallel requests', async function () {
// Launch 30 requests in parallel and see how many are honored and how many
// return 429s. The timing of this test is a bit delicate. We close the doc
// to increase the odds that results won't start coming back before all the
// requests have passed authorization. May need to do something more sophisticated
// if this proves unreliable.
await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
const reqs = [...Array(30).keys()].map(
i => axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy));
const responses = await Promise.all(reqs);
assert.lengthOf(responses.filter(r => r.status === 200), 10);
assert.lengthOf(responses.filter(r => r.status === 429), 20);
});
it('allows forced reloads', async function () {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force a reload.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force a reload through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/force-reload`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('allows assignments', async function () {
let resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 200);
// Check that support cannot force an assignment.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 403);
if (hasHomeApi) {
// Check that support can force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, support);
assert.equal(resp.status, 200);
// Check that regular user cannot force an assignment through housekeeping api.
resp = await axios.post(`${serverUrl}/api/housekeeping/docs/${docIds.Timesheets}/assign`, null, chimpy);
assert.equal(resp.status, 403);
}
});
it('honors urlIds', async function () {
// Make a document with a urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1);
try {
// Make sure an edit made by docId is visible when accessed via docId or urlId
await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
// Make sure an edit made by urlId is visible when accessed via docId or urlId
await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
A: ['Orange'], B: [42]
}, chimpy);
resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[1], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
}
});
it('filters urlIds by org', async function () {
// Make two documents with same urlId
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2', urlId: 'urlid'}, ws2);
try {
// Place a value in "docs" doc
await axios.post(`${serverUrl}/o/docs/api/docs/urlid/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
// Place a value in "nasa" doc
await axios.post(`${serverUrl}/o/nasa/api/docs/urlid/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
// Check the values made it to the right places
let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Apple');
resp = await axios.get(`${serverUrl}/api/docs/${doc2}/tables/Table1/data`, chimpy);
assert.equal(resp.data.A[0], 'Orange');
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it('allows docId access to any document from merged org', async function () {
// Make two documents
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
headers: {Authorization: 'Bearer api_key_for_chimpy'},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
const ws2 = (await nasaApi.getOrgWorkspaces('current'))[0].id;
const doc2 = await nasaApi.newDoc({name: 'testdoc2'}, ws2);
try {
// Should fail to write to a document in "docs" from "nasa" url
let resp = await axios.post(`${serverUrl}/o/nasa/api/docs/${doc1}/tables/Table1/data`, {
A: ['Apple'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
// Should successfully write to a document in "nasa" from "docs" url
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 200);
// Should fail to write to a document in "nasa" from "pr" url
resp = await axios.post(`${serverUrl}/o/pr/api/docs/${doc2}/tables/Table1/data`, {
A: ['Orange'], B: [99]
}, chimpy);
assert.equal(resp.status, 404);
} finally {
await userApi.deleteDoc(doc1);
await nasaApi.deleteDoc(doc2);
}
});
it("GET /docs/{did}/replace replaces one document with another", async function () {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const doc2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc3 = await userApi.newDoc({name: 'testdoc3'}, ws1);
const doc4 = await userApi.newDoc({name: 'testdoc4'}, ws1);
await userApi.updateDocPermissions(doc2, {users: {'kiwi@getgrist.com': 'editors'}});
await userApi.updateDocPermissions(doc3, {users: {'kiwi@getgrist.com': 'viewers'}});
await userApi.updateDocPermissions(doc4, {users: {'kiwi@getgrist.com': 'owners'}});
try {
// Put some material in doc3
let resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc3}/tables/Table1/data`, {
A: ['Orange']
}, chimpy);
assert.equal(resp.status, 200);
// Kiwi cannot replace doc2 with doc3, not an owner
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 403);
assert.match(resp.data.error, /Only owners can replace a document/);
// Kiwi can't replace doc1 with doc3, no access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc1}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 403);
assert.match(resp.data.error, /No view access/);
// Kiwi can't replace doc2 with doc1, no read access to doc1
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc2}/replace`, {
sourceDocId: doc1
}, kiwi);
assert.equal(resp.status, 403);
assert.match(resp.data.error, /access denied/);
// Kiwi cannot replace a doc with material they have only partial read access to.
resp = await axios.post(`${serverUrl}/api/docs/${doc3}/apply`, [
['AddRecord', '_grist_ACLResources', -1, {tableId: 'Table1', colIds: 'A'}],
['AddRecord', '_grist_ACLRules', null, {
resource: -1, aclFormula: 'user.Access not in [OWNER]', permissionsText: '-R',
}]
], chimpy);
assert.equal(resp.status, 200);
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc4}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 403);
assert.match(resp.data.error, /not authorized/);
resp = await axios.post(`${serverUrl}/api/docs/${doc3}/tables/_grist_ACLRules/data/delete`,
[2], chimpy);
assert.equal(resp.status, 200);
resp = await axios.post(`${serverUrl}/o/docs/api/docs/${doc4}/replace`, {
sourceDocId: doc3
}, kiwi);
assert.equal(resp.status, 200);
} finally {
await userApi.deleteDoc(doc1);
await userApi.deleteDoc(doc2);
await userApi.deleteDoc(doc3);
await userApi.deleteDoc(doc4);
}
});
it("GET /docs/{did}/snapshots retrieves a list of snapshots", async function () {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/snapshots`, chimpy);
assert.equal(resp.status, 200);
assert.isAtLeast(resp.data.snapshots.length, 1);
assert.hasAllKeys(resp.data.snapshots[0], ['docId', 'lastModified', 'snapshotId']);
});
it("POST /docs/{did}/states/remove removes old states", async function () {
// Check doc has plenty of states.
let resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
const states: DocState[] = resp.data.states;
assert.isAbove(states.length, 5);
// Remove all but 3.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 3}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 3);
assert.equal(resp.data.states[0].h, states[0].h);
assert.equal(resp.data.states[1].h, states[1].h);
assert.equal(resp.data.states[2].h, states[2].h);
// Remove all but 1.
resp = await axios.post(`${serverUrl}/api/docs/${docIds.Timesheets}/states/remove`, {keep: 1}, chimpy);
assert.equal(resp.status, 200);
resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/states`, chimpy);
assert.equal(resp.status, 200);
assert.lengthOf(resp.data.states, 1);
assert.equal(resp.data.states[0].h, states[0].h);
});
it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function () {
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc1 = userApi.getDocAPI(docId1);
const doc2 = userApi.getDocAPI(docId2);
// Stick some content in column A so it has a defined type
// so diffs are smaller and simpler.
await doc2.addRows('Table1', {A: [0]});
let comp = await doc1.compareDoc(docId2);
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary']);
assert.equal(comp.summary, 'unrelated');
assert.equal(comp.parent, null);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 1);
assert.equal(comp.right.n, 2);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 2);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, {detail: true});
assert.deepEqual(comp.details, {
leftChanges: {tableRenames: [], tableDeltas: {}},
rightChanges: {tableRenames: [], tableDeltas: {}}
});
await doc1.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'left');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 2);
assert.deepEqual(comp.right, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, {detail: true});
assert.deepEqual(comp.details!.rightChanges,
{tableRenames: [], tableDeltas: {}});
const addA1: ActionSummary = {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [],
removeRows: [],
addRows: [2],
columnDeltas: {
A: {[2]: [null, [1]]},
manualSort: {[2]: [null, [2]]},
},
columnRenames: [],
}
}
};
assert.deepEqual(comp.details!.leftChanges, addA1);
await doc2.addRows('Table1', {A: [1]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'both');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 3);
assert.equal(comp.parent!.n, 2);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, {detail: true});
assert.deepEqual(comp.details!.leftChanges, addA1);
assert.deepEqual(comp.details!.rightChanges, addA1);
await doc1.replace({sourceDocId: docId2});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'same');
assert.equal(comp.left.n, 3);
assert.deepEqual(comp.left, comp.right);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, {detail: true});
assert.deepEqual(comp.details, {
leftChanges: {tableRenames: [], tableDeltas: {}},
rightChanges: {tableRenames: [], tableDeltas: {}}
});
await doc2.addRows('Table1', {A: [2]});
comp = await doc1.compareDoc(docId2);
assert.equal(comp.summary, 'right');
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.left, comp.parent);
assert.equal(comp.details, undefined);
comp = await doc1.compareDoc(docId2, {detail: true});
assert.deepEqual(comp.details!.leftChanges,
{tableRenames: [], tableDeltas: {}});
const addA2: ActionSummary = {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [],
removeRows: [],
addRows: [3],
columnDeltas: {
A: {[3]: [null, [2]]},
manualSort: {[3]: [null, [3]]},
},
columnRenames: [],
}
}
};
assert.deepEqual(comp.details!.rightChanges, addA2);
});
it("GET /docs/{did}/compare tracks changes within a doc", async function () {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// Give the document some history.
await doc.addRows('Table1', {A: ['a1'], B: ['b1']});
await doc.addRows('Table1', {A: ['a2'], B: ['b2']});
await doc.updateRows('Table1', {id: [1], A: ['A1']});
// Examine the most recent change, from HEAD~ to HEAD.
let comp = await doc.compareVersion('HEAD~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 3);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, {tableRenames: [], tableDeltas: {}});
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [],
columnDeltas: {
A: {[1]: [['a1'], ['A1']]}
},
columnRenames: [],
}
}
});
// Check we get the same result with actual hashes.
assert.notMatch(comp.left.h, /HEAD/);
assert.notMatch(comp.right.h, /HEAD/);
const comp2 = await doc.compareVersion(comp.left.h, comp.right.h);
assert.deepEqual(comp, comp2);
// Check that comparing the HEAD with itself shows no changes.
comp = await doc.compareVersion('HEAD', 'HEAD');
assert.equal(comp.summary, 'same');
assert.deepEqual(comp.parent, comp.left);
assert.deepEqual(comp.parent, comp.right);
assert.deepEqual(comp.details!.leftChanges, {tableRenames: [], tableDeltas: {}});
assert.deepEqual(comp.details!.rightChanges, {tableRenames: [], tableDeltas: {}});
// Examine the combination of the last two changes.
comp = await doc.compareVersion('HEAD~~', 'HEAD');
assert.hasAllKeys(comp, ['left', 'right', 'parent', 'summary', 'details']);
assert.equal(comp.summary, 'right');
assert.deepEqual(comp.parent, comp.left);
assert.notDeepEqual(comp.parent, comp.right);
assert.hasAllKeys(comp.left, ['n', 'h']);
assert.hasAllKeys(comp.right, ['n', 'h']);
assert.equal(comp.left.n, 2);
assert.equal(comp.right.n, 4);
assert.deepEqual(comp.details!.leftChanges, {tableRenames: [], tableDeltas: {}});
assert.deepEqual(comp.details!.rightChanges, {
tableRenames: [],
tableDeltas: {
Table1: {
updateRows: [1],
removeRows: [],
addRows: [2],
columnDeltas: {
A: {
[1]: [['a1'], ['A1']],
[2]: [null, ['a2']]
},
B: {[2]: [null, ['b2']]},
manualSort: {[2]: [null, [2]]},
},
columnRenames: [],
}
}
});
});
it('doc worker endpoints ignore any /dw/.../ prefix', async function () {
const docWorkerUrl = docs.serverUrl;
let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${docWorkerUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
if (docWorkerUrl !== homeUrl) {
resp = await axios.get(`${homeUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 200);
assert.containsAllKeys(resp.data, ['A', 'B', 'C']);
resp = await axios.get(`${homeUrl}/dw/zing/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
assert.equal(resp.status, 404);
}
});
describe('webhooks related endpoints', async function () {
/*
Regression test for old _subscribe endpoint. /docs/{did}/webhooks should be used instead to subscribe
*/
async function oldSubscribeCheck(requestBody: any, status: number, ...errors: RegExp[]) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
requestBody, chimpy
);
assert.equal(resp.status, status);
for (const error of errors) {
assert.match(resp.data.details?.userError || resp.data.error, error);
}
}
async function postWebhookCheck(requestBody: any, status: number, ...errors: RegExp[]) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/webhooks`,
requestBody, chimpy
);
assert.equal(resp.status, status);
for (const error of errors) {
assert.match(resp.data.details?.userError || resp.data.error, error);
}
return resp.data;
}
it("GET /docs/{did}/webhooks retrieves a list of webhooks", async function () {
const registerResponse = await postWebhookCheck({webhooks:[{fields:{tableId: "Table1", eventTypes: ["add"], url: "https://example.com"}}]}, 200);
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/webhooks`, chimpy);
try{
assert.equal(resp.status, 200);
assert.isAtLeast(resp.data.webhooks.length, 1);
assert.containsAllKeys(resp.data.webhooks[0], ['id', 'fields']);
assert.containsAllKeys(resp.data.webhooks[0].fields,
['enabled', 'isReadyColumn', 'memo', 'name', 'tableId', 'eventTypes', 'url']);
}
finally{
//cleanup
await deleteWebhookCheck(registerResponse.webhooks[0].id);
}
});
it("POST /docs/{did}/tables/{tid}/_subscribe validates inputs", async function () {
await oldSubscribeCheck({}, 400, /eventTypes is missing/);
await oldSubscribeCheck({eventTypes: 0}, 400, /url is missing/, /eventTypes is not an array/);
await oldSubscribeCheck({eventTypes: []}, 400, /url is missing/);
await oldSubscribeCheck({eventTypes: [], url: "https://example.com"}, 400, /eventTypes must be a non-empty array/);
await oldSubscribeCheck({eventTypes: ["foo"], url: "https://example.com"}, 400, /eventTypes\[0] is none of "add", "update"/);
await oldSubscribeCheck({eventTypes: ["add"]}, 400, /url is missing/);
await oldSubscribeCheck({eventTypes: ["add"], url: "https://evil.com"}, 403, /Provided url is forbidden/);
await oldSubscribeCheck({eventTypes: ["add"], url: "http://example.com"}, 403, /Provided url is forbidden/); // not https
await oldSubscribeCheck({eventTypes: ["add"], url: "https://example.com", isReadyColumn: "bar"}, 404, /Column not found "bar"/);
});
// in this endpoint webhookID is in body, not in path, so it also should be verified
it("POST /docs/{did}/webhooks validates inputs", async function () {
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1"}}]}, 400,
/eventTypes is missing/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: 0}}]}, 400,
/url is missing/, /eventTypes is not an array/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: []}}]},
400, /url is missing/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: [],
url: "https://example.com"}}]},
400, /eventTypes must be a non-empty array/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: ["foo"],
url: "https://example.com"}}]},
400, /eventTypes\[0] is none of "add", "update"/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: ["add"]}}]},
400, /url is missing/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: ["add"],
url: "https://evil.com"}}]},
403, /Provided url is forbidden/);
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: ["add"],
url: "http://example.com"}}]},
403, /Provided url is forbidden/); // not https
await postWebhookCheck({webhooks:[{fields: {tableId: "Table1", eventTypes: ["add"],
url: "https://example.com", isReadyColumn: "bar"}}]},
404, /Column not found "bar"/);
await postWebhookCheck({webhooks:[{fields: {eventTypes: ["add"], url: "https://example.com"}}]},
400, /tableId is missing/);
await postWebhookCheck({}, 400, /webhooks is missing/);
});
async function userCheck(user: AxiosRequestConfig, requestBody: any, status: number, responseBody: any) {
const resp = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_unsubscribe`,
requestBody, user
);
assert.equal(resp.status, status);
if (status !== 200) {
responseBody = {error: responseBody};
}
assert.deepEqual(resp.data, responseBody);
}
async function userDeleteCheck(user: AxiosRequestConfig, webhookId: string, status: number, ...errors: RegExp[]) {
const resp = await axios.delete(
`${serverUrl}/api/docs/${docIds.Timesheets}/webhooks/${webhookId}`,
user
);
assert.equal(resp.status, status);
for (const error of errors) {
assert.match(resp.data.details?.userError || resp.data.error, error);
}
}
interface SubscriptionInfo{
unsubscribeKey: string;
webhookId: string;
}
async function subscribeWebhook(): Promise<SubscriptionInfo> {
const subscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/_subscribe`,
{eventTypes: ["add"], url: "https://example.com"}, chimpy
);
assert.equal(subscribeResponse.status, 200);
// Editor needs unsubscribeKey.
const {unsubscribeKey, webhookId} = subscribeResponse.data;
return {unsubscribeKey, webhookId};
}
it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs for owners", async function () {
// Owner doesn't need unsubscribeKey.
const {webhookId} = await subscribeWebhook();
const check = userCheck.bind(null, chimpy);
await check({webhookId: "foo"}, 404, `Webhook not found "foo"`);
await check({}, 404, `Webhook not found ""`);
// Actually unsubscribe
await check({webhookId}, 200, {success: true});
// Trigger is now deleted!
await check({webhookId}, 404, `Webhook not found "${webhookId}"`);
});
it("DELETE /docs/{did}/tables/webhooks validates inputs for owners", async function () {
// Owner doesn't need unsubscribeKey.
const {webhookId} = await subscribeWebhook();
const check = userDeleteCheck.bind(null, chimpy);
await check("foo", 404, /Webhook not found "foo"/);
await check("", 404, /not found/, new RegExp(`/api/docs/${docIds.Timesheets}/webhooks/`));
// Actually unsubscribe
await check(webhookId, 200);
// Trigger is now deleted!
await check(webhookId, 404, new RegExp(`Webhook not found "${webhookId}"`));
});
async function getRegisteredWebhooks() {
const response = await axios.get(
`${serverUrl}/api/docs/${docIds.Timesheets}/webhooks`, chimpy);
return response.data.webhooks;
}
async function deleteWebhookCheck(webhookId: any) {
const response = await axios.delete(
`${serverUrl}/api/docs/${docIds.Timesheets}/webhooks/${webhookId}`, chimpy);
return response.data;
}
it("POST /docs/{did}/webhooks is adding new webhook to table "+
"and DELETE /docs/{did}/webhooks/{wid} is removing new webhook from table", async function(){
const registeredWebhook = await postWebhookCheck({webhooks:[{fields:{tableId: "Table1", eventTypes: ["add"], url: "https://example.com"}}]}, 200);
let webhookList = await getRegisteredWebhooks();
assert.equal(webhookList.length, 1);
assert.equal(webhookList[0].id, registeredWebhook.webhooks[0].id);
await deleteWebhookCheck(registeredWebhook.webhooks[0].id);
webhookList = await getRegisteredWebhooks();
assert.equal(webhookList.length, 0);
});
it("POST /docs/{did}/webhooks is adding new webhook should be able to add many webhooks at once", async function(){
const response = await postWebhookCheck(
{
webhooks:[
{fields:{tableId: "Table1", eventTypes: ["add"], url: "https://example.com"}},
{fields:{tableId: "Table1", eventTypes: ["add"], url: "https://example.com/2"}},
{fields:{tableId: "Table1", eventTypes: ["add"], url: "https://example.com/3"}},
]}, 200);
assert.equal(response.webhooks.length, 3);
const webhookList = await getRegisteredWebhooks();
assert.equal(webhookList.length, 3);
});
it("POST /docs/{did}/tables/{tid}/_unsubscribe validates inputs for editors", async function () {
const subscribeResponse = await subscribeWebhook();
const delta = {
users: {"kiwi@getgrist.com": 'editors' as string | null}
};
let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
assert.equal(accessResp.status, 200);
const check = userCheck.bind(null, kiwi);
await check({webhookId: "foo"}, 404, `Webhook not found "foo"`);
//no unsubscribeKey - should be not accepted for role other that owner
await check({webhookId: subscribeResponse.webhookId}, 400, 'Bad request: unsubscribeKey required');
//wrong unsubscribeKey - it should not be accepted
//subscribeResponse = await subscribeWebhook();
await check({webhookId: subscribeResponse.webhookId, unsubscribeKey: "foo"},
401, 'Wrong unsubscribeKey');
//subscribeResponse = await subscribeWebhook();
// Actually unsubscribe with the same unsubscribeKey that was returned by registration
await check({webhookId: subscribeResponse.webhookId, unsubscribeKey:subscribeResponse.unsubscribeKey},
200, {success: true});
// Trigger is now deleted!
await check({webhookId: subscribeResponse.webhookId, unsubscribeKey:subscribeResponse.unsubscribeKey},
404, `Webhook not found "${subscribeResponse.webhookId}"`);
// Remove editor access
delta.users['kiwi@getgrist.com'] = null;
accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
assert.equal(accessResp.status, 200);
});
it("DELETE /docs/{did}/tables/webhooks should not be allowed for not-owner", async function () {
const subscribeResponse = await subscribeWebhook();
const check = userDeleteCheck.bind(null, kiwi);
const delta = {
users: {"kiwi@getgrist.com": 'editors' as string | null}
};
let accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
assert.equal(accessResp.status, 200);
// Actually unsubscribe with the same unsubscribeKey that was returned by registration - it shouldn't be accepted
await check(subscribeResponse.webhookId, 403, /No owner access/);
// Remove editor access
delta.users['kiwi@getgrist.com'] = null;
accessResp = await axios.patch(`${homeUrl}/api/docs/${docIds.Timesheets}/access`, {delta}, chimpy);
assert.equal(accessResp.status, 200);
});
});
describe("Daily API Limit", () => {
let redisClient: RedisClient;
before(async function () {
if (!process.env.TEST_REDIS_URL) {
this.skip();
}
redisClient = createClient(process.env.TEST_REDIS_URL);
});
it("limits daily API usage", async function () {
// Make a new document in a test product with a low daily limit
const api = makeUserApi('testdailyapilimit');
const workspaceId = await getWorkspaceId(api, 'TestDailyApiLimitWs');
const docId = await api.newDoc({name: 'TestDoc1'}, workspaceId);
const max = testDailyApiLimitFeatures.baseMaxApiUnitsPerDocumentPerDay;
for (let i = 1; i <= max + 2; i++) {
let success = true;
try {
// Make some doc request so that it fails or succeeds
await api.getTable(docId, "Table1");
} catch (e) {
success = false;
}
// Usually the first `max` requests should succeed and the rest should fail having exceeded the daily limit.
// If a new minute starts in the middle of the requests, an extra request will be allowed for that minute.
// If a new day starts in the middle of the requests, this test will fail.
if (success) {
assert.isAtMost(i, max + 1);
} else {
assert.isAtLeast(i, max + 1);
}
}
});
it("limits daily API usage and sets the correct keys in redis", async function () {
this.retries(3);
// Make a new document in a free team site, currently the only real product which limits daily API usage.
const freeTeamApi = makeUserApi('freeteam');
const workspaceId = await getWorkspaceId(freeTeamApi, 'FreeTeamWs');
const docId = await freeTeamApi.newDoc({name: 'TestDoc2'}, workspaceId);
// Rather than making 5000 requests, set high counts directly for the current and next daily and hourly keys
const used = 999999;
let m = moment.utc();
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
await redisClient.multi()
.set(currentDay, String(used))
.set(currentHour, String(used))
.set(nextDay, String(used))
.set(nextHour, String(used))
.execAsync();
// Make 9 requests. The first 4 should succeed by fitting into the allocation for the minute.
// (Free team plans get 5000 requests per day, and 5000/24/60 ~= 3.47 which is rounded up to 4)
// The last request should fail. Don't check the middle 4 in case we're on the boundary of a minute.
for (let i = 1; i <= 9; i++) {
const last = i === 9;
m = moment.utc(); // get this before delaying to calculate accurate keys below
const response = await axios.get(`${serverUrl}/api/docs/${docId}/tables/Table1/records`,
chimpy);
// Allow time for redis to be updated.
await delay(100);
if (i <= 4) {
assert.equal(response.status, 200);
// Keys of the periods we expect to be incremented.
// For the first request, the server's usage cache is empty and it hasn't seen the redis values.
// So it thinks there hasn't been any usage and increments the current day/hour.
// After that it increments the next day/hour.
// We're only checking this for the first 4 requests
// because once the limit is exceeded the counts aren't incremented.
const first = i === 1;
const day = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[0], m);
const hour = docPeriodicApiUsageKey(docId, first, docApiUsagePeriods[1], m);
const minute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
if (!first) {
// The first request takes longer to serve because the document gets loaded,
// so only check the TTL (which gets set before request processing starts) on subsequent requests.
assert.deepEqual(
await redisClient.multi()
.ttl(minute)
.ttl(hour)
.ttl(day)
.execAsync(),
[
2 * 60,
2 * 60 * 60,
2 * 60 * 60 * 24,
],
);
}
assert.deepEqual(
await redisClient.multi()
.get(minute)
.get(hour)
.get(day)
.execAsync(),
[
String(i),
String(used + (first ? 1 : i - 1)),
String(used + (first ? 1 : i - 1)),
],
);
}
if (last) {
assert.equal(response.status, 429);
assert.deepEqual(response.data, {error: `Exceeded daily limit for document ${docId}`});
}
}
});
it("correctly allocates API requests based on the day, hour, and minute", async function () {
const m = moment.utc("1999-12-31T23:59:59Z");
const docId = "myDocId";
const currentDay = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[0], m);
const currentHour = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[1], m);
const currentMinute = docPeriodicApiUsageKey(docId, true, docApiUsagePeriods[2], m);
const nextDay = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[0], m);
const nextHour = docPeriodicApiUsageKey(docId, false, docApiUsagePeriods[1], m);
assert.equal(currentDay, `doc-myDocId-periodicApiUsage-1999-12-31`);
assert.equal(currentHour, `doc-myDocId-periodicApiUsage-1999-12-31T23`);
assert.equal(currentMinute, `doc-myDocId-periodicApiUsage-1999-12-31T23:59`);
assert.equal(nextDay, `doc-myDocId-periodicApiUsage-2000-01-01`);
assert.equal(nextHour, `doc-myDocId-periodicApiUsage-2000-01-01T00`);
const usage = new LRUCache<string, number>({max: 1024});
function check(expected: string[] | undefined) {
assert.deepEqual(getDocApiUsageKeysToIncr(docId, usage, dailyMax, m), expected);
}
const dailyMax = 5000;
const hourlyMax = 209; // 5000/24 ~= 208.33
const minuteMax = 4; // 5000/24/60 ~= 3.47
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax - 1);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
check([nextDay, currentHour, currentMinute]); // used up daily allocation
usage.set(currentHour, hourlyMax - 1);
check([nextDay, currentHour, currentMinute]);
usage.set(currentHour, hourlyMax);
check([nextDay, nextHour, currentMinute]); // used up hourly allocation
usage.set(currentMinute, minuteMax - 1);
check([nextDay, nextHour, currentMinute]);
usage.set(currentMinute, minuteMax);
check(undefined); // used up minutely allocation
usage.set(currentDay, 0);
check([currentDay, currentHour, currentMinute]);
usage.set(currentDay, dailyMax);
usage.set(currentHour, 0);
check([nextDay, currentHour, currentMinute]);
});
after(async function () {
if (process.env.TEST_REDIS_URL) {
await redisClient.quitAsync();
}
});
});
describe("Webhooks", () => {
let serving: Serving; // manages the test webhook server
let requests: WebhookRequests;
let receivedLastEvent: Promise<void>;
// Requests corresponding to adding 200 rows, sent in two batches of 100
const expected200AddEvents = [
_.range(100).map(i => ({
id: 9 + i, manualSort: 9 + i, A3: 200 + i, B3: true,
})),
_.range(100).map(i => ({
id: 109 + i, manualSort: 109 + i, A3: 300 + i, B3: true,
})),
];
// Every event is sent to three webhook URLs which differ by the subscribed eventTypes
// Each request is an array of one or more events.
// Multiple events caused by the same action bundle get batched into a single request.
const expectedRequests: WebhookRequests = {
"add": [
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
// After isReady (B) went to false and then true again
// we treat this as creation even though it's really an update
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 3, A3: 13, B3: true, manualSort: 3},
{id: 5, A3: 15, B3: true, manualSort: 5}],
[{id: 7, A3: 18, B3: true, manualSort: 7}],
...expected200AddEvents,
],
"update": [
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// From the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}],
],
"add,update": [
// add
[{id: 1, A: 1, B: true, C: null, manualSort: 1}],
[{id: 2, A: 4, B: true, C: null, manualSort: 2}],
[{id: 2, A: 7, B: true, C: null, manualSort: 2}],
// update
[{id: 2, A: 8, B: true, C: null, manualSort: 2}],
// from the big applies
[{id: 1, A3: 101, B3: true, manualSort: 1}, // update
{id: 3, A3: 13, B3: true, manualSort: 3}, // add
{id: 5, A3: 15, B3: true, manualSort: 5}], // add
[{id: 7, A3: 18, B3: true, manualSort: 7}], // add
...expected200AddEvents,
]
};
let redisMonitor: any;
let redisCalls: any[];
// Create couple of promises that can be used to monitor
// if the endpoint was called.
const successCalled = signal();
const notFoundCalled = signal();
const longStarted = signal();
const longFinished = signal();
// /probe endpoint will return this status when aborted.
let probeStatus = 200;
let probeMessage: string | null = "OK";
// Create an abort controller for the latest request. We will
// use it to abort the delay on the longEndpoint.
let controller = new AbortController();
async function autoSubscribe(
endpoint: string, docId: string, options?: {
tableId?: string,
isReadyColumn?: string | null,
eventTypes?: string[]
}) {
// Subscribe helper that returns a method to unsubscribe.
const data = await subscribe(endpoint, docId, options);
return () => unsubscribe(docId, data, options?.tableId ?? 'Table1');
}
function unsubscribe(docId: string, data: any, tableId = 'Table1') {
return axios.post(
`${serverUrl}/api/docs/${docId}/tables/${tableId}/_unsubscribe`,
data, chimpy
);
}
async function subscribe(endpoint: string, docId: string, options?: {
tableId?: string,
isReadyColumn?: string|null,
eventTypes?: string[],
name?: string,
memo?: string,
}) {
// Subscribe helper that returns a method to unsubscribe.
const {data, status} = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/${options?.tableId ?? 'Table1'}/_subscribe`,
{
eventTypes: options?.eventTypes ?? ['add', 'update'],
url: `${serving.url}/${endpoint}`,
isReadyColumn: options?.isReadyColumn === undefined ? 'B' : options?.isReadyColumn,
...pick(options, 'name', 'memo'),
}, chimpy
);
assert.equal(status, 200);
return data as WebhookSubscription;
}
async function clearQueue(docId: string) {
const deleteResult = await axios.delete(
`${serverUrl}/api/docs/${docId}/webhooks/queue`, chimpy
);
assert.equal(deleteResult.status, 200);
}
async function readStats(docId: string): Promise<WebhookSummary[]> {
const result = await axios.get(
`${serverUrl}/api/docs/${docId}/webhooks`, chimpy
);
assert.equal(result.status, 200);
return result.data.webhooks;
}
before(async function () {
this.timeout(30000);
requests = {
"add,update": [],
"add": [],
"update": [],
};
let resolveReceivedLastEvent: () => void;
receivedLastEvent = new Promise<void>(r => {
resolveReceivedLastEvent = r;
});
// TODO test retries on failure and slowness in a new test
serving = await serveSomething(app => {
app.use(bodyParser.json());
app.post('/200', ({body}, res) => {
successCalled.emit(body[0].A);
res.sendStatus(200);
res.end();
});
app.post('/404', ({body}, res) => {
notFoundCalled.emit(body[0].A);
// Webhooks treats it as an error and will retry. Probably it shouldn't work this way.
res.sendStatus(404);
res.end();
});
app.post('/probe', async ({body}, res) => {
longStarted.emit(body.map((r: any) => r.A));
// We are scoping the controller to this call, so any subsequent
// call will have a new controller. Caller can save this value to abort the previous calls.
const scoped = new AbortController();
controller = scoped;
try {
await delayAbort(20000, scoped.signal); // We don't expect to wait for this, we should be aborted
assert.fail('Should have been aborted');
} catch (exc) {
res.status(probeStatus);
res.send(probeMessage);
res.end();
longFinished.emit(body.map((r: any) => r.A));
}
});
app.post('/long', async ({body}, res) => {
longStarted.emit(body[0].A);
// We are scoping the controller to this call, so any subsequent
// call will have a new controller. Caller can save this value to abort the previous calls.
const scoped = new AbortController();
controller = scoped;
try {
await delayAbort(20000, scoped.signal); // We don't expect to wait for this.
res.sendStatus(200);
res.end();
longFinished.emit(body[0].A);
} catch (exc) {
res.sendStatus(200); // Send ok, so that it won't be seen as an error.
res.end();
longFinished.emit([408, body[0].A]); // We will signal that this is success but after aborting timeout.
}
});
app.post('/:eventTypes', async ({body, params: {eventTypes}}, res) => {
requests[eventTypes as keyof WebhookRequests].push(body);
res.sendStatus(200);
if (
_.flattenDeep(_.values(requests)).length >=
_.flattenDeep(_.values(expectedRequests)).length
) {
resolveReceivedLastEvent();
}
});
}, webhooksTestPort);
});
after(async function () {
await serving.shutdown();
});
describe('table endpoints', function () {
before(async function () {
this.timeout(30000);
// We rely on the REDIS server in this test.
if (!process.env.TEST_REDIS_URL) {
this.skip();
}
requests = {
"add,update": [],
"add": [],
"update": [],
};
redisCalls = [];
redisMonitor = createClient(process.env.TEST_REDIS_URL);
redisMonitor.monitor();
redisMonitor.on("monitor", (_time: any, args: any, _rawReply: any) => {
redisCalls.push(args);
});
});
after(async function () {
if (process.env.TEST_REDIS_URL) {
await redisMonitor.quitAsync();
}
});
it("delivers expected payloads from combinations of changes, with retrying and batching",
async function () {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc'}, ws1);
const doc = userApi.getDocAPI(docId);
// For some reason B is turned into Numeric even when given bools
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
// Make a webhook for every combination of event types
const subscribeResponses = [];
const webhookIds: Record<string, string> = {};
for (const eventTypes of [
["add"],
["update"],
["add", "update"],
]) {
const {data, status} = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table1/_subscribe`,
{eventTypes, url: `${serving.url}/${eventTypes}`, isReadyColumn: "B"}, chimpy
);
assert.equal(status, 200);
subscribeResponses.push(data);
webhookIds[data.webhookId] = String(eventTypes);
}
// Add and update some rows, trigger some events
// Values of A where B is true and thus the record is ready are [1, 4, 7, 8]
// So those are the values seen in expectedEvents
await doc.addRows("Table1", {
A: [1, 2],
B: [true, false], // 1 is ready, 2 is not ready yet
});
await doc.updateRows("Table1", {id: [2], A: [3]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [4], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [5], B: [false]}); // not ready again
await doc.updateRows("Table1", {id: [2], A: [6]}); // still not ready
await doc.updateRows("Table1", {id: [2], A: [7], B: [true]}); // ready!
await doc.updateRows("Table1", {id: [2], A: [8]}); // still ready!
// The end result here is additions for column A (now A3) with values [13, 15, 18]
// and an update for 101
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['BulkAddRecord', 'Table1', [3, 4, 5, 6], {A: [9, 10, 11, 12], B: [true, true, false, false]}],
['BulkUpdateRecord', 'Table1', [1, 2, 3, 4, 5, 6], {
A: [101, 102, 13, 14, 15, 16],
B: [true, false, true, false, true, false],
}],
['RenameColumn', 'Table1', 'A', 'A3'],
['RenameColumn', 'Table1', 'B', 'B3'],
['RenameTable', 'Table1', 'Table12'],
// FIXME a double rename A->A2->A3 doesn't seem to get summarised correctly
// ['RenameColumn', 'Table12', 'A2', 'A3'],
// ['RenameColumn', 'Table12', 'B2', 'B3'],
['RemoveColumn', 'Table12', 'C'],
], chimpy);
// FIXME record changes after a RenameTable in the same bundle
// don't appear in the action summary
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['AddRecord', 'Table12', 7, {A3: 17, B3: false}],
['UpdateRecord', 'Table12', 7, {A3: 18, B3: true}],
['AddRecord', 'Table12', 8, {A3: 19, B3: true}],
['UpdateRecord', 'Table12', 8, {A3: 20, B3: false}],
['AddRecord', 'Table12', 9, {A3: 20, B3: true}],
['RemoveRecord', 'Table12', 9],
], chimpy);
// Add 200 rows. These become the `expected200AddEvents`
await doc.addRows("Table12", {
A3: _.range(200, 400),
B3: arrayRepeat(200, true),
});
await receivedLastEvent;
// Unsubscribe
await Promise.all(subscribeResponses.map(async subscribeResponse => {
const unsubscribeResponse = await axios.post(
`${serverUrl}/api/docs/${docId}/tables/Table12/_unsubscribe`,
subscribeResponse, chimpy
);
assert.equal(unsubscribeResponse.status, 200);
assert.deepEqual(unsubscribeResponse.data, {success: true});
}));
// Further changes should generate no events because the triggers are gone
await doc.addRows("Table12", {
A3: [88, 99],
B3: [true, false],
});
assert.deepEqual(requests, expectedRequests);
// Check that the events were all pushed to the redis queue
const queueRedisCalls = redisCalls.filter(args => args[1] === "webhook-queue-" + docId);
const redisPushes = _.chain(queueRedisCalls)
.filter(args => args[0] === "rpush") // Array<["rpush", key, ...events: string[]]>
.flatMap(args => args.slice(2)) // events: string[]
.map(JSON.parse) // events: WebhookEvent[]
.groupBy('id') // {[webHookId: string]: WebhookEvent[]}
.mapKeys((_value, key) => webhookIds[key]) // {[eventTypes: 'add'|'update'|'add,update']: WebhookEvent[]}
.mapValues(group => _.map(group, 'payload')) // {[eventTypes: 'add'|'update'|'add,update']: RowRecord[]}
.value();
const expectedPushes = _.mapValues(expectedRequests, value => _.flatten(value));
assert.deepEqual(redisPushes, expectedPushes);
// Check that the events were all removed from the redis queue
const redisTrims = queueRedisCalls.filter(args => args[0] === "ltrim")
.map(([, , start, end]) => {
assert.equal(end, '-1');
start = Number(start);
assert.isTrue(start > 0);
return start;
});
const expectedTrims = Object.values(redisPushes).map(value => value.length);
assert.equal(
_.sum(redisTrims),
_.sum(expectedTrims),
);
});
});
describe("/webhooks endpoint", function () {
let docId: string;
let doc: DocAPI;
let stats: WebhookSummary[];
before(async function () {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
docId = await userApi.newDoc({name: 'testdoc2'}, ws1);
doc = userApi.getDocAPI(docId);
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
await userApi.applyUserActions(docId, [['AddTable', 'Table2', [{id: 'Foo'}, {id: 'Bar'}]]]);
});
const waitForQueue = async (length: number) => {
await waitForIt(async () => {
stats = await readStats(docId);
assert.equal(length, _.sum(stats.map(x => x.usage?.numWaiting ?? 0)));
}, 1000, 200);
};
it("should clear the outgoing queue", async () => {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc2'}, ws1);
const doc = userApi.getDocAPI(docId);
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
// Try to clear the queue, even if it is empty.
await clearQueue(docId);
const cleanup: (() => Promise<any>)[] = [];
// Subscribe a valid webhook endpoint.
cleanup.push(await autoSubscribe('200', docId));
// Subscribe an invalid webhook endpoint.
cleanup.push(await autoSubscribe('404', docId));
// Prepare signals, we will be waiting for those two to be called.
successCalled.reset();
notFoundCalled.reset();
// Trigger both events.
await doc.addRows("Table1", {
A: [1],
B: [true],
});
// Wait for both of them to be called (this is correct order)
await successCalled.waitAndReset();
await notFoundCalled.waitAndReset();
// Broken endpoint will be called multiple times here, and any subsequent triggers for working
// endpoint won't be called.
await notFoundCalled.waitAndReset();
// But the working endpoint won't be called more then once.
assert.isFalse(successCalled.called());
// Trigger second event.
await doc.addRows("Table1", {
A: [2],
B: [true],
});
// Error endpoint will be called with the first row (still).
const firstRow = await notFoundCalled.waitAndReset();
assert.deepEqual(firstRow, 1);
// But the working endpoint won't be called till we reset the queue.
assert.isFalse(successCalled.called());
// Now reset the queue.
await clearQueue(docId);
assert.isFalse(successCalled.called());
assert.isFalse(notFoundCalled.called());
// Prepare for new calls.
successCalled.reset();
notFoundCalled.reset();
// Trigger them.
await doc.addRows("Table1", {
A: [3],
B: [true],
});
// We will receive data from the 3rd row only (the second one was omitted).
let thirdRow = await successCalled.waitAndReset();
assert.deepEqual(thirdRow, 3);
thirdRow = await notFoundCalled.waitAndReset();
assert.deepEqual(thirdRow, 3);
// And the situation will be the same, the working endpoint won't be called till we reset the queue, but
// the error endpoint will be called with the third row multiple times.
await notFoundCalled.waitAndReset();
assert.isFalse(successCalled.called());
// Cleanup everything, we will now test request timeouts.
await Promise.all(cleanup.map(fn => fn())).finally(() => cleanup.length = 0);
await clearQueue(docId);
// Create 2 webhooks, one that is very long.
cleanup.push(await autoSubscribe('200', docId));
cleanup.push(await autoSubscribe('long', docId));
successCalled.reset();
longFinished.reset();
longStarted.reset();
// Trigger them.
await doc.addRows("Table1", {
A: [4],
B: [true],
});
// 200 will be called immediately.
await successCalled.waitAndReset();
// Long will be started immediately.
await longStarted.waitAndReset();
// But it won't be finished.
assert.isFalse(longFinished.called());
// It will be aborted.
controller.abort();
assert.deepEqual(await longFinished.waitAndReset(), [408, 4]);
// Trigger another event.
await doc.addRows("Table1", {
A: [5],
B: [true],
});
// We are stuck once again on the long call. But this time we won't
// abort it till the end of this test.
assert.deepEqual(await successCalled.waitAndReset(), 5);
assert.deepEqual(await longStarted.waitAndReset(), 5);
assert.isFalse(longFinished.called());
// Remember this controller for cleanup.
const controller5 = controller;
// Trigger another event.
await doc.addRows("Table1", {
A: [6],
B: [true],
});
// We are now completely stuck on the 5th row webhook.
assert.isFalse(successCalled.called());
assert.isFalse(longFinished.called());
// Clear the queue, it will free webhooks requests, but it won't cancel long handler on the external server
// so it is still waiting.
assert.isTrue((await axios.delete(
`${serverUrl}/api/docs/${docId}/webhooks/queue`, chimpy
)).status === 200);
// Now we can release the stuck request.
controller5.abort();
// We will be cancelled from the 5th row.
assert.deepEqual(await longFinished.waitAndReset(), [408, 5]);
// We won't be called for the 6th row at all, as it was stuck and the queue was purged.
assert.isFalse(successCalled.called());
assert.isFalse(longStarted.called());
// Trigger next event.
await doc.addRows("Table1", {
A: [7],
B: [true],
});
// We will be called once again with a new 7th row.
assert.deepEqual(await successCalled.waitAndReset(), 7);
assert.deepEqual(await longStarted.waitAndReset(), 7);
// But we are stuck again.
assert.isFalse(longFinished.called());
// And we can abort current request from 7th row (6th row was skipped).
controller.abort();
assert.deepEqual(await longFinished.waitAndReset(), [408, 7]);
// Cleanup all
await Promise.all(cleanup.map(fn => fn())).finally(() => cleanup.length = 0);
await clearQueue(docId);
});
it("should not call to a deleted webhook", async () => {
// Create a test document.
const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
const docId = await userApi.newDoc({name: 'testdoc4'}, ws1);
const doc = userApi.getDocAPI(docId);
await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
], chimpy);
// Subscribe to 2 webhooks, we will remove the second one.
const webhook1 = await autoSubscribe('probe', docId);
const webhook2 = await autoSubscribe('200', docId);
probeStatus = 200;
successCalled.reset();
longFinished.reset();
// Trigger them.
await doc.addRows("Table1", {
A: [1],
B: [true],
});
// Wait for the first one to be called.
await longStarted.waitAndReset();
// Now why we are on the call remove the second one.
// Check that it is queued.
const stats = await readStats(docId);
assert.equal(2, _.sum(stats.map(x => x.usage?.numWaiting ?? 0)));
await webhook2();
// Let the first one finish.
controller.abort();
await longFinished.waitAndReset();
// The second one is not called.
assert.isFalse(successCalled.called());
// Triggering next event, we will get only calls to the probe (first webhook).
await doc.addRows("Table1", {
A: [2],
B: [true],
});
await longStarted.waitAndReset();
controller.abort();
await longFinished.waitAndReset();
// Unsubscribe.
await webhook1();
});
it("should return statistics", async () => {
await clearQueue(docId);
// Read stats, it should be empty.
assert.deepEqual(await readStats(docId), []);
// Now subscribe couple of webhooks.
const first = await subscribe('200', docId);
const second = await subscribe('404', docId);
// And compare stats.
assert.deepEqual(await readStats(docId), [
{
id: first.webhookId,
fields: {
url: `${serving.url}/200`,
unsubscribeKey: first.unsubscribeKey,
eventTypes: ['add', 'update'],
enabled: true,
isReadyColumn: 'B',
tableId: 'Table1',
name: '',
memo: '',
}, usage : {
status: 'idle',
numWaiting: 0,
lastEventBatch: null
}
},
{
id: second.webhookId,
fields: {
url: `${serving.url}/404`,
unsubscribeKey: second.unsubscribeKey,
eventTypes: ['add', 'update'],
enabled: true,
isReadyColumn: 'B',
tableId: 'Table1',
name: '',
memo: '',
}, usage : {
status: 'idle',
numWaiting: 0,
lastEventBatch: null
}
},
]);
// We should be able to unsubscribe using info that we got.
await unsubscribe(docId, first);
await unsubscribe(docId, second);
assert.deepEqual(await readStats(docId), []);
// Test that stats work when there is no ready column.
let unsubscribe1 = await autoSubscribe('200', docId, {isReadyColumn: null});
assert.isNull((await readStats(docId))[0].fields.isReadyColumn);
await unsubscribe1();
// Now test that we receive some useful information and the state transition works.
unsubscribe1 = await autoSubscribe('probe', docId);
// Test also dates update.
let now = Date.now();
// Webhook starts as idle (tested already). Now we will trigger it.
longStarted.reset();
longFinished.reset();
await doc.addRows("Table1", {
A: [1],
B: [true],
});
// It will call our probe endpoint, so we will be able to see changes as they happen.
await longStarted.waitAndReset();
stats = await readStats(docId);
assert.isNotNull(stats[0].usage);
assert.equal(stats[0].usage?.numWaiting, 1);
assert.equal(stats[0].usage?.status, 'sending');
assert.isNotNull(stats[0].usage?.updatedTime);
assert.isAbove(stats[0].usage?.updatedTime ?? 0, now);
assert.isNull(stats[0].usage?.lastErrorMessage);
assert.isNull(stats[0].usage?.lastSuccessTime);
assert.isNull(stats[0].usage?.lastFailureTime);
assert.isNull(stats[0].usage?.lastHttpStatus);
assert.isNull(stats[0].usage?.lastEventBatch);
// Ok, we can return success now.
probeStatus = 200;
controller.abort();
await longFinished.waitAndReset();
// After releasing the hook, we are not 100% sure stats are updated, so we will wait a bit.
// If we are checking stats while we are holding the hook (in the probe endpoint) it is safe
// to assume that stats are up to date.
await waitForIt(async () => {
stats = await readStats(docId);
assert.equal(stats[0].usage?.numWaiting, 0);
}, 1000, 200);
assert.equal(stats[0].usage?.numWaiting, 0);
assert.equal(stats[0].usage?.status, 'idle');
assert.isAtLeast(stats[0].usage?.updatedTime ?? 0, now);
assert.isNull(stats[0].usage?.lastErrorMessage);
assert.isNull(stats[0].usage?.lastFailureTime);
assert.equal(stats[0].usage?.lastHttpStatus, 200);
assert.isAtLeast(stats[0].usage?.lastSuccessTime ?? 0, now);
assert.deepEqual(stats[0].usage?.lastEventBatch, {
status: 'success',
attempts: 1,
size: 1,
errorMessage: null,
httpStatus: 200,
});
// Now trigger the endpoint once again.
now = Date.now();
await doc.addRows("Table1", {
A: [2],
B: [true],
});
await longStarted.waitAndReset();
// This time, return an error, so we will have another attempt.
probeStatus = 404;
probeMessage = null;
controller.abort();
await longFinished.waitAndReset();
// Wait for the second attempt.
await longStarted.waitAndReset();
stats = await readStats(docId);
assert.equal(stats[0].usage?.numWaiting, 1);
assert.equal(stats[0].usage?.status, 'retrying');
assert.isAtLeast(stats[0].usage?.updatedTime ?? 0, now);
// There was no body in the response yet.
assert.isNull(stats[0].usage?.lastErrorMessage);
// Now we have a failure, and the success was before.
assert.isAtLeast(stats[0].usage?.lastFailureTime ?? 0, now);
assert.isBelow(stats[0].usage?.lastSuccessTime ?? 0, now);
assert.equal(stats[0].usage?.lastHttpStatus, 404);
// Batch contains info about last attempt.
assert.deepEqual(stats[0].usage?.lastEventBatch, {
status: 'failure',
attempts: 1,
size: 1,
errorMessage: null,
httpStatus: 404,
});
// Now make an error with some message.
probeStatus = 500;
probeMessage = 'Some error';
controller.abort();
await longFinished.waitAndReset();
await longStarted.waitAndReset();
// We have 3rd attempt, with an error message.
stats = await readStats(docId);
assert.equal(stats[0].usage?.numWaiting, 1);
assert.equal(stats[0].usage?.status, 'retrying');
assert.equal(stats[0].usage?.lastHttpStatus, 500);
assert.equal(stats[0].usage?.lastErrorMessage, probeMessage);
assert.deepEqual(stats[0].usage?.lastEventBatch, {
status: 'failure',
attempts: 2,
size: 1,
errorMessage: probeMessage,
httpStatus: 500,
});
// Now we will succeed.
probeStatus = 200;
controller.abort();
await longFinished.waitAndReset();
// Give it some time to update stats.
await waitForIt(async () => {
stats = await readStats(docId);
assert.equal(stats[0].usage?.numWaiting, 0);
}, 1000, 200);
stats = await readStats(docId);
assert.equal(stats[0].usage?.numWaiting, 0);
assert.equal(stats[0].usage?.status, 'idle');
assert.equal(stats[0].usage?.lastHttpStatus, 200);
assert.equal(stats[0].usage?.lastErrorMessage, probeMessage);
assert.isAtLeast(stats[0].usage?.lastFailureTime ?? 0, now);
assert.isAtLeast(stats[0].usage?.lastSuccessTime ?? 0, now);
assert.deepEqual(stats[0].usage?.lastEventBatch, {
status: 'success',
attempts: 3,
size: 1,
// Errors are cleared.
errorMessage: null,
httpStatus: 200,
});
// Clear everything.
await clearQueue(docId);
stats = await readStats(docId);
assert.isNotNull(stats[0].usage);
assert.equal(stats[0].usage?.numWaiting, 0);
assert.equal(stats[0].usage?.status, 'idle');
// Now pile some events with two webhooks to the probe.
const unsubscribe2 = await autoSubscribe('probe', docId);
await doc.addRows("Table1", {
A: [3],
B: [true],
});
await doc.addRows("Table1", {
A: [4],
B: [true],
});
await doc.addRows("Table1", {
A: [5],
B: [true],
});
assert.deepEqual(await longStarted.waitAndReset(), [3]);
stats = await readStats(docId);
assert.lengthOf(stats, 2);
// First one is pending and second one didn't have a chance to be executed yet.
assert.equal(stats[0].usage?.status, 'sending');
assert.equal(stats[1].usage?.status, 'idle');
assert.isNull(stats[0].usage?.lastEventBatch);
assert.isNull(stats[1].usage?.lastEventBatch);
assert.equal(6, _.sum(stats.map(x => x.usage?.numWaiting ?? 0)));
// Now let them finish in deterministic order.
controller.abort();
assert.deepEqual(await longFinished.waitAndReset(), [3]);
// We had 6 events to go, we've just finished the first one.
const nextPass = async (length: number, A: number) => {
assert.deepEqual(await longStarted.waitAndReset(), [A]);
stats = await readStats(docId);
assert.equal(length, _.sum(stats.map(x => x.usage?.numWaiting ?? 0)));
controller.abort();
assert.deepEqual(await longFinished.waitAndReset(), [A]);
};
// Now we have 5 events to go.
await nextPass(5, 3);
await nextPass(4, 4);
await nextPass(3, 4);
await nextPass(2, 5);
await nextPass(1, 5);
await waitForQueue(0);
await unsubscribe2();
await unsubscribe1();
});
it("should monitor failures", async () => {
const webhook3 = await subscribe('probe', docId);
const webhook4 = await subscribe('probe', docId);
// Now we have two webhooks, both will fail, but the first one will
// be put in the idle state and server will start to send the second one.
probeStatus = 509;
probeMessage = "fail";
await doc.addRows("Table1", {
A: [5],
B: [true],
});
const pass = async () => {
await longStarted.waitAndReset();
controller.abort();
await longFinished.waitAndReset();
};
// Server will retry this 4 times (GRIST_TRIGGER_MAX_ATTEMPTS = 4)
await pass();
await pass();
await pass();
await pass();
// And will fail, next it will call the second webhook.
await longStarted.waitAndReset();
// Hold it a bit (by not aborting).
// Read stats, first one is idle and has an error message, second one is active.
// (We don't need to wait - stats are up to date since triggers are waiting for us).
stats = await readStats(docId);
assert.equal(stats.length, 2);
assert.equal(stats[0].id, webhook3.webhookId);
assert.equal(stats[1].id, webhook4.webhookId);
assert.equal(stats[0].usage?.status, 'postponed');
assert.equal(stats[1].usage?.status, 'sending');
assert.equal(stats[0].usage?.numWaiting, 1);
assert.equal(stats[1].usage?.numWaiting, 1);
assert.equal(stats[0].usage?.lastErrorMessage, probeMessage);
assert.equal(stats[0].usage?.lastHttpStatus, 509);
assert.equal(stats[0].usage?.lastEventBatch?.status, "failure");
assert.isNull(stats[1].usage?.lastErrorMessage);
// We will now drain the queue, using the second webhook.
// First webhook is postponed, and the second is waiting for us. We have 2 events in total.
// To drain the queue, and cause this webhook to fail we will need to generate 10 more events,
// max queue since is 10, but if we generate exactly 10 events it will not work
// as the queue size will be 9 when the triggers decide to reject them.
await waitForQueue(2);
const addRowProm = doc.addRows("Table1", {
A: arrayRepeat(5, 100), // there are 2 webhooks, so 5 events per webhook.
B: arrayRepeat(5, true)
}).catch(() => {
});
// WARNING: we can't wait for it, as the Webhooks will literally stop the document, and wait
// for the queue to drain. So we will carefully go further, and wait for the queue to drain.
// It will try 4 times before giving up (the first call is in progress)
probeStatus = 429;
// First.
controller.abort();
await longFinished.waitAndReset();
// Second and third.
await pass();
await pass();
// Before the last one, we will wait for the add rows operation but in a different way.
// We will count how many webhook events were added so far, we should have 10 in total.
await waitForQueue(12);
// We are good to go, after trying for the 4th time it will gave up and remove this
// event from the queue.
await pass();
// Wait for the first webhook to start.
await longStarted.waitAndReset();
// And make sure we have info about rejected batch.
stats = await readStats(docId);
assert.equal(stats.length, 2);
assert.equal(stats[0].id, webhook3.webhookId);
assert.equal(stats[0].usage?.status, 'sending');
assert.equal(stats[0].usage?.numWaiting, 6);
assert.equal(stats[0].usage?.lastErrorMessage, probeMessage);
assert.equal(stats[0].usage?.lastHttpStatus, 509);
assert.equal(stats[1].id, webhook4.webhookId);
assert.equal(stats[1].usage?.status, 'error'); // webhook is in error state, some events were lost.
assert.equal(stats[1].usage?.lastEventBatch?.status, "rejected");
assert.equal(stats[1].usage?.numWaiting, 5); // We skipped one event.
// Now unfreeze document by handling all events (they are aligned so will be handled in just 2 batches, first
// one is already waiting in our /probe endpoint).
probeStatus = 200;
controller.abort();
await longFinished.waitAndReset();
await pass();
await waitForQueue(0);
// Now can wait for the rows to process.
await addRowProm;
await unsubscribe(docId, webhook3);
await unsubscribe(docId, webhook4);
});
describe('webhook update', function () {
it('should work correctly', async function () {
async function check(fields: any, status: number, error?: RegExp | string,
expectedFieldsCallback?: (fields: any) => any) {
let savedTableId = 'Table1';
const origFields = {
tableId: 'Table1',
eventTypes: ['add'],
isReadyColumn: 'B',
name: 'My Webhook',
memo: 'Sync store',
};
// subscribe
const webhook = await subscribe('foo', docId, origFields);
const expectedFields = {
url: `${serving.url}/foo`,
unsubscribeKey: webhook.unsubscribeKey,
eventTypes: ['add'],
isReadyColumn: 'B',
tableId: 'Table1',
enabled: true,
name: 'My Webhook',
memo: 'Sync store',
};
let stats = await readStats(docId);
assert.equal(stats.length, 1, 'stats=' + JSON.stringify(stats));
assert.equal(stats[0].id, webhook.webhookId);
assert.deepEqual(stats[0].fields, expectedFields);
// update
const resp = await axios.patch(
`${serverUrl}/api/docs/${docId}/webhooks/${webhook.webhookId}`, fields, chimpy
);
// check resp
assert.equal(resp.status, status, JSON.stringify(pick(resp, ['data', 'status'])));
if (resp.status === 200) {
stats = await readStats(docId);
assert.equal(stats.length, 1);
assert.equal(stats[0].id, webhook.webhookId);
if (expectedFieldsCallback) {
expectedFieldsCallback(expectedFields);
}
assert.deepEqual(stats[0].fields, {...expectedFields, ...fields});
if (fields.tableId) {
savedTableId = fields.tableId;
}
} else {
if (error instanceof RegExp) {
assert.match(resp.data.details?.userError || resp.data.error, error);
} else {
assert.deepEqual(resp.data, {error});
}
}
// finally unsubscribe
const unsubscribeResp = await unsubscribe(docId, webhook, savedTableId);
assert.equal(unsubscribeResp.status, 200, JSON.stringify(pick(unsubscribeResp, ['data', 'status'])));
stats = await readStats(docId);
assert.equal(stats.length, 0, 'stats=' + JSON.stringify(stats));
}
await check({url: `${serving.url}/bar`}, 200);
await check({url: "https://evil.com"}, 403, "Provided url is forbidden");
await check({url: "http://example.com"}, 403, "Provided url is forbidden"); // not https
// changing table without changing the ready column should reset the latter
await check({tableId: 'Table2'}, 200, '', expectedFields => expectedFields.isReadyColumn = null);
await check({tableId: 'Santa'}, 404, `Table not found "Santa"`);
await check({tableId: 'Table2', isReadyColumn: 'Foo'}, 200);
await check({eventTypes: ['add', 'update']}, 200);
await check({eventTypes: []}, 400, "eventTypes must be a non-empty array");
await check({eventTypes: ["foo"]}, 400, /eventTypes\[0] is none of "add", "update"/);
await check({isReadyColumn: null}, 200);
await check({isReadyColumn: "bar"}, 404, `Column not found "bar"`);
});
});
});
});
describe("Allowed Origin", () => {
it('should allow only example.com', async () => {
async function checkOrigin(origin: string, allowed: boolean) {
const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`,
{...chimpy, headers: {...chimpy.headers, "Origin": origin}}
);
assert.equal(resp.headers['access-control-allow-credentials'], allowed ? 'true' : undefined);
assert.equal(resp.status, allowed ? 200 : 403);
}
await checkOrigin("https://www.toto.com", false);
await checkOrigin("https://badexample.com", false);
await checkOrigin("https://bad.com/example.com/toto", false);
await checkOrigin("https://example.com/path", true);
await checkOrigin("https://example.com:3000/path", true);
await checkOrigin("https://good.example.com/toto", true);
});
it("should respond with correct CORS headers", async function () {
const wid = await getWorkspaceId(userApi, 'Private');
const docId = await userApi.newDoc({name: 'CorsTestDoc'}, wid);
await userApi.updateDocPermissions(docId, {
users: {
'everyone@getgrist.com': 'owners',
}
});
const chimpyConfig = configForUser("Chimpy");
const anonConfig = configForUser("Anonymous");
delete chimpyConfig.headers["X-Requested-With"];
delete anonConfig.headers["X-Requested-With"];
const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
const data = {records: [{fields: {}}]};
// Normal same origin requests
anonConfig.headers.Origin = serverUrl;
let response: AxiosResponse;
for (response of [
await axios.post(url, data, anonConfig),
await axios.get(url, anonConfig),
await axios.options(url, anonConfig),
]) {
assert.equal(response.status, 200);
assert.equal(response.headers['access-control-allow-methods'], 'GET, PATCH, PUT, POST, DELETE, OPTIONS');
assert.equal(response.headers['access-control-allow-headers'], 'Authorization, Content-Type, X-Requested-With');
assert.equal(response.headers['access-control-allow-origin'], serverUrl);
assert.equal(response.headers['access-control-allow-credentials'], 'true');
}
// Cross origin requests from untrusted origin.
for (const config of [anonConfig, chimpyConfig]) {
config.headers.Origin = "https://evil.com/";
for (response of [
await axios.post(url, data, config),
await axios.get(url, config),
await axios.options(url, config),
]) {
if (config === anonConfig) {
// Requests without credentials are still OK.
assert.equal(response.status, 200);
} else {
assert.equal(response.status, 403);
assert.deepEqual(response.data, {error: 'Credentials not supported for cross-origin requests'});
}
assert.equal(response.headers['access-control-allow-methods'], 'GET, PATCH, PUT, POST, DELETE, OPTIONS');
// Authorization header is not allowed
assert.equal(response.headers['access-control-allow-headers'], 'Content-Type, X-Requested-With');
// Origin is not echoed back. Arbitrary origin is allowed, but credentials are not.
assert.equal(response.headers['access-control-allow-origin'], '*');
assert.equal(response.headers['access-control-allow-credentials'], undefined);
}
}
// POST requests without credentials require a custom header so that a CORS preflight request is triggered.
// One possible header is X-Requested-With, which we removed at the start of the test.
// The other is Content-Type: application/json, which we have been using implicitly above because axios
// automatically treats the given data object as data. Passing a string instead prevents this.
response = await axios.post(url, JSON.stringify(data), anonConfig);
assert.equal(response.status, 401);
assert.deepEqual(response.data, {
error: "Unauthenticated requests require one of the headers" +
"'Content-Type: application/json' or 'X-Requested-With: XMLHttpRequest'"
});
// ^ that's for requests without credentials, otherwise we get the same 403 as earlier.
response = await axios.post(url, JSON.stringify(data), chimpyConfig);
assert.equal(response.status, 403);
assert.deepEqual(response.data, {error: 'Credentials not supported for cross-origin requests'});
});
});
// PLEASE ADD MORE TESTS HERE
}
interface WebhookRequests {
add: object[][];
update: object[][];
"add,update": object[][];
}
const ORG_NAME = 'docs-1';
function setup(name: string, cb: () => Promise<void>) {
let api: UserAPIImpl;
before(async function () {
suitename = name;
dataDir = path.join(tmpDir, `${suitename}-data`);
await fse.mkdirs(dataDir);
await setupDataDir(dataDir);
await cb();
// create TestDoc as an empty doc into Private workspace
userApi = api = makeUserApi(ORG_NAME);
const wid = await getWorkspaceId(api, 'Private');
docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid);
});
after(async function () {
// remove TestDoc
await api.deleteDoc(docIds.TestDoc);
delete docIds.TestDoc;
// stop all servers
await home.stop();
await docs.stop();
});
}
function makeUserApi(org: string, user?: string) {
return new UserAPIImpl(`${home.serverUrl}/o/${org}`, {
headers: {Authorization: `Bearer api_key_for_${user || 'chimpy'}`},
fetch: fetch as any,
newFormData: () => new FormData() as any,
logger: log
});
}
async function getWorkspaceId(api: UserAPIImpl, name: string) {
const workspaces = await api.getOrgWorkspaces('current');
return workspaces.find((w) => w.name === name)!.id;
}
// TODO: deal with safe port allocation
const webhooksTestPort = 34365;
async function setupDataDir(dir: string) {
// we'll be serving Hello.grist content for various document ids, so let's make copies of it in
// tmpDir
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Timesheets + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Bananas + '.grist'));
await testUtils.copyFixtureDoc('Hello.grist', path.resolve(dir, docIds.Antartic + '.grist'));
await testUtils.copyFixtureDoc(
'ApiDataRecordsTest.grist',
path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
}