(core) Adds a UI panel for managing webhooks

Summary:
This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo:

  * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables.
  * Cell values that would create an error can now be denied and reverted (as for the rest of Grist).
  * Changes made by other users are integrated in a sane way.
  * Basic undo/redo support is added using the regular undo/redo stack.
  * The table list in the drop-down is now updated if schema changes.
  * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation.
  *  Factored out webhook specific logic from general virtual table support.
  * Made a bunch of fixes to various broken behavior.
  * Added tests.

The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case.

I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error.

I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky.

Contains a migration, so needs an extra reviewer for that.

Test Plan: added tests

Reviewers: jarek, dsagal

Reviewed By: jarek, dsagal

Differential Revision: https://phab.getgrist.com/D3856
This commit is contained in:
Paul Fitzpatrick
2023-05-08 18:06:24 -04:00
parent 5e9f2e06ea
commit 603238e966
37 changed files with 1698 additions and 376 deletions

Binary file not shown.

Binary file not shown.

BIN
test/fixtures/docs/Hooks-v37.grist vendored Normal file

Binary file not shown.

View File

@@ -0,0 +1,275 @@
import { DocCreationInfo } from 'app/common/DocListAPI';
import { DocAPI } from 'app/common/UserAPI';
import { assert, driver, Key } from 'mocha-webdriver';
import * as gu from 'test/nbrowser/gristUtils';
import { setupTestSuite } from 'test/nbrowser/testUtils';
import { EnvironmentSnapshot } from 'test/server/testUtils';
import { server } from 'test/nbrowser/testUtils';
//import { Deps as TriggersDeps } from 'app/server/lib/Triggers';
describe('WebhookPage', function () {
this.timeout(30000);
const cleanup = setupTestSuite();
let session: gu.Session;
let oldEnv: EnvironmentSnapshot;
let docApi: DocAPI;
let doc: DocCreationInfo;
let host: string;
before(async function () {
oldEnv = new EnvironmentSnapshot();
host = new URL(server.getHost()).host;
process.env.ALLOWED_WEBHOOK_DOMAINS = host;
await server.restart();
session = await gu.session().teamSite.login();
const api = session.createHomeApi();
doc = await session.tempDoc(cleanup, 'Hello.grist');
docApi = api.getDocAPI(doc.id);
await api.applyUserActions(doc.id, [
['AddTable', 'Table2', [{id: 'A'}, {id: 'B'}, {id: 'C'}, {id: 'D'}, {id: 'E'}]],
]);
await api.applyUserActions(doc.id, [
['AddTable', 'Table3', [{id: 'A'}, {id: 'B'}, {id: 'C'}, {id: 'D'}, {id: 'E'}]],
]);
await api.updateDocPermissions(doc.id, {
users: {
// for convenience, we'll be sending payloads to the document itself.
'anon@getgrist.com': 'editors',
// check another owner's perspective.
[gu.session().user('user2').email]: 'owners',
}
});
});
after(async function () {
oldEnv.restore();
});
it('starts with an empty card', async function () {
await openWebhookPage();
assert.equal(await gu.getCardListCount(), 1); // includes empty card
assert.sameDeepMembers(await gu.getCardFieldLabels(), [
'Name',
'Memo',
'Event Types',
'URL',
'Table',
'Ready Column',
'Webhook Id',
'Enabled',
'Status',
]);
});
it('can create a persistent webhook', async function () {
// Set up a webhook for Table1, and send it to Table2 (for ease of testing).
await openWebhookPage();
await setField(1, 'Event Types', 'add\nupdate\n');
await setField(1, 'URL', `http://${host}/api/docs/${doc.id}/tables/Table2/records?flat=1`);
await setField(1, 'Table', 'Table1');
// Once event types, URL, and table are set, the webhook is created.
// Up until that point, nothing we've entered is actually persisted,
// there is no back end for it.
await gu.waitToPass(async () => {
assert.include(await getField(1, 'Webhook Id'), '-');
});
const id = await getField(1, 'Webhook Id');
// Reload and make sure the webhook id is still there.
await driver.navigate().refresh();
await waitForWebhookPage();
await gu.waitToPass(async () => {
assert.equal(await getField(1, 'Webhook Id'), id);
});
// Now other fields like name and memo are persisted.
await setField(1, 'Name', 'Test Webhook');
await setField(1, 'Memo', 'Test Memo');
await gu.waitForServer();
await driver.navigate().refresh();
await waitForWebhookPage();
await gu.waitToPass(async () => {
assert.equal(await getField(1, 'Name'), 'Test Webhook');
assert.equal(await getField(1, 'Memo'), 'Test Memo');
});
// Make sure the webhook is actually working.
await docApi.addRows('Table1', {A: ['zig'], B: ['zag']});
// Make sure the data gets delivered, and that the webhook status is updated.
await gu.waitToPass(async () => {
assert.lengthOf((await docApi.getRows('Table2')).A, 1);
assert.equal((await docApi.getRows('Table2')).A[0], 'zig');
assert.match(await getField(1, 'Status'), /status...success/);
});
// Remove the webhook and make sure it is no longer listed.
assert.equal(await gu.getCardListCount(), 2);
await gu.getDetailCell({col: 'Name', rowNum: 1}).click();
await gu.sendKeys(Key.chord(await gu.modKey(), Key.DELETE));
await gu.confirm(true, true);
await gu.waitForServer();
assert.equal(await gu.getCardListCount(), 1);
await driver.navigate().refresh();
await waitForWebhookPage();
assert.equal(await gu.getCardListCount(), 1);
await docApi.removeRows('Table2', [1]);
assert.lengthOf((await docApi.getRows('Table2')).A, 0);
});
it('can create two webhooks', async function () {
await openWebhookPage();
await setField(1, 'Event Types', 'add\nupdate\n');
await setField(1, 'URL', `http://${host}/api/docs/${doc.id}/tables/Table2/records?flat=1`);
await setField(1, 'Table', 'Table1');
await gu.waitForServer();
await setField(2, 'Event Types', 'add\n');
await setField(2, 'URL', `http://${host}/api/docs/${doc.id}/tables/Table3/records?flat=1`);
await setField(2, 'Table', 'Table1');
await gu.waitForServer();
await docApi.addRows('Table1', {A: ['zig2'], B: ['zag2']});
await gu.waitToPass(async () => {
assert.lengthOf((await docApi.getRows('Table2')).A, 1);
assert.lengthOf((await docApi.getRows('Table3')).A, 1);
assert.match(await getField(1, 'Status'), /status...success/);
assert.match(await getField(2, 'Status'), /status...success/);
});
await docApi.updateRows('Table1', {id: [1], A: ['zig3'], B: ['zag3']});
await gu.waitToPass(async () => {
assert.lengthOf((await docApi.getRows('Table2')).A, 2);
assert.lengthOf((await docApi.getRows('Table3')).A, 1);
assert.match(await getField(1, 'Status'), /status...success/);
});
await driver.sleep(100);
// confirm that nothing shows up to Table3.
assert.lengthOf((await docApi.getRows('Table3')).A, 1);
// Break everything down.
await gu.getDetailCell({col: 'Name', rowNum: 1}).click();
await gu.sendKeys(Key.chord(await gu.modKey(), Key.DELETE));
await gu.confirm(true, true);
await gu.waitForServer();
await gu.getDetailCell({col: 'Memo', rowNum: 1}).click();
await gu.sendKeys(Key.chord(await gu.modKey(), Key.DELETE));
await gu.waitForServer();
assert.equal(await gu.getCardListCount(), 1);
await driver.navigate().refresh();
await waitForWebhookPage();
assert.equal(await gu.getCardListCount(), 1);
await docApi.removeRows('Table2', [1, 2]);
await docApi.removeRows('Table3', [1]);
assert.lengthOf((await docApi.getRows('Table2')).A, 0);
assert.lengthOf((await docApi.getRows('Table3')).A, 0);
});
it('can create and repair a dud webhook', async function () {
await openWebhookPage();
await setField(1, 'Event Types', 'add\nupdate\n');
await setField(1, 'URL', `http://${host}/notathing`);
await setField(1, 'Table', 'Table1');
await gu.waitForServer();
await docApi.addRows('Table1', {A: ['dud1']});
await gu.waitToPass(async () => {
assert.match(await getField(1, 'Status'), /status...failure/);
assert.match(await getField(1, 'Status'), /numWaiting..1/);
});
await setField(1, 'URL', `http://${host}/api/docs/${doc.id}/tables/Table2/records?flat=1`);
await driver.findContent('button', /Clear Queue/).click();
await gu.waitForServer();
await gu.waitToPass(async () => {
assert.match(await getField(1, 'Status'), /numWaiting..0/);
});
assert.lengthOf((await docApi.getRows('Table2')).A, 0);
await docApi.addRows('Table1', {A: ['dud2']});
await gu.waitToPass(async () => {
assert.lengthOf((await docApi.getRows('Table2')).A, 1);
assert.match(await getField(1, 'Status'), /status...success/);
});
// Break everything down.
await gu.getDetailCell({col: 'Name', rowNum: 1}).click();
await gu.sendKeys(Key.chord(await gu.modKey(), Key.DELETE));
await gu.confirm(true, true);
await gu.waitForServer();
await docApi.removeRows('Table2', [1]);
assert.lengthOf((await docApi.getRows('Table2')).A, 0);
});
it('can keep multiple sessions in sync', async function () {
await openWebhookPage();
// Open another tab.
await driver.executeScript("return window.open('about:blank', '_blank')");
const [ownerTab, owner2Tab] = await driver.getAllWindowHandles();
await driver.switchTo().window(owner2Tab);
const otherSession = await gu.session().teamSite.user('user2').login();
await otherSession.loadDoc(`/doc/${doc.id}`);
await openWebhookPage();
await setField(1, 'Event Types', 'add\nupdate\n');
await setField(1, 'URL', `http://${host}/multiple`);
await setField(1, 'Table', 'Table1');
await gu.waitForServer();
await driver.switchTo().window(ownerTab);
await gu.waitToPass(async () => {
assert.match(await getField(1, 'URL'), /multiple/);
});
assert.equal(await gu.getCardListCount(), 2);
await setField(1, 'Memo', 'multiple memo');
await driver.switchTo().window(owner2Tab);
await gu.waitToPass(async () => {
assert.match(await getField(1, 'Memo'), /multiple memo/);
});
// Basic undo support.
await driver.switchTo().window(ownerTab);
await gu.undo();
await gu.waitToPass(async () => {
assert.equal(await getField(1, 'Memo'), '');
});
await driver.switchTo().window(owner2Tab);
await gu.waitToPass(async () => {
assert.equal(await getField(1, 'Memo'), '');
});
// Basic redo support.
await driver.switchTo().window(ownerTab);
await gu.redo();
await gu.waitToPass(async () => {
assert.match(await getField(1, 'Memo'), /multiple memo/);
});
await driver.switchTo().window(owner2Tab);
await gu.waitToPass(async () => {
assert.match(await getField(1, 'Memo'), /multiple memo/);
});
await gu.getDetailCell({col: 'Name', rowNum: 1}).click();
await gu.sendKeys(Key.chord(await gu.modKey(), Key.DELETE));
await gu.confirm(true, true);
await driver.switchTo().window(ownerTab);
await gu.waitToPass(async () => {
assert.equal(await gu.getCardListCount(), 1);
});
await driver.switchTo().window(owner2Tab);
await driver.close();
await driver.switchTo().window(ownerTab);
});
});
async function setField(rowNum: number, col: string, text: string) {
await gu.getDetailCell({col, rowNum}).click();
await gu.enterCell(text);
}
async function getField(rowNum: number, col: string) {
const cell = await gu.getDetailCell({col, rowNum});
return cell.getText();
}
async function openWebhookPage() {
await gu.openDocumentSettings();
const button = await driver.findContentWait('a', /Manage Webhooks/, 3000);
await gu.scrollIntoView(button).click();
await waitForWebhookPage();
}
async function waitForWebhookPage() {
await driver.findContentWait('button', /Clear Queue/, 3000);
// No section, so no easy utility for setting focus. Click on a random cell.
await gu.getDetailCell({col: 'Webhook Id', rowNum: 1}).click();
}

View File

@@ -5,6 +5,7 @@ import { setupTestSuite } from 'test/nbrowser/testUtils';
describe("saveViewSection", function() {
this.timeout(20000);
setupTestSuite();
gu.bigScreen();
const cleanup = setupTestSuite();

View File

@@ -2,6 +2,7 @@
import {ActionSummary} from 'app/common/ActionSummary';
import {BulkColValues, UserAction} from 'app/common/DocActions';
import {arrayRepeat} from 'app/common/gutil';
import {WebhookSummary} from 'app/common/Triggers';
import {DocAPI, DocState, UserAPIImpl} from 'app/common/UserAPI';
import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
@@ -15,7 +16,6 @@ import {
} from 'app/server/lib/DocApi';
import log from 'app/server/lib/log';
import {delayAbort} from 'app/server/lib/serverUtils';
import {WebhookSummary} from 'app/server/lib/Triggers';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
import {delay} from 'bluebird';
import * as bodyParser from 'body-parser';
@@ -1421,26 +1421,28 @@ function testDocApi() {
it("should validate request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async (payload: any, error: { error: string, details: string }) => {
const test = async (payload: any, error: { error: string, details: {userError: string} }) => {
const resp = await axios.put(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await test({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await test({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
await test({records: [{fields: {}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
details: {userError: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require is missing',
});
}});
await test({records: [{require: {id: "1"}}]},
{
error: 'Invalid payload',
details: 'Error: ' +
details: {userError: 'Error: ' +
'body.records[0] is not a AddOrUpdateRecord; ' +
'body.records[0].require.id is not a number',
});
}});
});
});
@@ -1462,23 +1464,23 @@ function testDocApi() {
it("validates request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
const test = async (payload: any, error: { error: string, details: string }) => {
const test = async(payload: any, error: {error: string, details: {userError: string}}) => {
const resp = await axios.post(url, payload, chimpy);
checkError(400, error, resp);
};
await test({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await test({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await test({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
// All column types are allowed, except Arrays (or objects) without correct code.
const testField = async (A: any) => {
await test({records: [{id: 1, fields: {A}}]}, {
error: 'Invalid payload', details:
'Error: body.records[0] is not a NewRecord; ' +
'body.records[0].fields.A is not a CellValue; ' +
'body.records[0].fields.A is none of number, ' +
'string, boolean, null, 1 more; body.records[0].' +
'fields.A[0] is not a GristObjCode; body.records[0]' +
'.fields.A[0] is not a valid enum value'
});
await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a NewRecord; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'}});
};
// test no code at all
await testField([]);
@@ -1627,34 +1629,29 @@ function testDocApi() {
it("validates request schema", async function () {
const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
async function failsWithError(payload: any, error: { error: string, details?: string }) {
async function failsWithError(payload: any, error: { error: string, details?: {userError: string} }){
const resp = await axios.patch(url, payload, chimpy);
checkError(400, error, resp);
}
await failsWithError({}, {error: 'Invalid payload', details: 'Error: body.records is missing'});
await failsWithError({}, {error: 'Invalid payload', details: {userError: 'Error: body.records is missing'}});
await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
await failsWithError({records: 1}, {
error: 'Invalid payload',
details: {userError: 'Error: body.records is not an array'}});
await failsWithError({records: []}, {
error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; body.records[0] is not an object'
});
await failsWithError({records: []}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record; body.records[0] is not an object'}});
await failsWithError({records: [{}]}, {
error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n ' +
'body.records[0].id is missing\n ' +
'body.records[0].fields is missing'
});
await failsWithError({records: [{}]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record\n '+
'body.records[0].id is missing\n '+
'body.records[0].fields is missing'}});
await failsWithError({records: [{id: "1"}]}, {
error: 'Invalid payload', details:
'Error: body.records[0] is not a Record\n' +
' body.records[0].id is not a number\n' +
' body.records[0].fields is missing'
});
await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record\n' +
' body.records[0].id is not a number\n' +
' body.records[0].fields is missing'}});
await failsWithError(
{records: [{id: 1, fields: {A: 1}}, {id: 2, fields: {B: 3}}]},
@@ -1662,15 +1659,13 @@ function testDocApi() {
// Test invalid object codes
const fieldIsNotValid = async (A: any) => {
await failsWithError({records: [{id: 1, fields: {A}}]}, {
error: 'Invalid payload', details:
'Error: body.records[0] is not a Record; ' +
'body.records[0].fields.A is not a CellValue; ' +
'body.records[0].fields.A is none of number, ' +
'string, boolean, null, 1 more; body.records[0].' +
'fields.A[0] is not a GristObjCode; body.records[0]' +
'.fields.A[0] is not a valid enum value'
});
await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details: {userError:
'Error: body.records[0] is not a Record; '+
'body.records[0].fields.A is not a CellValue; '+
'body.records[0].fields.A is none of number, '+
'string, boolean, null, 1 more; body.records[0].'+
'fields.A[0] is not a GristObjCode; body.records[0]'+
'.fields.A[0] is not a valid enum value'}});
};
await fieldIsNotValid([]);
await fieldIsNotValid(['ZZ']);
@@ -2785,7 +2780,7 @@ function testDocApi() {
);
assert.equal(resp.status, status);
for (const error of errors) {
assert.match(resp.data.details || resp.data.error, error);
assert.match(resp.data.details?.userError || resp.data.error, error);
}
}
@@ -3133,8 +3128,10 @@ function testDocApi() {
async function subscribe(endpoint: string, docId: string, options?: {
tableId?: string,
isReadyColumn?: string | null,
eventTypes?: string[]
isReadyColumn?: string|null,
eventTypes?: string[],
name?: string,
memo?: string,
}) {
// Subscribe helper that returns a method to unsubscribe.
const {data, status} = await axios.post(
@@ -3142,7 +3139,8 @@ function testDocApi() {
{
eventTypes: options?.eventTypes ?? ['add', 'update'],
url: `${serving.url}/${endpoint}`,
isReadyColumn: options?.isReadyColumn === undefined ? 'B' : options?.isReadyColumn
isReadyColumn: options?.isReadyColumn === undefined ? 'B' : options?.isReadyColumn,
...pick(options, 'name', 'memo'),
}, chimpy
);
assert.equal(status, 200);
@@ -3640,8 +3638,10 @@ function testDocApi() {
eventTypes: ['add', 'update'],
enabled: true,
isReadyColumn: 'B',
tableId: 'Table1'
}, usage: {
tableId: 'Table1',
name: '',
memo: '',
}, usage : {
status: 'idle',
numWaiting: 0,
lastEventBatch: null
@@ -3655,8 +3655,10 @@ function testDocApi() {
eventTypes: ['add', 'update'],
enabled: true,
isReadyColumn: 'B',
tableId: 'Table1'
}, usage: {
tableId: 'Table1',
name: '',
memo: '',
}, usage : {
status: 'idle',
numWaiting: 0,
lastEventBatch: null
@@ -3966,6 +3968,8 @@ function testDocApi() {
tableId: 'Table1',
eventTypes: ['add'],
isReadyColumn: 'B',
name: 'My Webhook',
memo: 'Sync store',
};
// subscribe
@@ -3978,6 +3982,8 @@ function testDocApi() {
isReadyColumn: 'B',
tableId: 'Table1',
enabled: true,
name: 'My Webhook',
memo: 'Sync store',
};
let stats = await readStats(docId);
@@ -4005,7 +4011,7 @@ function testDocApi() {
}
} else {
if (error instanceof RegExp) {
assert.match(resp.data.details || resp.data.error, error);
assert.match(resp.data.details?.userError || resp.data.error, error);
} else {
assert.deepEqual(resp.data, {error});
}

View File

@@ -57,7 +57,7 @@ TODO: this hardcoded port numbers might cause conflicts in parallel tests execut
const webhooksTestPort = 34365;
const webhooksTestProxyPort = 22335;
describe('Webhooks proxy configuration', function () {
describe('Webhooks-Proxy', function () {
// A testDir of the form grist_test_{USER}_{SERVER_NAME}
// - its a directory that will be base for all test related files and activities
const username = process.env.USER || "nobody";