Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2024-10-27 20:44:07 +00:00
Code Issues Projects Releases Wiki Activity

(core) Test webhooks

Browse Source 
Summary:
Tests DocApi endpoints _subscribe and _unsubscribe, including various bad inputs.

Tests that webhooks are sent to a test express server, with retrying on failure, filtered by event type, and waiting for isReadyColumn.

Test Plan: this

Reviewers: paulfitz

Reviewed By: paulfitz

Differential Revision: https://phab.getgrist.com/D3042
This commit is contained in:
Alex Hall 2021-09-27 22:50:29 +02:00
parent 8684c9e930
commit 8c1f8bc9a6
2 changed files with 38 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/server/lib/DocApi.ts
View File

@ -391,11 +391,6 @@ export class DocWorkerApi {
throw new ApiError('Provided url is forbidden', 403); throw new ApiError('Provided url is forbidden', 403);
} }
const unsubscribeKey = uuidv4();
const webhook: WebHookSecret = {unsubscribeKey, url};
const secretValue = JSON.stringify(webhook);
const webhookId = (await this._dbManager.addSecret(secretValue, activeDoc.docName)).id;
const metaTables = await getMetaTables(activeDoc, req); const metaTables = await getMetaTables(activeDoc, req);
const tableRef = tableIdToRef(metaTables, req.params.tableId); const tableRef = tableIdToRef(metaTables, req.params.tableId);
@ -409,6 +404,11 @@ export class DocWorkerApi {
isReadyColRef = colRefs[colRowIndex]; isReadyColRef = colRefs[colRowIndex];
} }
const unsubscribeKey = uuidv4();
const webhook: WebHookSecret = {unsubscribeKey, url};
const secretValue = JSON.stringify(webhook);
const webhookId = (await this._dbManager.addSecret(secretValue, activeDoc.docName)).id;
const webhookAction: WebhookAction = {type: "webhook", id: webhookId}; const webhookAction: WebhookAction = {type: "webhook", id: webhookId};
const sandboxRes = await handleSandboxError("_grist_Triggers", [], activeDoc.applyUserActions( const sandboxRes = await handleSandboxError("_grist_Triggers", [], activeDoc.applyUserActions(

38
app/server/lib/Triggers.ts
View File

@ -114,15 +114,19 @@ export class TriggersHandler {
const filters = {id: [...recordDeltas.keys()]}; const filters = {id: [...recordDeltas.keys()]};
const bulkColValues = fromTableDataAction(await this._activeDoc.fetchQuery(docSession, {tableId, filters})); const bulkColValues = fromTableDataAction(await this._activeDoc.fetchQuery(docSession, {tableId, filters}));
triggers.forEach(trigger => { log.info(`Processing ${triggers.length} triggers for ${bulkColValues.id.length} records of ${tableId}`);
for (const trigger of triggers) {
const actions = JSON.parse(trigger.actions) as TriggerAction[]; const actions = JSON.parse(trigger.actions) as TriggerAction[];
bulkColValues.id.forEach((rowId, rowIndex) => {
// Handle triggers in parallel (talking to redis) for (let rowIndex = 0; rowIndex < bulkColValues.id.length; rowIndex++) {
this._handleTrigger( const rowId = bulkColValues.id[rowIndex];
// Handle triggers serially to make order predictable
await this._handleTrigger(
trigger, actions, bulkColValues, rowIndex, rowId, recordDeltas.get(rowId)! trigger, actions, bulkColValues, rowIndex, rowId, recordDeltas.get(rowId)!
).catch(() => log.error("Error handling trigger action")); );
}); }
}); }
} }
// Handles a single trigger for a single record, initiating all the corresponding actions // Handles a single trigger for a single record, initiating all the corresponding actions
@ -182,12 +186,10 @@ export class TriggersHandler {
// All the values in this record // All the values in this record
const event = _.mapValues(bulkColValues, col => col[rowIndex]); const event = _.mapValues(bulkColValues, col => col[rowIndex]);
actions.forEach(action => { // Handle actions serially to make order predictable
// Handle actions in parallel for (const action of actions) {
this._handleTriggerAction( await this._handleTriggerAction(action, event);
action, event }
).catch(() => log.error("Error handling trigger action"));
});
} }
private async _handleTriggerAction(action: TriggerAction, event: Event) { private async _handleTriggerAction(action: TriggerAction, event: Event) {
@ -226,7 +228,7 @@ function sendPendingEvents() {
const pending = pendingEvents; const pending = pendingEvents;
pendingEvents = []; pendingEvents = [];
for (const [url, group] of _.toPairs(_.groupBy(pending, "url"))) { for (const [url, group] of _.toPairs(_.groupBy(pending, "url"))) {
const body = JSON.stringify(_.map(group, "event").reverse()); const body = JSON.stringify(_.map(group, "event"));
sendWebhookWithRetries(url, body).catch(() => log.error("Webhook failed!")); sendWebhookWithRetries(url, body).catch(() => log.error("Webhook failed!"));
} }
} }
@ -236,6 +238,7 @@ async function sendWebhookWithRetries(url: string, body: string) {
const maxWait = 64; const maxWait = 64;
let wait = 1; let wait = 1;
for (let i = 0; i < maxAttempts; i++) { for (let i = 0; i < maxAttempts; i++) {
try {
const response = await fetch(url, { const response = await fetch(url, {
method: 'POST', method: 'POST',
body, body,
@ -245,13 +248,16 @@ async function sendWebhookWithRetries(url: string, body: string) {
}); });
if (response.status === 200) { if (response.status === 200) {
return; return;
} else { }
log.warn(`Webhook responded with status ${response.status}`);
} catch (e) {
log.warn(`Webhook error: ${e}`);
}
await delay((wait + Math.random()) * 1000); await delay((wait + Math.random()) * 1000);
if (wait < maxWait) { if (wait < maxWait) {
wait *= 2; wait *= 2;
} }
} }
}
throw new Error("Webhook failed!"); throw new Error("Webhook failed!");
} }