gristlabs_grist-core/app/server/lib/DocApi.ts

2453 lines
98 KiB
TypeScript
Raw Normal View History

(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
import {concatenateSummaries, summarizeAction} from "app/common/ActionSummarizer";
import {createEmptyActionSummary} from "app/common/ActionSummary";
import {QueryFilters} from 'app/common/ActiveDocAPI';
import {ApiError, LimitType} from 'app/common/ApiError';
import {BrowserSettings} from "app/common/BrowserSettings";
import {
BulkColValues,
ColValues,
fromTableDataAction,
TableColValues,
TableRecordValue,
UserAction
} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {
extractTypeFromColType,
getReferencedTableId,
isBlankValue,
isFullReferencingType,
isRaisedException,
} from "app/common/gristTypes";
import {INITIAL_FIELDS_COUNT} from "app/common/Forms";
import {buildUrlId, parseUrlId, SHARE_KEY_PREFIX} from "app/common/gristUrls";
import {isAffirmative, safeJsonParse, timeoutReached} from "app/common/gutil";
import {SchemaTypes} from "app/common/schema";
import {SortFunc} from 'app/common/SortFunc';
import {Sort} from 'app/common/SortSpec';
import {MetaRowRecord} from 'app/common/TableData';
import {TelemetryMetadataByLevel} from "app/common/Telemetry";
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
import {WebhookFields} from "app/common/Triggers";
import TriggersTI from 'app/common/Triggers-ti';
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
import {DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/HomeDBManager';
import * as Types from "app/plugin/DocApiTypes";
import DocApiTypesTI from "app/plugin/DocApiTypes-ti";
import {GristObjCode} from "app/plugin/GristData";
import GristDataTI from 'app/plugin/GristData-ti';
import {OpOptions} from "app/plugin/TableOperations";
import {
handleSandboxErrorOnPlatform,
TableOperationsImpl,
TableOperationsPlatform
} from 'app/plugin/TableOperationsImpl';
import {ActiveDoc, colIdToRef as colIdToReference, getRealTableId, tableIdToRef} from "app/server/lib/ActiveDoc";
import {appSettings} from "app/server/lib/AppSettings";
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
import {sendForCompletion} from 'app/server/lib/Assistance';
import {
assertAccess,
getAuthorizedUserId,
getOrSetDocAuth,
getTransitiveHeaders,
getUserId,
isAnonymousUser,
RequestWithLogin
} from 'app/server/lib/Authorizer';
import {DocManager} from "app/server/lib/DocManager";
import {docSessionFromRequest, getDocSessionShare, makeExceptionalDocSession,
OptDocSession} from "app/server/lib/DocSession";
import {DocWorker} from "app/server/lib/DocWorker";
import {IDocWorkerMap} from "app/server/lib/DocWorkerMap";
import {DownloadOptions, parseExportParameters} from "app/server/lib/Export";
import {downloadDSV} from "app/server/lib/ExportDSV";
import {collectTableSchemaInFrictionlessFormat} from "app/server/lib/ExportTableSchema";
import {streamXLSX} from "app/server/lib/ExportXLSX";
import {expressWrap} from 'app/server/lib/expressWrap';
import {filterDocumentInPlace} from "app/server/lib/filterUtils";
import {googleAuthTokenMiddleware} from "app/server/lib/GoogleAuth";
import {exportToDrive} from "app/server/lib/GoogleExport";
import {GristServer} from 'app/server/lib/GristServer';
import {HashUtil} from 'app/server/lib/HashUtil';
import {makeForkIds} from "app/server/lib/idUtils";
import log from 'app/server/lib/log';
import {
getDocId,
getDocScope,
getScope,
integerParam,
isParameterOn,
optBooleanParam,
optIntegerParam,
optStringParam,
sendOkReply,
sendReply,
stringParam
} from 'app/server/lib/requestUtils';
import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
import {localeFromRequest} from "app/server/lib/ServerLocale";
import {isUrlAllowed, WebhookAction, WebHookSecret} from "app/server/lib/Triggers";
import {fetchDoc, globalUploadSet, handleOptionalUpload, handleUpload,
makeAccessId} from "app/server/lib/uploads";
import * as assert from 'assert';
import contentDisposition from 'content-disposition';
import {Application, NextFunction, Request, RequestHandler, Response} from "express";
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
import * as _ from "lodash";
import LRUCache from 'lru-cache';
import * as moment from 'moment';
import fetch from 'node-fetch';
import * as path from 'path';
import * as t from "ts-interface-checker";
import {Checker} from "ts-interface-checker";
import uuidv4 from "uuid/v4";
import { Document } from "app/gen-server/entity/Document";
// Cap on the number of requests that can be outstanding on a single document via the
// rest doc api. When this limit is exceeded, incoming requests receive an immediate
// reply with status 429.
const MAX_PARALLEL_REQUESTS_PER_DOC = 10;
// This is NOT the number of docs that can be handled at a time.
// It's a very generous upper bound of what that number might be.
// If there are more docs than this for which API requests are being regularly made at any moment,
// then the _dailyUsage cache may become unreliable and users may be able to exceed their allocated requests.
const MAX_ACTIVE_DOCS_USAGE_CACHE = 1000;
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
// Maximum amount of time that a webhook endpoint can hold the mutex for in withDocTriggersLock.
const MAX_DOC_TRIGGERS_LOCK_MS = 15_000;
// Maximum duration of a call to /sql. Does not apply to internal calls to SQLite.
const MAX_CUSTOM_SQL_MSEC = appSettings.section('integrations')
.section('sql').flag('timeout').requireInt({
envVar: 'GRIST_SQL_TIMEOUT_MSEC',
defaultValue: 1000,
});
type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) => Promise<void>;
// Schema validators for api endpoints that creates or updates records.
const {
RecordsPatch, RecordsPost, RecordsPut,
ColumnsPost, ColumnsPatch, ColumnsPut,
SqlPost,
TablesPost, TablesPatch,
} = t.createCheckers(DocApiTypesTI, GristDataTI);
for (const checker of [RecordsPatch, RecordsPost, RecordsPut, ColumnsPost, ColumnsPatch,
SqlPost, TablesPost, TablesPatch]) {
checker.setReportedPath("body");
}
// Schema validators for api endpoints that creates or updates records.
const {
WebhookPatch,
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
WebhookSubscribe,
WebhookSubscribeCollection,
} = t.createCheckers(TriggersTI);
/**
* Middleware for validating request's body with a Checker instance.
*/
function validate(checker: Checker): RequestHandler {
return (req, res, next) => {
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
validateCore(checker, req, req.body);
next();
};
}
function validateCore(checker: Checker, req: Request, body: any) {
try {
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
checker.check(body);
} catch(err) {
log.warn(`Error during api call to ${req.path}: Invalid payload: ${String(err)}`);
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
throw new ApiError('Invalid payload', 400, {userError: String(err)});
}
}
export class DocWorkerApi {
// Map from docId to number of requests currently being handled for that doc
private _currentUsage = new Map<string, number>();
// Map from (docId, time period) combination produced by docPeriodicApiUsageKey
// to number of requests previously served for that combination.
// We multiply by 5 because there are 5 relevant keys per doc at any time (current/next day/hour and current minute).
private _dailyUsage = new LRUCache<string, number>({max: 5 * MAX_ACTIVE_DOCS_USAGE_CACHE});
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
constructor(private _app: Application, private _docWorker: DocWorker,
private _docWorkerMap: IDocWorkerMap, private _docManager: DocManager,
private _dbManager: HomeDBManager, private _grist: GristServer) {}
/**
* Adds endpoints for the doc api.
*
* Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside
* to apply to these routes.
*/
public addEndpoints() {
(core) add initial support for special shares Summary: This gives a mechanism for controlling access control within a document that is distinct from (though implemented with the same machinery as) granular access rules. It was hard to find a good way to insert this that didn't dissolve in a soup of complications, so here's what I went with: * When reading rules, if there are shares, extra rules are added. * If there are shares, all rules are made conditional on a "ShareRef" user property. * "ShareRef" is null when a doc is accessed in normal way, and the row id of a share when accessed via a share. There's no UI for controlling shares (George is working on it for forms), but you can do it by editing a `_grist_Shares` table in a document. Suppose you make a fresh document with a single page/table/widget, then to create an empty share you can do: ``` gristDocPageModel.gristDoc.get().docData.sendAction(['AddRecord', '_grist_Shares', null, {linkId: 'xyz', options: '{"publish": true}'}]) ``` If you look at the home db now there should be something in the `shares` table: ``` $ sqlite3 -table landing.db "select * from shares" +----+------------------------+------------------------+--------------+---------+ | id | key | doc_id | link_id | options | +----+------------------------+------------------------+--------------+---------+ | 1 | gSL4g38PsyautLHnjmXh2K | 4qYuace1xP2CTcPunFdtan | xyz | ... | +----+------------------------+------------------------+--------------+---------+ ``` If you take the key from that (gSL4g38PsyautLHnjmXh2K in this case) and replace the document's urlId in its URL with `s.<key>` (in this case `s.gSL4g38PsyautLHnjmXh2K` then you can use the regular document landing page (it will be quite blank initially) or API endpoint via the share. E.g. for me `http://localhost:8080/o/docs/s0gSL4g38PsyautLHnjmXh2K/share-inter-3` accesses the doc. To actually share some material - useful commands: ``` gristDocPageModel.gristDoc.get().docData.getMetaTable('_grist_Views_section').getRecords() gristDocPageModel.gristDoc.get().docData.sendAction(['UpdateRecord', '_grist_Views_section', 1, {shareOptions: '{"publish": true, "form": true}'}]) gristDocPageModel.gristDoc.get().docData.getMetaTable('_grist_Pages').getRecords() gristDocPageModel.gristDoc.get().docData.sendAction(['UpdateRecord', '_grist_Pages', 1, {shareRef: 1}]) ``` For a share to be effective, at least one page needs to have its shareRef set to the rowId of the share, and at least one widget on one of those pages needs to have its shareOptions set to {"publish": "true", "form": "true"} (meaning turn on sharing, and include form sharing), and the share itself needs {"publish": true} on its options. I think special shares are kind of incompatible with public sharing, since by their nature (allowing access to all endpoints) they easily expose the docId, and changing that would be hard. Test Plan: tests added Reviewers: dsagal, georgegevoian Reviewed By: dsagal, georgegevoian Subscribers: jarek, dsagal Differential Revision: https://phab.getgrist.com/D4144
2024-01-03 16:53:20 +00:00
this._app.use((req, res, next) => {
if (req.url.startsWith('/api/s/')) {
req.url = req.url.replace('/api/s/', `/api/docs/${SHARE_KEY_PREFIX}`);
}
next();
});
// check document exists (not soft deleted) and user can view it
const canView = expressWrap(this._assertAccess.bind(this, 'viewers', false));
// check document exists (not soft deleted) and user can edit it
const canEdit = expressWrap(this._assertAccess.bind(this, 'editors', false));
const checkAnonymousCreation = expressWrap(this._checkAnonymousCreation.bind(this));
const isOwner = expressWrap(this._assertAccess.bind(this, 'owners', false));
// check user can edit document, with soft-deleted documents being acceptable
const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true));
// converts google code to access token and adds it to request object
const decodeGoogleToken = expressWrap(googleAuthTokenMiddleware.bind(null));
// check that limit can be increased by 1
const checkLimit = (type: LimitType) => expressWrap(this._checkLimit.bind(this, type));
// Middleware to limit number of outstanding requests per document. Will also
// handle errors like expressWrap would.
const throttled = this._apiThrottle.bind(this);
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
const withDoc = (callback: WithDocHandler) => throttled(this._requireActiveDoc(callback));
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
// Like withDoc, but only one such callback can run at a time per active doc.
// This is used for webhook endpoints to prevent simultaneous changes to configuration
// or clearing queues which could lead to weird problems.
const withDocTriggersLock = (callback: WithDocHandler) => withDoc(
async (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) =>
await activeDoc.triggersLock.runExclusive(async () => {
// We don't want to hold the mutex indefinitely so that if one call gets stuck
// (especially while trying to apply user actions which are stalled by a full queue)
// another call which would clear a queue, disable a webhook, or fix something related
// can eventually succeed.
if (await timeoutReached(MAX_DOC_TRIGGERS_LOCK_MS, callback(activeDoc, req, resp), {rethrow: true})) {
log.rawError(`Webhook endpoint timed out, releasing mutex`,
{method: req.method, path: req.path, docId: activeDoc.docName});
}
})
);
// Apply user actions to a document.
this._app.post('/api/docs/:docId/apply', canEdit, withDoc(async (activeDoc, req, res) => {
const parseStrings = !isAffirmative(req.query.noparse);
res.json(await activeDoc.applyUserActions(docSessionFromRequest(req), req.body, {parseStrings}));
}));
async function readTable(
req: RequestWithLogin,
activeDoc: ActiveDoc,
tableId: string,
filters: QueryFilters,
params: QueryParameters & {immediate?: boolean}) {
(core) get all tests working under python3/gvisor Summary: This verifies that all existing tests are capable of running under python3/gvisor, and fixes the small issues that came up. It does not yet activate python3 tests on all diffs, only diffs that specifically request them. * Adds a suffix in test names and output directories for tests run with PYTHON_VERSION=3, so that results of the same test run with and without the flag can be aggregated cleanly. * Adds support for checkpointing to the gvisor sandbox adapter. * Prepares a checkpoint made after grist python code has loaded in the gvisor sandbox. * Changes how `DOC_URL` is passed to the sandbox, since it can no longer be passed in as an environment variable when using checkpoints. * Uses the checkpoint to speed up tests using the gvisor sandbox, otherwise a lot of tests need more time (especially on mac under docker). * Directs jenkins to run all tests with python2 and python3 when a new file `buildtools/changelogs/python.txt` is touched (this diff counts as touching that file). * Tweaks miscellaneous tests - some needed fixes exposed by slightly different timing - a small number actually give different results in py3 (removal of `u` prefixes). - some needed a little more time The DOC_URL change is not the ultimate solution we want for DOC_URL. Eventually it should be a variable that gets updated, like the date perhaps. This is just a small pragmatic change to preserve existing behavior. Tests are run mindlessly as py3, and for some tests it won't change anything (e.g. if they do not use NSandbox). Tests are not run in parallel, doubling overall test time. Checkpoints could be useful in deployment, though this diff doesn't use them there. The application of checkpoints doesn't check for other configuration like 3-versus-5-pipe that we don't actually use. Python2 tests run using pynbox as always for now. The diff got sufficiently bulky that I didn't tackle running py3 on "regular" diffs in it. My preference, given that most tests don't appear to stress the python side of things, would be to make a selection of the tests that do and a few wild cards, and run those tests on both pythons rather then all of them. For diffs making a significant python change, I'd propose touching buildtools/changelogs/python.txt for full tests. But this is a conversation in progress. A total of 6886 tests ran on this diff. Test Plan: this is a step in preparing tests for py3 transition Reviewers: dsagal Reviewed By: dsagal Subscribers: dsagal Differential Revision: https://phab.getgrist.com/D3066
2021-10-18 17:37:51 +00:00
// Option to skip waiting for document initialization.
const immediate = isAffirmative(params.immediate);
if (!Object.keys(filters).every(col => Array.isArray(filters[col]))) {
throw new ApiError("Invalid query: filter values must be arrays", 400);
}
const session = docSessionFromRequest(req);
const {tableData} = await handleSandboxError(tableId, [], activeDoc.fetchQuery(
session, {tableId, filters}, !immediate));
// For metaTables we don't need to specify columns, search will infer it from the sort expression.
const isMetaTable = tableId.startsWith('_grist');
const columns = isMetaTable ? null :
await handleSandboxError('', [], activeDoc.getTableCols(session, tableId, true));
// Apply sort/limit parameters, if set. TODO: move sorting/limiting into data engine
// and sql.
return applyQueryParameters(fromTableDataAction(tableData), params, columns);
}
async function getTableData(activeDoc: ActiveDoc, req: RequestWithLogin, optTableId?: string) {
const filters = req.query.filter ? JSON.parse(String(req.query.filter)) : {};
// Option to skip waiting for document initialization.
const immediate = isAffirmative(req.query.immediate);
const tableId = await getRealTableId(optTableId || req.params.tableId, {activeDoc, req});
const params = getQueryParameters(req);
return await readTable(req, activeDoc, tableId, filters, {...params, immediate});
}
function asRecords(
columnData: TableColValues,
opts?: {
optTableId?: string;
includeHidden?: boolean;
includeId?: boolean;
}
): TableRecordValue[] {
const fieldNames = Object.keys(columnData).filter((k) => {
if (!opts?.includeId && k === "id") {
return false;
}
if (
!opts?.includeHidden &&
(k === "manualSort" || k.startsWith("gristHelper_"))
) {
return false;
}
return true;
});
return columnData.id.map((id, index) => {
const result: TableRecordValue = { id, fields: {} };
for (const key of fieldNames) {
let value = columnData[key][index];
if (isRaisedException(value)) {
_.set(result, ["errors", key], (value as string[])[1]);
value = null;
}
result.fields[key] = value;
}
return result;
});
}
async function getTableRecords(
activeDoc: ActiveDoc, req: RequestWithLogin, opts?: { optTableId?: string; includeHidden?: boolean }
): Promise<TableRecordValue[]> {
const columnData = await getTableData(activeDoc, req, opts?.optTableId);
return asRecords(columnData, opts);
}
// Get the specified table in column-oriented format
this._app.get('/api/docs/:docId/tables/:tableId/data', canView,
withDoc(async (activeDoc, req, res) => {
res.json(await getTableData(activeDoc, req));
})
);
// Get the specified table in record-oriented format
this._app.get('/api/docs/:docId/tables/:tableId/records', canView,
withDoc(async (activeDoc, req, res) => {
const records = await getTableRecords(activeDoc, req,
{ includeHidden: isAffirmative(req.query.hidden) }
);
res.json({records});
})
);
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
const registerWebhook = async (activeDoc: ActiveDoc, req: RequestWithLogin, webhook: WebhookFields) => {
const {fields, url} = await getWebhookSettings(activeDoc, req, null, webhook);
if (!fields.eventTypes?.length) {
throw new ApiError(`eventTypes must be a non-empty array`, 400);
}
if (!isUrlAllowed(url)) {
throw new ApiError('Provided url is forbidden', 403);
}
if (!fields.tableRef) {
throw new ApiError(`tableId is required`, 400);
}
const unsubscribeKey = uuidv4();
const webhookSecret: WebHookSecret = {unsubscribeKey, url};
const secretValue = JSON.stringify(webhookSecret);
const webhookId = (await this._dbManager.addSecret(secretValue, activeDoc.docName)).id;
try {
const webhookAction: WebhookAction = {type: "webhook", id: webhookId};
const sandboxRes = await handleSandboxError("_grist_Triggers", [], activeDoc.applyUserActions(
docSessionFromRequest(req),
[['AddRecord', "_grist_Triggers", null, {
enabled: true,
...fields,
actions: JSON.stringify([webhookAction])
}]]));
return {
unsubscribeKey,
triggerId: sandboxRes.retValues[0],
webhookId,
};
} catch (err) {
// remove webhook
await this._dbManager.removeWebhook(webhookId, activeDoc.docName, '', false);
throw err;
} finally {
await activeDoc.sendWebhookNotification();
}
};
const removeWebhook = async (activeDoc: ActiveDoc, req: RequestWithLogin, res: Response) => {
const {unsubscribeKey} = req.body as WebhookSubscription;
const webhookId = req.params.webhookId??req.body.webhookId;
// owner does not need to provide unsubscribeKey
const checkKey = !(await this._isOwner(req));
const triggerRowId = activeDoc.triggers.getWebhookTriggerRecord(webhookId).id;
// Validate unsubscribeKey before deleting trigger from document
await this._dbManager.removeWebhook(webhookId, activeDoc.docName, unsubscribeKey, checkKey);
activeDoc.triggers.webhookDeleted(webhookId);
await handleSandboxError("_grist_Triggers", [], activeDoc.applyUserActions(
docSessionFromRequest(req),
[['RemoveRecord', "_grist_Triggers", triggerRowId]]));
await activeDoc.sendWebhookNotification();
res.json({success: true});
};
async function getWebhookSettings(activeDoc: ActiveDoc, req: RequestWithLogin,
webhookId: string|null, webhook: WebhookFields) {
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
const metaTables = await getMetaTables(activeDoc, req);
const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
const trigger = webhookId ? activeDoc.triggers.getWebhookTriggerRecord(webhookId) : undefined;
let currentTableId = trigger ? tablesTable.getValue(trigger.tableRef, 'tableId')! : undefined;
const {url, eventTypes, watchedColIds, isReadyColumn, name} = webhook;
const tableId = await getRealTableId(req.params.tableId || webhook.tableId, {metaTables});
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
const fields: Partial<SchemaTypes['_grist_Triggers']> = {};
if (url && !isUrlAllowed(url)) {
throw new ApiError('Provided url is forbidden', 403);
}
if (eventTypes) {
if (!eventTypes.length) {
throw new ApiError(`eventTypes must be a non-empty array`, 400);
}
fields.eventTypes = [GristObjCode.List, ...eventTypes];
}
if (tableId !== undefined) {
if (watchedColIds) {
if (tableId !== currentTableId && currentTableId) {
// if the tableId changed, we need to reset the watchedColIds
fields.watchedColRefList = [GristObjCode.List];
} else {
if (!tableId) {
throw new ApiError(`Cannot find columns "${watchedColIds}" because table is not known`, 404);
}
fields.watchedColRefList = [GristObjCode.List, ...watchedColIds
.filter(colId => colId.trim() !== "")
.map(
colId => { return colIdToReference(metaTables, tableId, colId.trim().replace(/^\$/, '')); }
)];
}
} else {
fields.watchedColRefList = [GristObjCode.List];
}
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
fields.tableRef = tableIdToRef(metaTables, tableId);
currentTableId = tableId;
}
if (isReadyColumn !== undefined) {
// When isReadyColumn is defined let's explicitly change the ready column to the new col
// id, null or empty string being a special case that unsets it.
if (isReadyColumn !== null && isReadyColumn !== '') {
if (!currentTableId) {
throw new ApiError(`Cannot find column "${isReadyColumn}" because table is not known`, 404);
}
fields.isReadyColRef = colIdToReference(metaTables, currentTableId, isReadyColumn);
} else {
fields.isReadyColRef = 0;
}
} else if (tableId) {
// When isReadyColumn is undefined but tableId was changed, let's unset the ready column
fields.isReadyColRef = 0;
}
// assign other field properties
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
Object.assign(fields, _.pick(webhook, ['enabled', 'memo']));
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
if (name) {
fields.label = name;
}
return {
fields,
url,
};
}
// Get the columns of the specified table in recordish format
this._app.get('/api/docs/:docId/tables/:tableId/columns', canView,
withDoc(async (activeDoc, req, res) => {
const tableId = await getRealTableId(req.params.tableId, {activeDoc, req});
const includeHidden = isAffirmative(req.query.hidden);
const columns = await handleSandboxError('', [],
activeDoc.getTableCols(docSessionFromRequest(req), tableId, includeHidden));
res.json({columns});
})
);
// Get the tables of the specified document in recordish format
this._app.get('/api/docs/:docId/tables', canView,
withDoc(async (activeDoc, req, res) => {
const records = await getTableRecords(activeDoc, req, { optTableId: "_grist_Tables" });
const tables: Types.RecordWithStringId[] = records.map((record) => ({
id: String(record.fields.tableId),
fields: {
..._.omit(record.fields, "tableId"),
tableRef: record.id,
}
})).filter(({id}) => id);
res.json({tables});
})
);
// The upload should be a multipart post with an 'upload' field containing one or more files.
// Returns the list of rowIds for the rows created in the _grist_Attachments table.
this._app.post('/api/docs/:docId/attachments', canEdit, withDoc(async (activeDoc, req, res) => {
const uploadResult = await handleUpload(req, res);
res.json(await activeDoc.addAttachments(docSessionFromRequest(req), uploadResult.uploadId));
}));
// Select the fields from an attachment record that we want to return to the user,
// and convert the timeUploaded from a number to an ISO string.
function cleanAttachmentRecord(record: MetaRowRecord<"_grist_Attachments">) {
const {fileName, fileSize, timeUploaded: time} = record;
const timeUploaded = (typeof time === 'number') ? new Date(time).toISOString() : undefined;
return {fileName, fileSize, timeUploaded};
}
// Returns cleaned metadata for all attachments in /records format.
this._app.get('/api/docs/:docId/attachments', canView, withDoc(async (activeDoc, req, res) => {
const rawRecords = await getTableRecords(activeDoc, req, { optTableId: "_grist_Attachments" });
const records = rawRecords.map(r => ({
id: r.id,
fields: cleanAttachmentRecord(r.fields as MetaRowRecord<"_grist_Attachments">),
}));
res.json({records});
}));
// Returns cleaned metadata for a given attachment ID (i.e. a rowId in _grist_Attachments table).
this._app.get('/api/docs/:docId/attachments/:attId', canView, withDoc(async (activeDoc, req, res) => {
const attId = integerParam(req.params.attId, 'attId');
const attRecord = activeDoc.getAttachmentMetadata(attId);
res.json(cleanAttachmentRecord(attRecord));
}));
// Responds with attachment contents, with suitable Content-Type and Content-Disposition.
this._app.get('/api/docs/:docId/attachments/:attId/download', canView, withDoc(async (activeDoc, req, res) => {
const attId = integerParam(req.params.attId, 'attId');
const tableId = optStringParam(req.params.tableId, 'tableId');
const colId = optStringParam(req.params.colId, 'colId');
const rowId = optIntegerParam(req.params.rowId, 'rowId');
if ((tableId || colId || rowId) && !(tableId && colId && rowId)) {
throw new ApiError('define all of tableId, colId and rowId, or none.', 400);
}
const attRecord = activeDoc.getAttachmentMetadata(attId);
const cell = (tableId && colId && rowId) ? {tableId, colId, rowId} : undefined;
const fileIdent = attRecord.fileIdent as string;
const ext = path.extname(fileIdent);
const origName = attRecord.fileName as string;
const fileName = ext ? path.basename(origName, path.extname(origName)) + ext : origName;
const fileData = await activeDoc.getAttachmentData(docSessionFromRequest(req), attRecord, {cell});
res.status(200)
.type(ext)
// Construct a content-disposition header of the form 'attachment; filename="NAME"'
.set('Content-Disposition', contentDisposition(fileName, {type: 'attachment'}))
.set('Cache-Control', 'private, max-age=3600')
.send(fileData);
}));
// Mostly for testing
this._app.post('/api/docs/:docId/attachments/updateUsed', canEdit, withDoc(async (activeDoc, req, res) => {
await activeDoc.updateUsedAttachmentsIfNeeded();
res.json(null);
}));
this._app.post('/api/docs/:docId/attachments/removeUnused', isOwner, withDoc(async (activeDoc, req, res) => {
const expiredOnly = isAffirmative(req.query.expiredonly);
const verifyFiles = isAffirmative(req.query.verifyfiles);
await activeDoc.removeUnusedAttachments(expiredOnly);
if (verifyFiles) {
await verifyAttachmentFiles(activeDoc);
}
res.json(null);
}));
this._app.post('/api/docs/:docId/attachments/verifyFiles', isOwner, withDoc(async (activeDoc, req, res) => {
await verifyAttachmentFiles(activeDoc);
res.json(null);
}));
async function verifyAttachmentFiles(activeDoc: ActiveDoc) {
assert.deepStrictEqual(
await activeDoc.docStorage.all(`SELECT DISTINCT fileIdent AS ident FROM _grist_Attachments ORDER BY ident`),
await activeDoc.docStorage.all(`SELECT ident FROM _gristsys_Files ORDER BY ident`),
);
}
// Adds records given in a column oriented format,
// returns an array of row IDs
this._app.post('/api/docs/:docId/tables/:tableId/data', canEdit,
withDoc(async (activeDoc, req, res) => {
const colValues = req.body as BulkColValues;
const count = colValues[Object.keys(colValues)[0]].length;
const op = await getTableOperations(req, activeDoc);
const ids = await op.addRecords(count, colValues);
res.json(ids);
})
);
// Adds records given in a record oriented format,
// returns in the same format as GET /records but without the fields object for now
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
// WARNING: The `req.body` object is modified in place.
this._app.post('/api/docs/:docId/tables/:tableId/records', canEdit,
withDoc(async (activeDoc, req, res) => {
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
let body = req.body;
if (isAffirmative(req.query.flat)) {
if (!body.records && Array.isArray(body)) {
for (const [i, rec] of body.entries()) {
if (!rec.fields) {
// If ids arrive in a loosely formatted flat payload,
// remove them since we cannot honor them. If not loosely
// formatted, throw an error later. TODO: would be useful
// to have a way to exclude or rename fields via query
// parameters.
if (rec.id) { delete rec.id; }
body[i] = {fields: rec};
}
}
body = {records: body};
}
}
validateCore(RecordsPost, req, body);
const ops = await getTableOperations(req, activeDoc);
const records = await ops.create(body.records);
if (req.query.utm_source === 'grist-forms') {
activeDoc.logTelemetryEvent(docSessionFromRequest(req), 'submittedForm');
}
res.json({records});
})
);
// A GET /sql endpoint that takes a query like ?q=select+*+from+Table1
// Not very useful, apart from testing - see the POST endpoint for
// serious use.
// If SQL statements that modify the DB are ever supported, they should
// not be permitted by this endpoint.
this._app.get(
'/api/docs/:docId/sql', canView,
withDoc(async (activeDoc, req, res) => {
const sql = stringParam(req.query.q, 'q');
await this._runSql(activeDoc, req, res, { sql });
}));
// A POST /sql endpoint, accepting a body like:
// { "sql": "select * from Table1 where name = ?", "args": ["Paul"] }
// Only SELECT statements are currently supported.
this._app.post(
'/api/docs/:docId/sql', canView, validate(SqlPost),
withDoc(async (activeDoc, req, res) => {
await this._runSql(activeDoc, req, res, req.body);
}));
// Create columns in a table, given as records of the _grist_Tables_column metatable.
this._app.post('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPost),
withDoc(async (activeDoc, req, res) => {
const body = req.body as Types.ColumnsPost;
const tableId = await getRealTableId(req.params.tableId, {activeDoc, req});
const actions = body.columns.map(({fields, id: colId}) =>
// AddVisibleColumn adds the column to all widgets of the table.
// This isn't necessarily what the user wants, but it seems like a good default.
// Maybe there should be a query param to control this?
["AddVisibleColumn", tableId, colId, fields || {}]
);
const {retValues} = await handleSandboxError(tableId, [],
activeDoc.applyUserActions(docSessionFromRequest(req), actions)
);
const columns = retValues.map(({colId}) => ({id: colId}));
res.json({columns});
})
);
// Create new tables in a doc. Unlike POST /records or /columns, each 'record' (table) should have a `columns`
// property in the same format as POST /columns above, and no `fields` property.
this._app.post('/api/docs/:docId/tables', canEdit, validate(TablesPost),
withDoc(async (activeDoc, req, res) => {
const body = req.body as Types.TablesPost;
const actions = body.tables.map(({columns, id}) => {
const colInfos = columns.map(({fields, id: colId}) => ({...fields, id: colId}));
return ["AddTable", id, colInfos];
});
const {retValues} = await activeDoc.applyUserActions(docSessionFromRequest(req), actions);
const tables = retValues.map(({table_id}) => ({id: table_id}));
res.json({tables});
})
);
this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => {
const rowIds = req.body;
const op = await getTableOperations(req, activeDoc);
await op.destroy(rowIds);
res.json(null);
}));
// Download full document
// TODO: look at download behavior if ActiveDoc is shutdown during call (cannot
// use withDoc wrapper)
this._app.get('/api/docs/:docId/download', canView, throttled(async (req, res) => {
// Support a dryRun flag to check if user has the right to download the
// full document.
const dryRun = isAffirmative(req.query.dryrun || req.query.dryRun);
const dryRunSuccess = () => res.status(200).json({dryRun: 'allowed'});
const filename = await this._getDownloadFilename(req);
// We want to be have a way download broken docs that ActiveDoc may not be able
// to load. So, if the user owns the document, we unconditionally let them
// download.
if (await this._isOwner(req, {acceptTrunkForSnapshot: true})) {
if (dryRun) { dryRunSuccess(); return; }
try {
// We carefully avoid creating an ActiveDoc for the document being downloaded,
// in case it is broken in some way. It is convenient to be able to download
// broken files for diagnosis/recovery.
return await this._docWorker.downloadDoc(req, res, this._docManager.storageManager, filename);
} catch (e) {
if (e.message && e.message.match(/does not exist yet/)) {
// The document has never been seen on file system / s3. It may be new, so
// we try again after having created an ActiveDoc for the document.
await this._getActiveDoc(req);
return this._docWorker.downloadDoc(req, res, this._docManager.storageManager, filename);
} else {
throw e;
}
}
} else {
// If the user is not an owner, we load the document as an ActiveDoc, and then
// check if the user has download permissions.
const activeDoc = await this._getActiveDoc(req);
if (!await activeDoc.canDownload(docSessionFromRequest(req))) {
throw new ApiError('not authorized to download this document', 403);
}
if (dryRun) { dryRunSuccess(); return; }
return this._docWorker.downloadDoc(req, res, this._docManager.storageManager, filename);
}
}));
// Fork the specified document.
this._app.post('/api/docs/:docId/fork', canView, withDoc(async (activeDoc, req, res) => {
const result = await activeDoc.fork(docSessionFromRequest(req));
res.json(result);
}));
// Initiate a fork. Used internally to implement ActiveDoc.fork. Only usable via a Permit.
this._app.post('/api/docs/:docId/create-fork', canEdit, throttled(async (req, res) => {
const docId = stringParam(req.params.docId, 'docId');
const srcDocId = stringParam(req.body.srcDocId, 'srcDocId');
if (srcDocId !== req.specialPermit?.otherDocId) { throw new Error('access denied'); }
const fname = await this._docManager.storageManager.prepareFork(srcDocId, docId);
await filterDocumentInPlace(docSessionFromRequest(req), fname);
res.json({srcDocId, docId});
}));
// Update records given in column format
// The records to update are identified by their id column.
this._app.patch('/api/docs/:docId/tables/:tableId/data', canEdit,
withDoc(async (activeDoc, req, res) => {
const columnValues = req.body;
const rowIds = columnValues.id;
// sandbox expects no id column
delete columnValues.id;
const ops = await getTableOperations(req, activeDoc);
await ops.updateRecords(columnValues, rowIds);
res.json(null);
})
);
// Update records given in records format
this._app.patch('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPatch),
withDoc(async (activeDoc, req, res) => {
const body = req.body as Types.RecordsPatch;
const ops = await getTableOperations(req, activeDoc);
await ops.update(body.records);
res.json(null);
})
);
// Update columns given in records format
this._app.patch('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPatch),
withDoc(async (activeDoc, req, res) => {
const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column");
const tableId = await getRealTableId(req.params.tableId, {activeDoc, req});
const tableRef = tablesTable.findMatchingRowId({tableId});
if (!tableRef) {
throw new ApiError(`Table not found "${tableId}"`, 404);
}
const body = req.body as Types.ColumnsPatch;
const columns: Types.Record[] = body.columns.map((col) => {
const id = columnsTable.findMatchingRowId({parentId: tableRef, colId: col.id});
if (!id) {
throw new ApiError(`Column not found "${col.id}"`, 404);
}
return {...col, id};
});
const ops = await getTableOperations(req, activeDoc, "_grist_Tables_column");
await ops.update(columns);
res.json(null);
})
);
// Update tables given in records format
this._app.patch('/api/docs/:docId/tables', canEdit, validate(TablesPatch),
withDoc(async (activeDoc, req, res) => {
const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
const body = req.body as Types.TablesPatch;
const tables: Types.Record[] = body.tables.map((table) => {
const id = tablesTable.findMatchingRowId({tableId: table.id});
if (!id) {
throw new ApiError(`Table not found "${table.id}"`, 404);
}
return {...table, id};
});
const ops = await getTableOperations(req, activeDoc, "_grist_Tables");
await ops.update(tables);
res.json(null);
})
);
// Add or update records given in records format
this._app.put('/api/docs/:docId/tables/:tableId/records', canEdit, validate(RecordsPut),
withDoc(async (activeDoc, req, res) => {
const ops = await getTableOperations(req, activeDoc);
const body = req.body as Types.RecordsPut;
const options = {
add: !isAffirmative(req.query.noadd),
update: !isAffirmative(req.query.noupdate),
onMany: stringParam(req.query.onmany || "first", "onmany", {
allowed: ["first", "none", "all"],
}) as 'first'|'none'|'all'|undefined,
allowEmptyRequire: isAffirmative(req.query.allow_empty_require),
};
await ops.upsert(body.records, options);
res.json(null);
})
);
// Add or update records given in records format
this._app.put('/api/docs/:docId/tables/:tableId/columns', canEdit, validate(ColumnsPut),
withDoc(async (activeDoc, req, res) => {
const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
const columnsTable = activeDoc.docData!.getMetaTable("_grist_Tables_column");
const tableId = await getRealTableId(req.params.tableId, {activeDoc, req});
const tableRef = tablesTable.findMatchingRowId({tableId});
if (!tableRef) {
throw new ApiError(`Table not found "${tableId}"`, 404);
}
const body = req.body as Types.ColumnsPut;
const addActions: UserAction[] = [];
const updateActions: UserAction[] = [];
const updatedColumnsIds = new Set();
for (const col of body.columns) {
const id = columnsTable.findMatchingRowId({parentId: tableRef, colId: col.id});
if (id) {
updateActions.push( ['UpdateRecord', '_grist_Tables_column', id, col.fields] );
updatedColumnsIds.add( id );
} else {
addActions.push( ['AddVisibleColumn', tableId, col.id, col.fields] );
}
}
const getRemoveAction = async () => {
const columns = await handleSandboxError('', [],
activeDoc.getTableCols(docSessionFromRequest(req), tableId));
const columnsToRemove = columns
.map(col => col.fields.colRef as number)
.filter(colRef => !updatedColumnsIds.has(colRef));
return [ 'BulkRemoveRecord', '_grist_Tables_column', columnsToRemove ];
};
const actions = [
...(!isAffirmative(req.query.noupdate) ? updateActions : []),
...(!isAffirmative(req.query.noadd) ? addActions : []),
...(isAffirmative(req.query.replaceall) ? [ await getRemoveAction() ] : [] )
];
await handleSandboxError(tableId, [],
activeDoc.applyUserActions(docSessionFromRequest(req), actions)
);
res.json(null);
})
);
this._app.delete('/api/docs/:docId/tables/:tableId/columns/:colId', canEdit,
withDoc(async (activeDoc, req, res) => {
const {colId} = req.params;
const tableId = await getRealTableId(req.params.tableId, {activeDoc, req});
const actions = [ [ 'RemoveColumn', tableId, colId ] ];
await handleSandboxError(tableId, [colId],
activeDoc.applyUserActions(docSessionFromRequest(req), actions)
);
res.json(null);
})
);
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
// Add a new webhook and trigger
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
this._app.post('/api/docs/:docId/webhooks', isOwner, validate(WebhookSubscribeCollection),
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(async (activeDoc, req, res) => {
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
const registeredWebhooks: Array<WebhookSubscription> = [];
for(const webhook of req.body.webhooks) {
const registeredWebhook = await registerWebhook(activeDoc, req, webhook.fields);
registeredWebhooks.push(registeredWebhook);
}
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
res.json({webhooks: registeredWebhooks.map(rw=> {
return {id: rw.webhookId};
})});
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
})
);
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
/**
@deprecated please call to POST /webhooks instead, this endpoint is only for sake of backward compatibility
*/
this._app.post('/api/docs/:docId/tables/:tableId/_subscribe', isOwner, validate(WebhookSubscribe),
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(async (activeDoc, req, res) => {
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
const registeredWebhook = await registerWebhook(activeDoc, req, req.body);
res.json(registeredWebhook);
})
);
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
// Clears all outgoing webhooks in the queue for this document.
this._app.delete('/api/docs/:docId/webhooks/queue', isOwner,
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(async (activeDoc, req, res) => {
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
await activeDoc.clearWebhookQueue();
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
await activeDoc.sendWebhookNotification();
(core) Initial webhooks implementation Summary: See https://grist.quip.com/VKd3ASF99ezD/Outgoing-Webhooks - 2 new DocApi endpoints: _subscribe and _unsubscribe, not meant to be user friendly or publicly documented. _unsubscribe should be given the response from _subscribe in the body, e.g: ``` $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_subscribe" -H "Content-type: application/json" -d '{"url": "https://webhook.site/a916b526-8afc-46e6-aa8f-a625d0d83ec3", "eventTypes": ["add"], "isReadyColumn": "C"}' {"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"} $ curl -X POST -H "Authorization: Bearer 8fd4dc59ecb05ab29ae5a183c03101319b8e6ca9" "http://localhost:8080/api/docs/6WYa23FqWxGNe3AR6DLjCJ/tables/Table2/_unsubscribe" -H "Content-type: application/json" -d '{"unsubscribeKey":"3246f158-55b5-4fc7-baa5-093b75ffa86c","triggerId":2,"webhookId":"853b4bfa-9d39-4639-aa33-7d45354903c0"}' {"success":true} ``` - New DB entity Secret to hold the webhook URL and unsubscribe key - New document metatable _grist_Triggers subscribes to table changes and points to a secret to use for a webhook - New file Triggers.ts processes action summaries and uses the two new tables to send webhooks. - Also went on a bit of a diversion and made a typesafe subclass of TableData for metatables. I think this is essentially good enough for a first diff, to keep the diffs manageable and to talk about the overall structure. Future diffs can add tests and more robustness using redis etc. After this diff I can also start building the Zapier integration privately. Test Plan: Tested manually: see curl commands in summary for an example. Payloads can be seen in https://webhook.site/#!/a916b526-8afc-46e6-aa8f-a625d0d83ec3/0b9fe335-33f7-49fe-b90b-2db5ba53382d/1 . Great site for testing webhooks btw. Reviewers: dsagal, paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D3019
2021-09-22 23:06:23 +00:00
res.json({success: true});
})
);
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
// Remove webhook and trigger created above
this._app.delete('/api/docs/:docId/webhooks/:webhookId', isOwner,
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(removeWebhook)
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
);
/**
@deprecated please call to DEL /webhooks instead, this endpoint is only for sake of backward compatibility
*/
this._app.post('/api/docs/:docId/tables/:tableId/_unsubscribe', canEdit,
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(removeWebhook)
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
);
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
// Update a webhook
this._app.patch(
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
'/api/docs/:docId/webhooks/:webhookId', isOwner, validate(WebhookPatch),
withDocTriggersLock(async (activeDoc, req, res) => {
const docId = activeDoc.docName;
const webhookId = req.params.webhookId;
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
const {fields, url} = await getWebhookSettings(activeDoc, req, webhookId, req.body);
if (fields.enabled === false) {
await activeDoc.triggers.clearSingleWebhookQueue(webhookId);
}
const triggerRowId = activeDoc.triggers.getWebhookTriggerRecord(webhookId).id;
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
// update url in homedb
if (url) {
await this._dbManager.updateWebhookUrl(webhookId, docId, url);
activeDoc.triggers.webhookDeleted(webhookId); // clear cache
}
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
// then update document
if (Object.keys(fields).length) {
await handleSandboxError("_grist_Triggers", [], activeDoc.applyUserActions(
docSessionFromRequest(req),
[['UpdateRecord', "_grist_Triggers", triggerRowId, fields]]));
}
(core) Adds a UI panel for managing webhooks Summary: This adds a UI panel for managing webhooks. Work started by Cyprien Pindat. You can find the UI on a document's settings page. Main changes relative to Cyprien's demo: * Changed behavior of virtual table to be more consistent with the rest of Grist, by factoring out part of the implementation of on-demand tables. * Cell values that would create an error can now be denied and reverted (as for the rest of Grist). * Changes made by other users are integrated in a sane way. * Basic undo/redo support is added using the regular undo/redo stack. * The table list in the drop-down is now updated if schema changes. * Added a notification from back-end when webhook status is updated so constant polling isn't needed to support multi-user operation. * Factored out webhook specific logic from general virtual table support. * Made a bunch of fixes to various broken behavior. * Added tests. The code remains somewhat unpolished, and behavior in the presence of errors is imperfect in general but may be adequate for this case. I assume that we'll soon be lifting the restriction on the set of domains that are supported for webhooks - otherwise we'd want to provide some friendly way to discover that list of supported domains rather than just throwing an error. I don't actually know a lot about how the front-end works - it looks like tables/columns/fields/sections can be safely added if they have string ids that won't collide with bone fide numeric ids from the back end. Sneaky. Contains a migration, so needs an extra reviewer for that. Test Plan: added tests Reviewers: jarek, dsagal Reviewed By: jarek, dsagal Differential Revision: https://phab.getgrist.com/D3856
2023-05-08 22:06:24 +00:00
await activeDoc.sendWebhookNotification();
res.json({success: true});
})
);
// Clears a single webhook in the queue for this document.
this._app.delete('/api/docs/:docId/webhooks/queue/:webhookId', isOwner,
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(async (activeDoc, req, res) => {
const webhookId = req.params.webhookId;
await activeDoc.clearSingleWebhookQueue(webhookId);
await activeDoc.sendWebhookNotification();
res.json({success: true});
})
);
// Lists all webhooks and their current status in the document.
this._app.get('/api/docs/:docId/webhooks', isOwner,
(core) Remove DB transaction from webhook update, add mutex to all webhook endpoints Summary: This removes problematic code that was holding a HomeDB transaction while applying user actions which could hang indefinitely, especially if the webhook queue is full as in https://grist.slack.com/archives/C05DBJ6LA1F/p1698159750945949. The discussion about adding this code is here: https://phab.getgrist.com/D3821#inline-45054 The initial motivation was to roll back HomeDB changes if something went wrong while applying user actions, to avoid saving only part of the changes the user requested. I think it's actually fine to just allow such a partial save to happen - I don't see anything particularly undesirable about keeping an update to the webhook URL if other updates requested by the user didn't also get applied, as the fields don't affect each other. The comment approving the transaction approach said "so we shouldn't end up leave the transaction hanging around too long" which has been falsified. It looks like there was also some desire to prevent a mess caused by multiple simultaneous calls to this endpoint, which the transaction may have helped with a little, but didn't really seem like a solution. Comments in `Triggers.ts` also mention fears of race conditions when clearing (some of) the queue and the need for some locking. So I wrapped all webhook-related endpoints in a simple `Mutex` held by the `ActiveDoc` to prevent simultaneous changes. I *think* this is a good thing. These endpoints shouldn't be called frequently enough to create a performance issue, and this shouldn't affect actually sending webhook events when records are added/updated. And it does seem like interleaving calls to these endpoints could cause very weird problems. Test Plan: Nothing yet, I'd like to hear if others think this is sensible. Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D4111
2023-11-14 13:46:33 +00:00
withDocTriggersLock(async (activeDoc, req, res) => {
res.json(await activeDoc.webhooksSummary());
})
);
// Reload a document forcibly (in fact this closes the doc, it will be automatically
// reopened on use).
this._app.post('/api/docs/:docId/force-reload', canEdit, throttled(async (req, res) => {
const activeDoc = await this._getActiveDoc(req);
await activeDoc.reloadDoc();
res.json(null);
}));
this._app.post('/api/docs/:docId/recover', canEdit, throttled(async (req, res) => {
const recoveryModeRaw = req.body.recoveryMode;
const recoveryMode = (typeof recoveryModeRaw === 'boolean') ? recoveryModeRaw : undefined;
if (!await this._isOwner(req)) { throw new Error('Only owners can control recovery mode'); }
this._docManager.setRecovery(getDocId(req), recoveryMode ?? true);
const activeDoc = await this._docManager.fetchDoc(docSessionFromRequest(req), getDocId(req), recoveryMode);
res.json({
recoveryMode: activeDoc.recoveryMode
});
}));
// DELETE /api/docs/:docId
// Delete the specified doc.
this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => {
await this._removeDoc(req, res, true);
}));
// POST /api/docs/:docId/remove
// Soft-delete the specified doc. If query parameter "permanent" is set,
// delete permanently.
this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => {
await this._removeDoc(req, res, isParameterOn(req.query.permanent));
}));
this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
const {snapshots} = await activeDoc.getSnapshots(docSession, isAffirmative(req.query.raw));
res.json({snapshots});
}));
this._app.get('/api/docs/:docId/usersForViewAs', isOwner, withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
res.json(await activeDoc.getUsersForViewAs(docSession));
}));
this._app.post('/api/docs/:docId/snapshots/remove', isOwner, withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
const snapshotIds = req.body.snapshotIds as string[];
if (snapshotIds) {
await activeDoc.removeSnapshots(docSession, snapshotIds);
res.json({snapshotIds});
return;
}
if (req.body.select === 'unlisted') {
// Remove any snapshots not listed in inventory. Ideally, there should be no
// snapshots, and this undocumented feature is just for fixing up problems.
const full = (await activeDoc.getSnapshots(docSession, true)).snapshots.map(s => s.snapshotId);
const listed = new Set((await activeDoc.getSnapshots(docSession)).snapshots.map(s => s.snapshotId));
const unlisted = full.filter(snapshotId => !listed.has(snapshotId));
await activeDoc.removeSnapshots(docSession, unlisted);
res.json({snapshotIds: unlisted});
return;
}
if (req.body.select === 'past') {
// Remove all but the latest snapshot. Useful for sanitizing history if something
// bad snuck into previous snapshots and they are not valuable to preserve.
const past = (await activeDoc.getSnapshots(docSession, true)).snapshots.map(s => s.snapshotId);
past.shift(); // remove current version.
await activeDoc.removeSnapshots(docSession, past);
res.json({snapshotIds: past});
return;
}
throw new Error('please specify snapshotIds to remove');
}));
this._app.post('/api/docs/:docId/flush', canEdit, throttled(async (req, res) => {
const activeDocPromise = this._getActiveDocIfAvailable(req);
if (!activeDocPromise) {
// Only need to flush if doc is actually open.
res.json(false);
return;
}
const activeDoc = await activeDocPromise;
await activeDoc.flushDoc();
res.json(true);
}));
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
// Administrative endpoint, that checks if a document is in the expected group,
// and frees it for reassignment if not. Has no effect if document is in the
// expected group. Does not require specific rights. Returns true if the document
// is freed up for reassignment, otherwise false.
//
// Optionally accepts a `group` query param for updating the document's group prior
// to (possible) reassignment. A blank string unsets the current group, if any.
// (Requires a special permit.)
this._app.post('/api/docs/:docId/assign', canEdit, throttled(async (req, res) => {
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
const docId = getDocId(req);
const group = optStringParam(req.query.group, 'group');
if (group !== undefined && req.specialPermit?.action === 'assign-doc') {
if (group.trim() === '') {
await this._docWorkerMap.removeDocGroup(docId);
} else {
await this._docWorkerMap.updateDocGroup(docId, group);
}
}
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
const status = await this._docWorkerMap.getDocWorker(docId);
if (!status) { res.json(false); return; }
const workerGroup = await this._docWorkerMap.getWorkerGroup(status.docWorker.id);
const docGroup = await this._docWorkerMap.getDocGroup(docId);
if (docGroup === workerGroup) { res.json(false); return; }
const activeDoc = await this._getActiveDoc(req);
await activeDoc.flushDoc();
// flushDoc terminates once there's no pending operation on the document.
// There could still be async operations in progress. We mute their effect,
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
// as if they never happened.
activeDoc.docClients.interruptAllClients();
activeDoc.setMuted();
await activeDoc.shutdown();
await this._docWorkerMap.releaseAssignment(status.docWorker.id, docId);
res.json(true);
}));
// This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it
// starts with to become muted.
this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => {
const docSession = docSessionFromRequest(req);
const activeDoc = await this._getActiveDoc(req);
const options: DocReplacementOptions = {};
if (req.body.sourceDocId) {
options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId));
// Make sure that if we wanted to download the full source, we would be allowed.
2024-03-25 15:01:48 +00:00
const homeUrl = this._grist.getHomeInternalUrl(req, `/api/docs/${options.sourceDocId}/download?dryrun=1`);
const result = await fetch(homeUrl, {
method: 'GET',
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
if (result.status !== 200) {
const jsonResult = await result.json();
throw new ApiError(jsonResult.error, result.status);
}
// We should make sure the source document has flushed recently.
// It may not be served by the same worker, so work through the api.
2024-03-25 15:01:48 +00:00
await fetch(this._grist.getHomeInternalUrl(req, `/api/docs/${options.sourceDocId}/flush`), {
method: 'POST',
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
if (req.body.resetTutorialMetadata) {
const scope = getDocScope(req);
const tutorialTrunkId = options.sourceDocId;
await this._dbManager.connection.transaction(async (manager) => {
// Fetch the tutorial trunk doc so we can replace the tutorial doc's name.
const tutorialTrunk = await this._dbManager.getDoc({...scope, urlId: tutorialTrunkId}, manager);
await this._dbManager.updateDocument(
scope,
{
name: tutorialTrunk.name,
options: {
tutorial: {
...tutorialTrunk.options?.tutorial,
// For now, the only state we need to reset is the slide position.
lastSlideIndex: 0,
},
},
},
manager
);
});
const {forkId} = parseUrlId(scope.urlId);
activeDoc.logTelemetryEvent(docSession, 'tutorialRestarted', {
full: {
tutorialForkIdDigest: forkId,
tutorialTrunkIdDigest: tutorialTrunkId,
},
});
}
}
if (req.body.snapshotId) {
options.snapshotId = String(req.body.snapshotId);
}
await activeDoc.replace(docSession, options);
res.json(null);
}));
this._app.get('/api/docs/:docId/states', canView, withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
res.json(await this._getStates(docSession, activeDoc));
}));
this._app.post('/api/docs/:docId/states/remove', isOwner, withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
const keep = integerParam(req.body.keep, 'keep');
res.json(await activeDoc.deleteActions(docSession, keep));
}));
this._app.get('/api/docs/:docId/compare/:docId2', canView, withDoc(async (activeDoc, req, res) => {
const showDetails = isAffirmative(req.query.detail);
const docSession = docSessionFromRequest(req);
const {states} = await this._getStates(docSession, activeDoc);
2024-03-25 15:01:48 +00:00
const ref = await fetch(this._grist.getHomeInternalUrl(req, `/api/docs/${req.params.docId2}/states`), {
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
const states2: DocState[] = (await ref.json()).states;
const left = states[0];
const right = states2[0];
if (!left || !right) {
// This should not arise unless there's a bug.
throw new Error('document with no history');
}
const rightHashes = new Set(states2.map(state => state.h));
const parent = states.find(state => rightHashes.has(state.h )) || null;
const leftChanged = parent && parent.h !== left.h;
const rightChanged = parent && parent.h !== right.h;
const summary = leftChanged ? (rightChanged ? 'both' : 'left') :
(rightChanged ? 'right' : (parent ? 'same' : 'unrelated'));
const comparison: DocStateComparison = {
left, right, parent, summary
};
if (showDetails && parent) {
// Calculate changes from the parent to the current version of this document.
const leftChanges = (await this._getChanges(docSession, activeDoc, states, parent.h,
'HEAD')).details!.rightChanges;
// Calculate changes from the (common) parent to the current version of the other document.
const url = `/api/docs/${req.params.docId2}/compare?left=${parent.h}`;
2024-03-25 15:01:48 +00:00
const rightChangesReq = await fetch(this._grist.getHomeInternalUrl(req, url), {
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
const rightChanges = (await rightChangesReq.json()).details!.rightChanges;
// Add the left and right changes as details to the result.
comparison.details = { leftChanges, rightChanges };
}
res.json(comparison);
}));
// Give details about what changed between two versions of a document.
this._app.get('/api/docs/:docId/compare', canView, withDoc(async (activeDoc, req, res) => {
// This could be a relatively slow operation if actions are large.
const left = stringParam(req.query.left || 'HEAD', 'left');
const right = stringParam(req.query.right || 'HEAD', 'right');
const docSession = docSessionFromRequest(req);
const {states} = await this._getStates(docSession, activeDoc);
res.json(await this._getChanges(docSession, activeDoc, states, left, right));
}));
// Do an import targeted at a specific workspace. Although the URL fits ApiServer, this
// endpoint is handled only by DocWorker, so is handled here. (Note: this does not handle
// actual file uploads, so no worries here about large request bodies.)
this._app.post('/api/workspaces/:wid/import', expressWrap(async (req, res) => {
const mreq = req as RequestWithLogin;
const userId = getUserId(req);
const wsId = integerParam(req.params.wid, 'wid');
const uploadId = integerParam(req.body.uploadId, 'uploadId');
const result = await this._docManager.importDocToWorkspace(mreq, {
userId,
uploadId,
workspaceId: wsId,
browserSettings: req.body.browserSettings,
telemetryMetadata: {
limited: {
isImport: true,
sourceDocIdDigest: undefined,
},
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
},
});
this._logCreatedFileImportDocTelemetryEvent(req, {
full: {
docIdDigest: result.id,
},
});
res.json(result);
}));
this._app.get('/api/docs/:docId/download/table-schema', canView, withDoc(async (activeDoc, req, res) => {
const doc = await this._dbManager.getDoc(req);
const options = await this._getDownloadOptions(req, doc);
const tableSchema = await collectTableSchemaInFrictionlessFormat(activeDoc, req, options);
const apiPath = await this._grist.getResourceUrl(doc, 'api');
const query = new URLSearchParams(req.query as {[key: string]: string});
const tableSchemaPath = `${apiPath}/download/csv?${query.toString()}`;
res.send({
format: "csv",
mediatype: "text/csv",
encoding: "utf-8",
path: tableSchemaPath,
dialect: {
delimiter: ",",
doubleQuote: true,
},
...tableSchema,
});
}));
this._app.get('/api/docs/:docId/download/csv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadDSV(activeDoc, req, res, {...options, delimiter: ','});
}));
this._app.get('/api/docs/:docId/download/tsv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadDSV(activeDoc, req, res, {...options, delimiter: '\t'});
}));
this._app.get('/api/docs/:docId/download/dsv', canView, withDoc(async (activeDoc, req, res) => {
const options = await this._getDownloadOptions(req);
await downloadDSV(activeDoc, req, res, {...options, delimiter: '💩'});
}));
this._app.get('/api/docs/:docId/download/xlsx', canView, withDoc(async (activeDoc, req, res) => {
const options: DownloadOptions = (!_.isEmpty(req.query) && !_.isEqual(Object.keys(req.query), ["title"]))
? await this._getDownloadOptions(req)
: {
filename: await this._getDownloadFilename(req),
tableId: '',
viewSectionId: undefined,
filters: [],
sortOrder: [],
header: 'label'
};
await downloadXLSX(activeDoc, req, res, options);
}));
this._app.get('/api/docs/:docId/send-to-drive', canView, decodeGoogleToken, withDoc(exportToDrive));
/**
* Send a request to the formula assistant to get completions for a formula. Increases the
* usage of the formula assistant for the billing account in case of success.
*/
this._app.post('/api/docs/:docId/assistant', canView, checkLimit('assistant'),
withDoc(async (activeDoc, req, res) => {
const docSession = docSessionFromRequest(req);
const request = req.body;
const result = await sendForCompletion(docSession, activeDoc, request);
const limit = await this._increaseLimit('assistant', req);
res.json({
...result,
limit: !limit ? undefined : {
usage: limit.usage,
limit: limit.limit,
},
});
})
);
/**
* Create a document.
*
* When an upload is included, it is imported as the initial state of the document.
*
* When a source document id is included, its structure and (optionally) data is
* included in the new document.
*
* In all other cases, the document is left empty.
*
* If a workspace id is included, the document will be saved there instead of
* being left "unsaved".
*
* Returns the id of the created document.
*
* TODO: unify this with the other document creation and import endpoints.
*/
this._app.post('/api/docs', checkAnonymousCreation, expressWrap(async (req, res) => {
const mreq = req as RequestWithLogin;
const userId = getUserId(req);
let uploadId: number|undefined;
let parameters: {[key: string]: any};
if (req.is('multipart/form-data')) {
const formResult = await handleOptionalUpload(req, res);
if (formResult.upload) {
uploadId = formResult.upload.uploadId;
}
parameters = formResult.parameters || {};
} else {
parameters = req.body;
}
const sourceDocumentId = optStringParam(parameters.sourceDocumentId, 'sourceDocumentId');
const workspaceId = optIntegerParam(parameters.workspaceId, 'workspaceId');
const browserSettings: BrowserSettings = {};
if (parameters.timezone) { browserSettings.timezone = parameters.timezone; }
browserSettings.locale = localeFromRequest(req);
let docId: string;
if (sourceDocumentId !== undefined) {
docId = await this._copyDocToWorkspace(req, {
userId,
sourceDocumentId,
workspaceId: integerParam(parameters.workspaceId, 'workspaceId'),
documentName: stringParam(parameters.documentName, 'documentName'),
asTemplate: optBooleanParam(parameters.asTemplate, 'asTemplate'),
});
} else if (uploadId !== undefined) {
const result = await this._docManager.importDocToWorkspace(mreq, {
userId,
uploadId,
documentName: optStringParam(parameters.documentName, 'documentName'),
workspaceId,
browserSettings,
telemetryMetadata: {
limited: {
isImport: true,
sourceDocIdDigest: undefined,
},
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
},
});
docId = result.id;
this._logCreatedFileImportDocTelemetryEvent(req, {
full: {
docIdDigest: docId,
},
});
} else if (workspaceId !== undefined) {
docId = await this._createNewSavedDoc(req, {
workspaceId: workspaceId,
documentName: optStringParam(parameters.documentName, 'documentName'),
});
} else {
docId = await this._createNewUnsavedDoc(req, {
userId,
browserSettings,
});
}
return res.status(200).json(docId);
}));
/**
* Get the specified view section's form data.
*/
this._app.get('/api/docs/:docId/forms/:vsId', canView,
withDoc(async (activeDoc, req, res) => {
if (!activeDoc.docData) {
throw new ApiError('DocData not available', 500);
}
const sectionId = integerParam(req.params.vsId, 'vsId');
const docSession = docSessionFromRequest(req);
const linkId = getDocSessionShare(docSession);
if (linkId) {
/* If accessed via a share, the share's `linkId` will be present and
* we'll need to check that the form is in fact published, and that the
* share key is associated with the form, before granting access to the
* form. */
this._assertIsPublishedForm({
docData: activeDoc.docData,
linkId,
sectionId,
});
}
const Views_section = activeDoc.docData.getMetaTable('_grist_Views_section');
const section = Views_section.getRecord(sectionId);
if (!section) {
throw new ApiError('Form not found', 404, {code: 'FormNotFound'});
}
const Views_section_field = activeDoc.docData.getMetaTable('_grist_Views_section_field');
const Tables_column = activeDoc.docData.getMetaTable('_grist_Tables_column');
const fields = Views_section_field
.filterRecords({parentId: sectionId})
.filter(f => {
const col = Tables_column.getRecord(f.colRef);
// Formulas and attachments are currently unsupported.
return col && !(col.isFormula && col.formula) && col.type !== 'Attachments';
});
let {layoutSpec: formLayoutSpec} = section;
if (!formLayoutSpec) {
formLayoutSpec = JSON.stringify({
type: 'Layout',
children: [
{type: 'Label'},
{type: 'Label'},
{
type: 'Section',
children: [
{type: 'Label'},
{type: 'Label'},
...fields.slice(0, INITIAL_FIELDS_COUNT).map(f => ({
type: 'Field',
leaf: f.id,
})),
],
},
],
});
}
// Cache the table reads based on tableId. We are caching only the promise, not the result.
const table = _.memoize((tableId: string) =>
readTable(req, activeDoc, tableId, {}, {}).then(r => asRecords(r, {includeId: true})));
const getTableValues = async (tableId: string, colId: string) => {
const records = await table(tableId);
return records.map(r => [r.id as number, r.fields[colId]] as const);
};
const Tables = activeDoc.docData.getMetaTable('_grist_Tables');
const getRefTableValues = async (col: MetaRowRecord<'_grist_Tables_column'>) => {
const refTableId = getReferencedTableId(col.type);
let refColId: string;
if (col.visibleCol) {
const refCol = Tables_column.getRecord(col.visibleCol);
if (!refCol) { return []; }
refColId = refCol.colId as string;
} else {
refColId = 'id';
}
if (!refTableId || typeof refTableId !== 'string' || !refColId) { return []; }
const values = await getTableValues(refTableId, refColId);
return values.filter(([_id, value]) => !isBlankValue(value));
};
const formFields = await Promise.all(fields.map(async (field) => {
const col = Tables_column.getRecord(field.colRef);
if (!col) { throw new Error(`Column ${field.colRef} not found`); }
const fieldOptions = safeJsonParse(field.widgetOptions as string, {});
const colOptions = safeJsonParse(col.widgetOptions as string, {});
const options = {...colOptions, ...fieldOptions};
const type = extractTypeFromColType(col.type as string);
const colId = col.colId as string;
return [field.id, {
colId,
description: fieldOptions.description || col.description,
question: options.question || col.label || colId,
options,
type,
refValues: isFullReferencingType(col.type) ? await getRefTableValues(col) : null,
}] as const;
}));
const formFieldsById = Object.fromEntries(formFields);
const getTableName = () => {
const rawSectionRef = Tables.getRecord(section.tableRef)?.rawViewSectionRef;
if (!rawSectionRef) { return null; }
const rawSection = activeDoc.docData!
.getMetaTable('_grist_Views_section')
.getRecord(rawSectionRef);
return rawSection?.title ?? null;
};
const formTableId = await getRealTableId(String(section.tableRef), {activeDoc, req});
const formTitle = section.title || getTableName() || formTableId;
this._grist.getTelemetry().logEvent(req, 'visitedForm', {
full: {
docIdDigest: activeDoc.docName,
userId: req.userId,
altSessionId: req.altSessionId,
},
});
res.status(200).json({
formFieldsById,
formLayoutSpec,
formTableId,
formTitle,
});
})
);
// GET /api/docs/:docId/timings
// Checks if timing is on for the document.
this._app.get('/api/docs/:docId/timing', isOwner, withDoc(async (activeDoc, req, res) => {
if (!activeDoc.isTimingOn) {
res.json({status: 'disabled'});
} else {
const timing = await activeDoc.getTimings();
const status = timing ? 'active' : 'pending';
res.json({status, timing});
}
}));
// POST /api/docs/:docId/timings/start
// Start a timing for the document.
this._app.post('/api/docs/:docId/timing/start', isOwner, withDoc(async (activeDoc, req, res) => {
if (activeDoc.isTimingOn) {
res.status(400).json({error:`Timing already started for ${activeDoc.docName}`});
return;
}
// isTimingOn flag is switched synchronously.
await activeDoc.startTiming();
res.sendStatus(200);
}));
// POST /api/docs/:docId/timings/stop
// Stop a timing for the document.
this._app.post('/api/docs/:docId/timing/stop', isOwner, withDoc(async (activeDoc, req, res) => {
if (!activeDoc.isTimingOn) {
res.status(400).json({error:`Timing not started for ${activeDoc.docName}`});
return;
}
res.json(await activeDoc.stopTiming());
}));
}
/**
* Throws if the specified section is not a published form.
*/
private _assertIsPublishedForm(params: {
docData: DocData,
linkId: string,
sectionId: number,
}) {
const {docData, linkId, sectionId} = params;
// Check that the request is for a valid section in the document.
const sections = docData.getMetaTable('_grist_Views_section');
const section = sections.getRecord(sectionId);
if (!section) { throw new ApiError('Form not found', 404, {code: 'FormNotFound'}); }
// Check that the section is for a form.
const sectionShareOptions = safeJsonParse(section.shareOptions, {});
if (!sectionShareOptions.form) { throw new ApiError('Form not found', 404, {code: 'FormNotFound'}); }
// Check that the form is associated with a share.
const viewId = section.parentId;
const pages = docData.getMetaTable('_grist_Pages');
const page = pages.getRecords().find(p => p.viewRef === viewId);
if (!page) { throw new ApiError('Form not found', 404, {code: 'FormNotFound'}); }
const shares = docData.getMetaTable('_grist_Shares');
const share = shares.getRecord(page.shareRef);
if (!share) { throw new ApiError('Form not found', 404, {code: 'FormNotFound'}); }
// Check that the share's link id matches the expected link id.
if (share.linkId !== linkId) { throw new ApiError('Form not found', 404, {code: 'FormNotFound'}); }
// Finally, check that both the section and share are published.
if (!sectionShareOptions.publish || !safeJsonParse(share.options, {})?.publish) {
throw new ApiError('Form not published', 404, {code: 'FormNotPublished'});
}
}
private async _copyDocToWorkspace(req: Request, options: {
userId: number,
sourceDocumentId: string,
workspaceId: number,
documentName: string,
asTemplate?: boolean,
}): Promise<string> {
const mreq = req as RequestWithLogin;
const {userId, sourceDocumentId, workspaceId, documentName, asTemplate = false} = options;
// First, upload a copy of the document.
let uploadResult;
try {
const accessId = makeAccessId(req, getAuthorizedUserId(req));
uploadResult = await fetchDoc(this._grist, sourceDocumentId, req, accessId, asTemplate);
globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, `${documentName}.grist`);
} catch (err) {
if ((err as ApiError).status === 403) {
throw new ApiError('Insufficient access to document to copy it entirely', 403);
}
throw err;
}
// Then, import the copy to the workspace.
const result = await this._docManager.importDocToWorkspace(mreq, {
userId,
uploadId: uploadResult.uploadId,
documentName,
workspaceId,
telemetryMetadata: {
limited: {
isImport: false,
sourceDocIdDigest: sourceDocumentId,
},
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
},
});
const sourceDocument = await this._dbManager.getRawDocById(sourceDocumentId);
const isTemplateCopy = sourceDocument.type === 'template';
if (isTemplateCopy) {
this._grist.getTelemetry().logEvent(mreq, 'copiedTemplate', {
full: {
templateId: parseUrlId(sourceDocument.urlId || sourceDocument.id).trunkId,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
});
}
this._grist.getTelemetry().logEvent(
mreq,
`createdDoc-${isTemplateCopy ? 'CopyTemplate' : 'CopyDoc'}`,
{
full: {
docIdDigest: result.id,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
}
);
return result.id;
}
private async _createNewSavedDoc(req: Request, options: {
workspaceId: number,
documentName?: string,
}): Promise<string> {
const {documentName, workspaceId} = options;
const {status, data, errMessage} = await this._dbManager.addDocument(getScope(req), workspaceId, {
name: documentName ?? 'Untitled document',
});
const docId = data!;
if (status !== 200) {
throw new ApiError(errMessage || 'unable to create document', status);
}
this._logDocumentCreatedTelemetryEvent(req, {
limited: {
docIdDigest: docId,
sourceDocIdDigest: undefined,
isImport: false,
fileType: undefined,
isSaved: true,
},
});
this._logCreatedEmptyDocTelemetryEvent(req, {
full: {
docIdDigest: docId,
},
});
return docId;
}
private async _createNewUnsavedDoc(req: Request, options: {
userId: number,
browserSettings?: BrowserSettings,
}): Promise<string> {
const {userId, browserSettings} = options;
const isAnonymous = isAnonymousUser(req);
const result = makeForkIds({
userId,
isAnonymous,
trunkDocId: NEW_DOCUMENT_CODE,
trunkUrlId: NEW_DOCUMENT_CODE,
});
const docId = result.docId;
await this._docManager.createNamedDoc(
makeExceptionalDocSession('nascent', {
req: req as RequestWithLogin,
browserSettings,
}),
docId
);
this._logDocumentCreatedTelemetryEvent(req, {
limited: {
docIdDigest: docId,
sourceDocIdDigest: undefined,
isImport: false,
fileType: undefined,
isSaved: false,
},
});
this._logCreatedEmptyDocTelemetryEvent(req, {
full: {
docIdDigest: docId,
},
});
return docId;
}
private _logDocumentCreatedTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
const mreq = req as RequestWithLogin;
this._grist.getTelemetry().logEvent(mreq, 'documentCreated', _.merge({
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
}, metadata));
}
private _logCreatedEmptyDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
this._logCreatedDocTelemetryEvent(req, 'createdDoc-Empty', metadata);
}
private _logCreatedFileImportDocTelemetryEvent(req: Request, metadata: TelemetryMetadataByLevel) {
this._logCreatedDocTelemetryEvent(req, 'createdDoc-FileImport', metadata);
}
private _logCreatedDocTelemetryEvent(
req: Request,
event: 'createdDoc-Empty' | 'createdDoc-FileImport',
metadata: TelemetryMetadataByLevel,
) {
const mreq = req as RequestWithLogin;
this._grist.getTelemetry().logEvent(mreq, event, _.merge({
full: {
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
}, metadata));
}
/**
* Check for read access to the given document, and return its
* canonical docId. Throws error if read access not available.
* This method is used for documents that are not the main document
* associated with the request, but are rather an extra source to be
* read from, so the access information is not cached in the
* request.
*/
private async _confirmDocIdForRead(req: Request, urlId: string): Promise<string> {
const docAuth = await makeDocAuthResult(this._dbManager.getDoc({...getScope(req), urlId}));
if (docAuth.error) { throw docAuth.error; }
assertAccess('viewers', docAuth);
return docAuth.docId!;
}
private async _getDownloadFilename(req: Request, tableId?: string, optDoc?: Document): Promise<string> {
let filename = optStringParam(req.query.title, 'title');
if (!filename) {
// Query DB for doc metadata to get the doc data.
const doc = optDoc || await this._dbManager.getDoc(req);
const docTitle = doc.name;
const suffix = tableId ? (tableId === docTitle ? '' : `-${tableId}`) : '';
filename = docTitle + suffix || 'document';
}
return filename;
}
private async _getDownloadOptions(req: Request, doc?: Document): Promise<DownloadOptions> {
const params = parseExportParameters(req);
(core) API reworked to use POST to create webhook and DELET to remove it Summary: introduces POST /api/docs/{docId}/webhooks and DELETE /api/docs/{docId}/webhooks/{webhookId} on place of old _subscribe and _unsubscribe endpoints. Remove checking for unsubscribeKey while deleting webhook - only owner can delete webhook using DELETE endpoint. subscription key is still needed for _unsubscribe endpoint. old _unsubscribe and _subscribe endpoints are still active and work as before - no changes there. Posting schema: ``` POST /api/docs/[docId]/webhooks ``` Request Body: ``` { "webhooks": [ { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb1", "eventTypes": [ "add", "update" ], "enabled": true, "name": "WebhookName", "memo": "just a text", "tableId": "Table1" } }, { "fields": { "url": "https://webhook.site/3bd02246-f122-445e-ba7f-bf5ea5bb6eb2", "eventTypes": [ "add", ], "enabled": true, "name": "OtherWebhookName", "memo": "just a text", "tableId": "Table1" } } ] } ``` Expected response: WebhookId for each webhook posted: ``` { "webhooks": [ { "id": "85c77108-f1e1-4217-a50d-acd1c5996da2" }, { "id": "d87a6402-cfd7-4822-878c-657308fcc8c3" } ] } ``` Deleting webhooks: ``` DELETE api/docs/[docId]/webhooks/[webhookId] ``` there is no payload in DELETE request. Therefore only one webhook can be deleted at once Response: ``` { "success": true } ``` Test Plan: Old unit test improved to handle new endpoints, and one more added to check if endpoints are in fact created/removed Reviewers: alexmojaki Reviewed By: alexmojaki Subscribers: paulfitz, alexmojaki Differential Revision: https://phab.getgrist.com/D3916
2023-07-14 10:05:22 +00:00
return {
...params,
filename: await this._getDownloadFilename(req, params.tableId, doc),
2022-12-27 18:35:03 +00:00
};
}
private _getActiveDoc(req: RequestWithLogin): Promise<ActiveDoc> {
return this._docManager.fetchDoc(docSessionFromRequest(req), getDocId(req));
}
private _getActiveDocIfAvailable(req: RequestWithLogin): Promise<ActiveDoc>|undefined {
return this._docManager.getActiveDoc(getDocId(req));
}
/**
* Middleware to track the number of requests outstanding on each document, and to
* throw an exception when the maximum number of requests are already outstanding.
* Also throws an exception if too many requests (based on the user's product plan)
* have been made today for this document.
* Access to a document must already have been authorized.
*/
private _apiThrottle(callback: (req: RequestWithLogin,
resp: Response,
next: NextFunction) => void | Promise<void>): RequestHandler {
return async (req, res, next) => {
const docId = getDocId(req);
try {
const count = this._currentUsage.get(docId) || 0;
this._currentUsage.set(docId, count + 1);
if (count + 1 > MAX_PARALLEL_REQUESTS_PER_DOC) {
throw new ApiError(`Too many backlogged requests for document ${docId} - ` +
`try again later?`, 429);
}
if (await this._checkDailyDocApiUsage(req, docId)) {
throw new ApiError(`Exceeded daily limit for document ${docId}`, 429);
}
await callback(req as RequestWithLogin, res, next);
} catch (err) {
next(err);
} finally {
const count = this._currentUsage.get(docId);
if (count) {
if (count === 1) {
this._currentUsage.delete(docId);
} else {
this._currentUsage.set(docId, count - 1);
}
}
}
};
}
/**
* Usually returns true if too many requests (based on the user's product plan)
* have been made today for this document and the request should be rejected.
* Access to a document must already have been authorized.
* This is called frequently so it uses caches to check quickly in the common case,
* which allows a few ways for users to exceed the limit slightly if the timing works out,
* but these should be acceptable.
*/
private async _checkDailyDocApiUsage(req: Request, docId: string): Promise<boolean> {
// Use the cached doc to avoid a database call.
// This leaves a small window (currently 5 seconds) for the user to bypass this limit after downgrading,
// or to be wrongly rejected after upgrading.
const doc = (req as RequestWithLogin).docAuth!.cachedDoc!;
const max = doc.workspace.org.billingAccount?.product.features.baseMaxApiUnitsPerDocumentPerDay;
if (!max) {
// This doc has no associated product (happens to new unsaved docs)
// or the product has no API limit. Allow the request through.
return false;
}
// Check the counts in the dailyUsage cache rather than waiting for redis.
// The cache will not have counts if this is the first request for this document served by this worker process
// or if so many other documents have been served since then that the keys were evicted from the LRU cache.
// Both scenarios are temporary and unlikely when usage has been exceeded.
// Note that if the limits are exceeded then `keys` below will be undefined,
// otherwise it will be an array of three keys corresponding to a day, hour, and minute.
const m = moment.utc();
const keys = getDocApiUsageKeysToIncr(docId, this._dailyUsage, max, m);
if (!keys) {
// The limit has been exceeded, reject the request.
return true;
}
// If Redis isn't configured, this is as far as we can go with checks.
if (!process.env.REDIS_URL) { return false; }
// Note the increased API usage on redis and in our local cache.
// Update redis in the background so that the rest of the request can continue without waiting for redis.
const cli = this._docWorkerMap.getRedisClient();
if (!cli) { throw new Error('redis unexpectedly not available'); }
const multi = cli.multi();
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
// Incrementing the local count immediately prevents many requests from being squeezed through every minute
// before counts are received from redis.
// But this cache is not 100% reliable and the count from redis may be higher.
this._dailyUsage.set(key, (this._dailyUsage.get(key) ?? 0) + 1);
const period = docApiUsagePeriods[i];
// Expire the key just so that it cleans itself up and saves memory on redis.
// Expire after two periods to handle 'next' buckets.
const expiry = 2 * 24 * 60 * 60 / period.periodsPerDay;
multi.incr(key).expire(key, expiry);
}
multi.execAsync().then(result => {
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
const newCount = Number(result![i * 2]); // incrs are at even positions, expires at odd positions
// Theoretically this could be overwritten by a lower count that was requested earlier
// but somehow arrived after.
// This doesn't really matter, and the count on redis will still increase reliably.
this._dailyUsage.set(key, newCount);
}
}).catch(e => console.error(`Error tracking API usage for doc ${docId}`, e));
// Allow the request through.
return false;
}
/**
* Creates a middleware that checks the current usage of a limit and rejects the request if it is exceeded.
*/
private async _checkLimit(limit: LimitType, req: Request, res: Response, next: NextFunction) {
await this._dbManager.increaseUsage(getDocScope(req), limit, {dryRun: true, delta: 1});
next();
}
/**
* Increases the current usage of a limit by 1.
*/
private async _increaseLimit(limit: LimitType, req: Request) {
return await this._dbManager.increaseUsage(getDocScope(req), limit, {delta: 1});
}
/**
* Disallow document creation for anonymous users if GRIST_ANONYMOUS_CREATION is set to false.
*/
private async _checkAnonymousCreation(req: Request, res: Response, next: NextFunction) {
const isAnonPlayground = isAffirmative(process.env.GRIST_ANON_PLAYGROUND ?? true);
if (isAnonymousUser(req) && !isAnonPlayground) {
throw new ApiError('Anonymous document creation is disabled', 403);
}
next();
}
private async _assertAccess(role: 'viewers'|'editors'|'owners'|null, allowRemoved: boolean,
req: Request, res: Response, next: NextFunction) {
const scope = getDocScope(req);
allowRemoved = scope.showAll || scope.showRemoved || allowRemoved;
const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, this._grist, scope.urlId);
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
if (role) { assertAccess(role, docAuth, {allowRemoved}); }
next();
}
/**
* Check if user is an owner of the document.
* If acceptTrunkForSnapshot is set, being an owner of the trunk of the document (if it is a snapshot)
* is sufficient. Uses cachedDoc, which could be stale if access has changed recently.
*/
private async _isOwner(req: Request, options?: { acceptTrunkForSnapshot?: boolean }) {
const scope = getDocScope(req);
const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, this._grist, scope.urlId);
if (docAuth.access === 'owners') {
return true;
}
if (options?.acceptTrunkForSnapshot && docAuth.cachedDoc?.trunkAccess === 'owners') {
const parts = parseUrlId(scope.urlId);
if (parts.snapshotId) { return true; }
}
return false;
}
// Helper to generate a 503 if the ActiveDoc has been muted.
private _checkForMute(activeDoc: ActiveDoc|undefined) {
if (activeDoc && activeDoc.muted) {
throw new ApiError('Document in flux - try again later', 503);
}
}
/**
* Throws an error if, during processing, the ActiveDoc becomes "muted". Also replaces any
* other error that may have occurred if the ActiveDoc becomes "muted", since the document
* shutting down during processing may have caused a variety of errors.
*
* Expects to be called within a handler that catches exceptions.
*/
private _requireActiveDoc(callback: WithDocHandler): RequestHandler {
return async (req, res) => {
let activeDoc: ActiveDoc|undefined;
try {
activeDoc = await this._getActiveDoc(req as RequestWithLogin);
await callback(activeDoc, req as RequestWithLogin, res);
if (!res.headersSent) { this._checkForMute(activeDoc); }
} catch (err) {
this._checkForMute(activeDoc);
throw err;
}
};
}
private async _getStates(docSession: OptDocSession, activeDoc: ActiveDoc): Promise<DocStates> {
const states = await activeDoc.getRecentStates(docSession);
return {
states,
};
}
/**
*
* Calculate changes between two document versions identified by leftHash and rightHash.
* If rightHash is the latest version of the document, the ActionSummary for it will
* contain a copy of updated and added rows.
*
* Currently will fail if leftHash is not an ancestor of rightHash (this restriction could
* be lifted, but is adequate for now).
*
*/
private async _getChanges(docSession: OptDocSession, activeDoc: ActiveDoc, states: DocState[],
leftHash: string, rightHash: string): Promise<DocStateComparison> {
const finder = new HashUtil(states);
const leftOffset = finder.hashToOffset(leftHash);
const rightOffset = finder.hashToOffset(rightHash);
if (rightOffset > leftOffset) {
throw new Error('Comparisons currently require left to be an ancestor of right');
}
const actionNums: number[] = states.slice(rightOffset, leftOffset).map(state => state.n);
const actions = (await activeDoc.getActions(actionNums)).reverse();
let totalAction = createEmptyActionSummary();
for (const action of actions) {
if (!action) { continue; }
const summary = summarizeAction(action);
totalAction = concatenateSummaries([totalAction, summary]);
}
const result: DocStateComparison = {
left: states[leftOffset],
right: states[rightOffset],
parent: states[leftOffset],
summary: (leftOffset === rightOffset) ? 'same' : 'right',
details: {
leftChanges: {tableRenames: [], tableDeltas: {}},
rightChanges: totalAction
}
};
return result;
}
private async _removeDoc(req: Request, res: Response, permanent: boolean) {
const mreq = req as RequestWithLogin;
const scope = getDocScope(req);
const docId = getDocId(req);
if (permanent) {
const {forkId} = parseUrlId(docId);
if (!forkId) {
// Soft delete the doc first, to de-list the document.
await this._dbManager.softDeleteDocument(scope);
}
// Delete document content from storage. Include forks if doc is a trunk.
const forks = forkId ? [] : await this._dbManager.getDocForks(docId);
const docsToDelete = [
docId,
...forks.map((fork) =>
buildUrlId({forkId: fork.id, forkUserId: fork.createdBy!, trunkId: docId})),
];
await Promise.all(docsToDelete.map(docName => this._docManager.deleteDoc(null, docName, true)));
// Permanently delete from database.
const query = await this._dbManager.deleteDocument(scope);
this._dbManager.checkQueryResult(query);
this._grist.getTelemetry().logEvent(mreq, 'deletedDoc', {
full: {
docIdDigest: docId,
userId: mreq.userId,
altSessionId: mreq.altSessionId,
},
});
await sendReply(req, res, query);
} else {
await this._dbManager.softDeleteDocument(scope);
await sendOkReply(req, res);
}
await this._dbManager.flushSingleDocAuthCache(scope, docId);
await this._docManager.interruptDocClients(docId);
}
private async _runSql(activeDoc: ActiveDoc, req: RequestWithLogin, res: Response,
options: Types.SqlPost) {
if (!await activeDoc.canCopyEverything(docSessionFromRequest(req))) {
throw new ApiError('insufficient document access', 403);
}
const statement = options.sql;
// A very loose test, just for early error message
if (!(statement.toLowerCase().includes('select'))) {
throw new ApiError('only select statements are supported', 400);
}
const sqlOptions = activeDoc.docStorage.getOptions();
if (!sqlOptions?.canInterrupt || !sqlOptions?.bindableMethodsProcessOneStatement) {
throw new ApiError('The available SQLite wrapper is not adequate', 500);
}
const timeout =
Math.max(0, Math.min(MAX_CUSTOM_SQL_MSEC,
optIntegerParam(options.timeout, 'timeout') || MAX_CUSTOM_SQL_MSEC));
// Wrap in a select to commit to the SELECT branch of SQLite
// grammar. Note ; isn't a problem.
//
// The underlying SQLite functions used will only process the
// first statement in the supplied text. For node-sqlite3, the
// remainder is placed in a "tail string" ignored by that library.
// So a Robert'); DROP TABLE Students;-- style attack isn't applicable.
//
// Since Grist is used with multiple SQLite wrappers, not just
// node-sqlite3, we have added a bindableMethodsProcessOneStatement
// flag that will need adding for each wrapper, and this endpoint
// will not operate unless that flag is set to true.
//
// The text is wrapped in select * from (USER SUPPLIED TEXT) which
// puts SQLite unconditionally onto the SELECT branch of its
// grammar. It is straightforward to break out of such a wrapper
// with multiple statements, but again, only the first statement
// is processed.
const wrappedStatement = `select * from (${statement})`;
const interrupt = setTimeout(async () => {
await activeDoc.docStorage.interrupt();
}, timeout);
try {
const records = await activeDoc.docStorage.all(wrappedStatement,
...(options.args || []));
res.status(200).json({
statement,
records: records.map(
rec => ({
fields: rec,
})
),
});
} catch (e) {
if (e?.code === 'SQLITE_INTERRUPT') {
res.status(400).json({
error: "a slow statement resulted in a database interrupt",
});
} else if (e?.code === 'SQLITE_ERROR') {
res.status(400).json({
error: e?.message,
});
} else {
throw e;
}
} finally {
clearTimeout(interrupt);
}
}
}
export function addDocApiRoutes(
(core) support GRIST_WORKER_GROUP to place worker into an exclusive group Summary: In an emergency, we may want to serve certain documents with "old" workers as we fix problems. This diff adds some support for that. * Creates duplicate task definitions and services for staging and production doc workers (called grist-docs-staging2 and grist-docs-prod2), pulling from distinct docker tags (staging2 and prod2). The services are set to have zero workers until we need them. * These new workers are started with a new env variable `GRIST_WORKER_GROUP` set to `secondary`. * The `GRIST_WORKER_GROUP` variable, if set, makes the worker available to documents in the named group, and only that group. * An unauthenticated `/assign` endpoint is added to documents which, when POSTed to, checks that the doc is served by a worker in the desired group for that doc (as set manually in redis), and if not frees the doc up for reassignment. This makes it possible to move individual docs between workers without redeployments. The bash scripts added are a record of how the task definitions + services were created. The services could just have been copied manually, but the task definitions will need to be updated whenever the definitions for the main doc workers are updated, so it is worth scripting that. For example, if a certain document were to fail on a new deployment of Grist, but rolling back the full deployment wasn't practical: * Set prod2 tag in docker to desired codebase for that document * Set desired_count for grist-docs-prod2 service to non-zero * Set doc-<docid>-group for that doc in redis to secondary * Hit /api/docs/<docid>/assign to move the doc to grist-docs-prod2 (If the document needs to be reverted to a previous snapshot, that currently would need doing manually - could be made simpler, but not in scope of this diff). Test Plan: added tests Reviewers: dsagal Reviewed By: dsagal Differential Revision: https://phab.getgrist.com/D2649
2020-11-02 19:24:46 +00:00
app: Application, docWorker: DocWorker, docWorkerMap: IDocWorkerMap, docManager: DocManager, dbManager: HomeDBManager,
grist: GristServer
) {
const api = new DocWorkerApi(app, docWorker, docWorkerMap, docManager, dbManager, grist);
api.addEndpoints();
}
/**
* Options for returning results from a query about document data.
* Currently these option don't affect the query itself, only the
* results returned to the user.
*/
export interface QueryParameters {
sort?: string[]; // Columns names to sort by (ascending order by default,
// prepend "-" for descending order, can contain flags,
// see more in Sort.SortSpec).
limit?: number; // Limit on number of rows to return.
}
/**
* Extract a sort parameter from a request, if present. Follows
* https://jsonapi.org/format/#fetching-sorting for want of a better
* standard - comma separated, defaulting to ascending order, keys
* prefixed by "-" for descending order.
*
* The sort parameter can either be given as a query parameter, or
* as a header.
*/
function getSortParameter(req: Request): string[]|undefined {
const sortString: string|undefined = optStringParam(req.query.sort, 'sort') || req.get('X-Sort');
if (!sortString) { return undefined; }
return sortString.split(',');
}
/**
* Extract a limit parameter from a request, if present. Should be a
* simple integer. The limit parameter can either be given as a query
* parameter, or as a header.
*/
function getLimitParameter(req: Request): number|undefined {
const limitString: string|undefined = optStringParam(req.query.limit, 'limit') || req.get('X-Limit');
if (!limitString) { return undefined; }
const limit = parseInt(limitString, 10);
if (isNaN(limit)) { throw new Error('limit is not a number'); }
return limit;
}
/**
* Extract sort and limit parameters from request, if they are present.
*/
function getQueryParameters(req: Request): QueryParameters {
return {
sort: getSortParameter(req),
limit: getLimitParameter(req),
};
}
/**
* Sort table contents being returned. Sort keys with a '-' prefix
* are sorted in descending order, otherwise ascending. Contents are
* modified in place. Sort keys can contain sort options.
* Columns can be either expressed as a colId (name string) or as colRef (rowId number).
*/
function applySort(
values: TableColValues,
sort: string[],
_columns: TableRecordValue[]|null = null) {
if (!sort) { return values; }
// First we need to prepare column description in ColValue format (plain objects).
// This format is used by ServerColumnGetters.
let properColumns: ColValues[] = [];
// We will receive columns information only for user tables, not for metatables. So
// if this is the case, we will infer them from the result.
if (!_columns) {
_columns = Object.keys(values).map((col, index) => ({ id: col, fields: { colRef: index }}));
}
// For user tables, we will not get id column (as this column is not in the schema), so we need to
// make sure the column is there.
else {
// This is enough information for ServerGetters
_columns = [..._columns, { id : 'id', fields: {colRef: 0 }}];
}
// Once we have proper columns, we can convert them to format that ServerColumnGetters
// understand.
properColumns = _columns.map(c => ({
...c.fields,
id : c.fields.colRef,
colId: c.id
}));
// We will sort row indices in the values object, not rows ids.
const rowIndices = values.id.map((__, i) => i);
const getters = new ServerColumnGetters(rowIndices, values, properColumns);
const sortFunc = new SortFunc(getters);
const colIdToRef = new Map(properColumns.map(({id, colId}) => [colId as string, id as number]));
sortFunc.updateSpec(Sort.parseNames(sort, colIdToRef));
rowIndices.sort(sortFunc.compare.bind(sortFunc));
// Sort resulting values according to the sorted index.
for (const key of Object.keys(values)) {
const col = values[key];
values[key] = rowIndices.map(i => col[i]);
}
return values;
}
/**
* Truncate columns to the first N values. Columns are modified in place.
*/
function applyLimit(values: TableColValues, limit: number) {
// for no limit, or 0 limit, do not apply any restriction
if (!limit) { return values; }
for (const key of Object.keys(values)) {
values[key].splice(limit);
}
return values;
}
/**
* Apply query parameters to table contents. Contents are modified in place.
*/
export function applyQueryParameters(
values: TableColValues,
params: QueryParameters,
columns: TableRecordValue[]|null = null): TableColValues {
if (params.sort) { applySort(values, params.sort, columns); }
if (params.limit) { applyLimit(values, params.limit); }
return values;
}
function getErrorPlatform(tableId: string): TableOperationsPlatform {
return {
async getTableId() { return tableId; },
throwError(verb, text, status) {
throw new ApiError(verb + (verb ? ' ' : '') + text, status);
},
applyUserActions() {
throw new Error('no document');
}
};
}
export async function getMetaTables(activeDoc: ActiveDoc, req: RequestWithLogin) {
return await handleSandboxError("", [],
activeDoc.fetchMetaTables(docSessionFromRequest(req)));
}
async function getTableOperations(
req: RequestWithLogin,
activeDoc: ActiveDoc,
tableId?: string): Promise<TableOperationsImpl> {
const options: OpOptions = {
parseStrings: !isAffirmative(req.query.noparse)
};
const realTableId = await getRealTableId(tableId ?? req.params.tableId, {activeDoc, req});
const platform: TableOperationsPlatform = {
...getErrorPlatform(realTableId),
applyUserActions(actions, opts) {
if (!activeDoc) { throw new Error('no document'); }
return activeDoc.applyUserActions(
docSessionFromRequest(req),
actions,
opts
);
}
};
return new TableOperationsImpl(platform, options);
}
async function handleSandboxError<T>(tableId: string, colNames: string[], p: Promise<T>): Promise<T> {
return handleSandboxErrorOnPlatform(tableId, colNames, p, getErrorPlatform(tableId));
}
export interface DocApiUsagePeriod {
unit: 'day' | 'hour' | 'minute',
format: string;
periodsPerDay: number;
}
export const docApiUsagePeriods: DocApiUsagePeriod[] = [
{
unit: 'day',
format: 'YYYY-MM-DD',
periodsPerDay: 1,
},
{
unit: 'hour',
format: 'YYYY-MM-DDTHH',
periodsPerDay: 24,
},
{
unit: 'minute',
format: 'YYYY-MM-DDTHH:mm',
periodsPerDay: 24 * 60,
},
];
/**
* Returns a key used for redis and a local cache
* which store the number of API requests made for the given document in the given period.
* The key contains the current UTC date (and maybe hour and minute)
* so that counts from previous periods are simply ignored and eventually evicted.
* This means that the daily measured usage conceptually 'resets' at UTC midnight.
* If `current` is false, returns a key for the next day/hour.
*/
export function docPeriodicApiUsageKey(docId: string, current: boolean, period: DocApiUsagePeriod, m: moment.Moment) {
if (!current) {
m = m.clone().add(1, period.unit);
}
return `doc-${docId}-periodicApiUsage-${m.format(period.format)}`;
}
/**
* Checks whether the doc API usage fits within the daily maximum.
* If so, returns an array of keys for each unit of time whose usage should be incremented.
* If not, returns undefined.
*
* Description of the algorithm this is implementing:
*
* Maintain up to 5 buckets: current day, next day, current hour, next hour, current minute.
* For each API request, check in order:
* - if current_day < DAILY_LIMIT, allow; increment all 3 current buckets
* - else if current_hour < DAILY_LIMIT/24, allow; increment next_day, current_hour, and current_minute buckets.
* - else if current_minute < DAILY_LIMIT/24/60, allow; increment next_day, next_hour, and current_minute buckets.
* - else reject.
* I think it has pretty good properties:
* - steady low usage may be maintained even if a burst exhausted the daily limit
* - user could get close to twice the daily limit on the first day with steady usage after a burst,
* but would then be limited to steady usage the next day.
*/
export function getDocApiUsageKeysToIncr(
docId: string, usage: LRUCache<string, number>, dailyMax: number, m: moment.Moment
): string[] | undefined {
// Start with keys for the current day, minute, and hour
const keys = docApiUsagePeriods.map(p => docPeriodicApiUsageKey(docId, true, p, m));
for (let i = 0; i < docApiUsagePeriods.length; i++) {
const period = docApiUsagePeriods[i];
const key = keys[i];
const periodMax = Math.ceil(dailyMax / period.periodsPerDay);
const count = usage.get(key) || 0;
if (count < periodMax) {
return keys;
}
// Allocation for the current day/hour/minute has been exceeded, increment the next day/hour/minute instead.
keys[i] = docPeriodicApiUsageKey(docId, false, period, m);
}
// Usage exceeded all the time buckets, so return undefined to reject the request.
}
export interface WebhookSubscription {
unsubscribeKey: string;
webhookId: string;
}
/**
* Converts `activeDoc` to XLSX and sends the converted data through `res`.
*/
export async function downloadXLSX(activeDoc: ActiveDoc, req: Request,
res: Response, options: DownloadOptions) {
const {filename} = options;
res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
res.setHeader('Content-Disposition', contentDisposition(filename + '.xlsx'));
return streamXLSX(activeDoc, req, res, options);
}