From e1df6039c2e047ebecadc226b2a363bd1412b61d Mon Sep 17 00:00:00 2001 From: John Cant Date: Sun, 30 Jul 2023 20:13:43 +0100 Subject: [PATCH] REQUEST now supports POST (#588) * REQUEST now supports POST * Add extra flag for enabling REQUEST, also update README and comments Co-authored-by: John Cant Co-authored-by: Alex Hall --- README.md | 1 + app/common/ActionBundle.ts | 2 ++ app/server/devServerMain.ts | 5 +++ app/server/lib/Requests.ts | 11 ++++-- sandbox/grist/functions/info.py | 63 +++++++++++++++++++++++++++++---- sandbox/grist/test_requests.py | 44 ++++++++++++++++++++++- 6 files changed, 116 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ca000a78..11671716 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ GRIST_DEFAULT_PRODUCT | if set, this controls enabled features and limits of ne GRIST_DEFAULT_LOCALE | Locale to use as fallback when Grist cannot honour the browser locale. GRIST_DOMAIN | in hosted Grist, Grist is served from subdomains of this domain. Defaults to "getgrist.com". GRIST_EXPERIMENTAL_PLUGINS | enables experimental plugins +GRIST_ENABLE_REQUEST_FUNCTION | enables the REQUEST function. This function performs HTTP requests in a similar way to `requests.request`. This function presents a significant security risk, since it can let users call internal endpoints when Grist is available publicly. This function can also cause performance issues. Unset by default. GRIST_HIDE_UI_ELEMENTS | comma-separated list of UI features to disable. Allowed names of parts: `helpCenter,billing,templates,multiSite,multiAccounts,sendToDrive,tutorials`. If a part also exists in GRIST_UI_FEATURES, it will still be disabled. GRIST_HOME_INCLUDE_STATIC | if set, home server also serves static resources GRIST_HOST | hostname to use when listening on a port. diff --git a/app/common/ActionBundle.ts b/app/common/ActionBundle.ts index f48e584a..35fe34d1 100644 --- a/app/common/ActionBundle.ts +++ b/app/common/ActionBundle.ts @@ -72,6 +72,8 @@ export interface SandboxActionBundle { // Represents a unique call to the Python REQUEST function export interface SandboxRequest { url: string; + method: string; + body?: string; params: Record | null; headers: Record | null; deps: unknown; // pass back to the sandbox unchanged in the response diff --git a/app/server/devServerMain.ts b/app/server/devServerMain.ts index a7f20a23..7e7012a5 100644 --- a/app/server/devServerMain.ts +++ b/app/server/devServerMain.ts @@ -66,6 +66,11 @@ export async function main() { process.env.GRIST_EXPERIMENTAL_PLUGINS = "1"; } + // Experimental plugins are enabled by default for devs + if (!process.env.GRIST_ENABLE_REQUEST_FUNCTION) { + process.env.GRIST_ENABLE_REQUEST_FUNCTION = "1"; + } + // For tests, it is useful to start with the database in a known state. // If TEST_CLEAN_DATABASE is set, we reset the database before starting. if (process.env.TEST_CLEAN_DATABASE) { diff --git a/app/server/lib/Requests.ts b/app/server/lib/Requests.ts index 4cee6a2e..69f5dd9e 100644 --- a/app/server/lib/Requests.ts +++ b/app/server/lib/Requests.ts @@ -80,16 +80,21 @@ export class DocRequests { private async _handleSingleRequestRaw(request: SandboxRequest): Promise { try { - if (process.env.GRIST_EXPERIMENTAL_PLUGINS != '1') { + if (process.env.GRIST_ENABLE_REQUEST_FUNCTION != '1') { throw new Error("REQUEST is not enabled"); } - const {url, params, headers} = request; + const {url, method, body, params, headers} = request; const urlObj = new URL(url); log.rawInfo("Handling sandbox request", {host: urlObj.host, docId: this._activeDoc.docName}); for (const [param, value] of Object.entries(params || {})) { urlObj.searchParams.append(param, value); } - const response = await fetch(urlObj.toString(), {headers: headers || {}, agent: proxyAgent(urlObj)}); + const response = await fetch(urlObj.toString(), { + headers: headers || {}, + agent: proxyAgent(urlObj), + method, + body + }); const content = await response.buffer(); const {status, statusText} = response; const encoding = httpEncoding(response.headers.get('content-type'), content); diff --git a/sandbox/grist/functions/info.py b/sandbox/grist/functions/info.py index 87640ef2..99ab6424 100644 --- a/sandbox/grist/functions/info.py +++ b/sandbox/grist/functions/info.py @@ -4,13 +4,14 @@ from __future__ import absolute_import import datetime import hashlib -import json +import json as json_module import math import numbers import re import chardet import six +from six.moves import urllib_parse import column import docmodel @@ -653,20 +654,70 @@ def is_error(value): or (isinstance(value, float) and math.isnan(value))) +def _replicate_requests_body_args(data=None, json=None): + """ + Replicate some of the behaviour of requests.post, specifically the data and + json args. + + Returns a tuple of (body, extra_headers) + """ + if data is None and json is None: + return None, {} + + elif data is not None and json is None: + if isinstance(data, str): + body = data + extra_headers = {} + else: + body = urllib_parse.urlencode(data) + extra_headers = { + "Content-Type": "application/x-www-form-urlencoded", + } + return body, extra_headers + + elif json is not None and data is None: + if isinstance(json, str): + body = json + else: + body = json_module.dumps(json) + extra_headers = { + "Content-Type": "application/json", + } + return body, extra_headers + + elif data is not None and json is not None: + # From testing manually with requests 2.28.2, data overrides json if both + # supplied. However, this is probably a mistake on behalf of the caller, so + # we choose to throw an error instead + raise ValueError("`data` and `json` cannot be supplied to REQUEST at the same time") + + @unimplemented # ^ This excludes this function from autocomplete while in beta # and marks it as unimplemented in the docs. # It also makes grist-help expect to see the string 'raise NotImplemented' in the function source, # which it does now, because of this comment. Removing this comment will currently break the docs. -def REQUEST(url, params=None, headers=None): - # Makes a GET HTTP request with an API similar to `requests.get`. +def REQUEST(url, params=None, headers=None, method="GET", data=None, json=None): + # Makes an HTTP request with an API similar to `requests.request`. # Actually jumps through hoops internally to make the request asynchronously (usually) # while feeling synchronous to the formula writer. + # When making a POST or PUT request, REQUEST supports `data` and `json` args, from `requests.request`: + # - `args` as str: Used as the request body + # - `args` as other types: Form encoded and used as the request body. The correct header is also set. + # - `json` as str: Used as the request body. The correct header is also set. + # - `json` as other types: JSON encoded and set as the request body. The correct header is also set. + body, _headers = _replicate_requests_body_args(data=data, json=json) + + # Extra headers that make us consistent with requests.post must not override + # user-supplied headers. + _headers.update(headers or {}) + # Requests are identified by a string key in various places. # The same arguments should produce the same key so the request is only made once. - args = dict(url=url, params=params, headers=headers) - args_json = json.dumps(args, sort_keys=True) + args = dict(url=url, params=params, headers=_headers, method=method, body=body) + + args_json = json_module.dumps(args, sort_keys=True) key = hashlib.sha256(args_json.encode()).hexdigest() # This may either return the raw response data or it may raise a special exception @@ -701,7 +752,7 @@ class Response(object): return self.content.decode(self.encoding) def json(self, **kwargs): - return json.loads(self.text, **kwargs) + return json_module.loads(self.text, **kwargs) @property def ok(self): diff --git a/sandbox/grist/test_requests.py b/sandbox/grist/test_requests.py index 4a57c168..d5960f6b 100644 --- a/sandbox/grist/test_requests.py +++ b/sandbox/grist/test_requests.py @@ -4,6 +4,7 @@ import unittest import test_engine import testutil from functions import CaseInsensitiveDict, Response, HTTPError +from functions.info import _replicate_requests_body_args class TestCaseInsensitiveDict(unittest.TestCase): @@ -73,6 +74,45 @@ class TestResponse(unittest.TestCase): self.assertEqual(r.text, text) +class TestRequestsPostInterface(unittest.TestCase): + def test_no_post_args(self): + body, headers = _replicate_requests_body_args() + + assert body is None + assert headers == {} + + def test_data_as_dict(self): + body, headers = _replicate_requests_body_args(data={"foo": "bar"}) + + assert body == "foo=bar" + assert headers == {"Content-Type": "application/x-www-form-urlencoded"} + + def test_data_as_string(self): + body, headers = _replicate_requests_body_args(data="some_content") + + assert body == "some_content" + assert headers == {} + + def test_json_as_dict(self): + body, headers = _replicate_requests_body_args(json={"foo": "bar"}) + + assert body == '{"foo": "bar"}' + assert headers == {"Content-Type": "application/json"} + + def test_json_as_string(self): + body, headers = _replicate_requests_body_args(json="invalid_but_ignored") + + assert body == "invalid_but_ignored" + assert headers == {"Content-Type": "application/json"} + + def test_data_and_json_together(self): + with self.assertRaises(ValueError): + body, headers = _replicate_requests_body_args( + json={"foo": "bar"}, + data={"quux": "jazz"} + ) + + class TestRequestFunction(test_engine.EngineTestCase): sample = testutil.parse_test_sample({ "SCHEMA": [ @@ -98,12 +138,14 @@ r = REQUEST('my_url', headers={'foo': 'bar'}, params={'b': 1, 'a': 2}) r.__dict__ """ out_actions = self.modify_column("Table1", "Request", formula=formula) - key = '9d305be9664924aaaf7ebb0bab2e4155d1fa1b9dcde53e417f1a9f9a2c7e09b9' + key = 'd7f8cedf177ab538bf7dadf66e77a525486a29a41ce4520b2c89a33e39095fed' deps = {'Table1': {'Request': [1, 2]}} args = { 'url': 'my_url', 'headers': {'foo': 'bar'}, 'params': {'a': 2, 'b': 1}, + 'method': 'GET', + 'body': None, 'deps': deps, } self.assertEqual(out_actions.requests, {key: args})