(core) Proxy Agent moved to the separate file, Triggers are using proxy now to perform fetch

Summary:
- Webhooks form Triggers.ts should now use proxy if it's configured
- Proxy handling code separated to ProxyAgent.ts
- Tests for ProxyAgent
- Integration/API Tests for using Proxy in webhooks
- a bit of refactor - proxy test uses mostly the same codebase as DocApi.ts, but because last one if over 4000 lines long, I've put it into separated file, and extract some common parts (there is some duplicates tho)
- some cleanup in files that I've touched

Test Plan:
Manual test to check if proxy is used on the staging env

Automatic test checking if (fake) proxy was called

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3860

app/server/lib/Requests.ts

@@ -2,16 +2,15 @@ import {SandboxRequest} from 'app/common/ActionBundle';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import {httpEncoding} from 'app/server/lib/httpEncoding';
-import {HttpsProxyAgent} from 'https-proxy-agent';
-import {HttpProxyAgent} from 'http-proxy-agent';
 import fetch from 'node-fetch';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import * as fse from 'fs-extra';
+import log from 'app/server/lib/log';
+import {proxyAgent} from "app/server/utils/ProxyAgent";
 import chunk = require('lodash/chunk');
 import fromPairs = require('lodash/fromPairs');
 import zipObject = require('lodash/zipObject');
-import * as fse from 'fs-extra';
-import log from 'app/server/lib/log';
 
 export class DocRequests {
   // Request responses are briefly cached in files only to handle multiple requests in a formula
@@ -118,11 +117,3 @@ interface RequestError {
 
 type Response = RequestError | SuccessfulResponse;
 
-function proxyAgent(requestUrl: URL) {
-  const proxy = process.env.GRIST_HTTPS_PROXY;
-  if (!proxy) {
-    return undefined;
-  }
-  const ProxyAgent = requestUrl.protocol === "https:" ? HttpsProxyAgent : HttpProxyAgent;
-  return new ProxyAgent(proxy);
-}

app/server/lib/Triggers.ts

@@ -13,6 +13,7 @@ import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
 import {matchesBaseDomain} from 'app/server/lib/requestUtils';
 import {delayAbort} from 'app/server/lib/serverUtils';
+import {proxyAgent} from 'app/server/utils/ProxyAgent';
 import {promisifyAll} from 'bluebird';
 import * as _ from 'lodash';
 import {AbortController, AbortSignal} from 'node-abort-controller';
@@ -738,6 +739,7 @@ export class DocTriggers {
             'Content-Type': 'application/json',
           },
           signal,
+          agent: proxyAgent(new URL(url)),
         });
         if (response.status === 200) {
           await this._stats.logBatch(id, 'success', { size, httpStatus: 200, error: null, attempts: attempt + 1 });

app/server/utils/ProxyAgent.ts

@@ -0,0 +1,11 @@
+import {HttpsProxyAgent} from "https-proxy-agent";
+import {HttpProxyAgent} from "http-proxy-agent";
+
+export function proxyAgent(requestUrl: URL): HttpProxyAgent | HttpsProxyAgent | undefined {
+    const proxy = process.env.GRIST_HTTPS_PROXY;
+    if (!proxy) {
+        return undefined;
+    }
+    const ProxyAgent = requestUrl.protocol === "https:" ? HttpsProxyAgent : HttpProxyAgent;
+    return new ProxyAgent(proxy);
+}