(core) Proxy Agent moved to the separate file, Triggers are using proxy now to perform fetch

Summary:
- Webhooks form Triggers.ts should now use proxy if it's configured
- Proxy handling code separated to ProxyAgent.ts
- Tests for ProxyAgent
- Integration/API Tests for using Proxy in webhooks
- a bit of refactor - proxy test uses mostly the same codebase as DocApi.ts, but because last one if over 4000 lines long, I've put it into separated file, and extract some common parts (there is some duplicates tho)
- some cleanup in files that I've touched

Test Plan:
Manual test to check if proxy is used on the staging env

Automatic test checking if (fake) proxy was called

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D3860

app/server/lib/Requests.ts

@@ -2,16 +2,15 @@ import {SandboxRequest} from 'app/common/ActionBundle';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import {httpEncoding} from 'app/server/lib/httpEncoding';
-import {HttpsProxyAgent} from 'https-proxy-agent';
-import {HttpProxyAgent} from 'http-proxy-agent';
 import fetch from 'node-fetch';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import * as fse from 'fs-extra';
+import log from 'app/server/lib/log';
+import {proxyAgent} from "app/server/utils/ProxyAgent";
 import chunk = require('lodash/chunk');
 import fromPairs = require('lodash/fromPairs');
 import zipObject = require('lodash/zipObject');
-import * as fse from 'fs-extra';
-import log from 'app/server/lib/log';
 
 export class DocRequests {
   // Request responses are briefly cached in files only to handle multiple requests in a formula
@@ -118,11 +117,3 @@ interface RequestError {
 
 type Response = RequestError | SuccessfulResponse;
 
-function proxyAgent(requestUrl: URL) {
-  const proxy = process.env.GRIST_HTTPS_PROXY;
-  if (!proxy) {
-    return undefined;
-  }
-  const ProxyAgent = requestUrl.protocol === "https:" ? HttpsProxyAgent : HttpProxyAgent;
-  return new ProxyAgent(proxy);
-}

app/server/lib/Triggers.ts

@@ -13,6 +13,7 @@ import {makeExceptionalDocSession} from 'app/server/lib/DocSession';
 import log from 'app/server/lib/log';
 import {matchesBaseDomain} from 'app/server/lib/requestUtils';
 import {delayAbort} from 'app/server/lib/serverUtils';
+import {proxyAgent} from 'app/server/utils/ProxyAgent';
 import {promisifyAll} from 'bluebird';
 import * as _ from 'lodash';
 import {AbortController, AbortSignal} from 'node-abort-controller';
@@ -738,6 +739,7 @@ export class DocTriggers {
             'Content-Type': 'application/json',
           },
           signal,
+          agent: proxyAgent(new URL(url)),
         });
         if (response.status === 200) {
           await this._stats.logBatch(id, 'success', { size, httpStatus: 200, error: null, attempts: attempt + 1 });

app/server/utils/ProxyAgent.ts

@@ -0,0 +1,11 @@
+import {HttpsProxyAgent} from "https-proxy-agent";
+import {HttpProxyAgent} from "http-proxy-agent";
+
+export function proxyAgent(requestUrl: URL): HttpProxyAgent | HttpsProxyAgent | undefined {
+    const proxy = process.env.GRIST_HTTPS_PROXY;
+    if (!proxy) {
+        return undefined;
+    }
+    const ProxyAgent = requestUrl.protocol === "https:" ? HttpsProxyAgent : HttpProxyAgent;
+    return new ProxyAgent(proxy);
+}

test/server/lib/DocApi.ts

@@ -6,31 +6,38 @@ import {DocAPI, DocState, UserAPIImpl} from 'app/common/UserAPI';
 import {testDailyApiLimitFeatures} from 'app/gen-server/entity/Product';
 import {AddOrUpdateRecord, Record as ApiRecord} from 'app/plugin/DocApiTypes';
 import {CellValue, GristObjCode} from 'app/plugin/GristData';
-import {applyQueryParameters, docApiUsagePeriods, docPeriodicApiUsageKey,
+import {
-  getDocApiUsageKeysToIncr, WebhookSubscription} from 'app/server/lib/DocApi';
+  applyQueryParameters,
+  docApiUsagePeriods,
+  docPeriodicApiUsageKey,
+  getDocApiUsageKeysToIncr,
+  WebhookSubscription
+} from 'app/server/lib/DocApi';
 import log from 'app/server/lib/log';
+import {delayAbort} from 'app/server/lib/serverUtils';
 import {WebhookSummary} from 'app/server/lib/Triggers';
-import {waitForIt} from 'test/server/wait';
-import {delayAbort, exitPromise} from 'app/server/lib/serverUtils';
-import {connectTestingHooks, TestingHooksClient} from 'app/server/lib/TestingHooks';
 import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
 import {delay} from 'bluebird';
 import * as bodyParser from 'body-parser';
 import {assert} from 'chai';
-import {ChildProcess, execFileSync, spawn} from 'child_process';
 import FormData from 'form-data';
 import * as fse from 'fs-extra';
 import * as _ from 'lodash';
 import LRUCache from 'lru-cache';
 import * as moment from 'moment';
+import {AbortController} from 'node-abort-controller';
 import fetch from 'node-fetch';
 import {tmpdir} from 'os';
 import * as path from 'path';
 import {createClient, RedisClient} from 'redis';
-import {AbortController} from 'node-abort-controller';
 import {configForUser} from 'test/gen-server/testUtils';
 import {serveSomething, Serving} from 'test/server/customUtil';
+import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
+import {prepareFilesystemDirectoryForTests} from 'test/server/lib/helpers/PrepareFilesystemDirectoryForTests';
+import {signal} from 'test/server/lib/helpers/Signal';
+import {TestServer} from 'test/server/lib/helpers/TestServer';
 import * as testUtils from 'test/server/testUtils';
+import {waitForIt} from 'test/server/wait';
 import clone = require('lodash/clone');
 import defaultsDeep = require('lodash/defaultsDeep');
 import pick = require('lodash/pick');
@@ -76,21 +83,15 @@ describe('DocApi', function() {
     }
 
     // Create the tmp dir removing any previous one
-    await fse.remove(tmpDir);
+    await prepareFilesystemDirectoryForTests(tmpDir);
-    await fse.mkdirs(tmpDir);
+
-    log.warn(`Test logs and data are at: ${tmpDir}/`);
 
     // Let's create a sqlite db that we can share with servers that run in other processes, hence
     // not an in-memory db. Running seed.ts directly might not take in account the most recent value
     // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
     // configuration (in-memory for instance). Spawning a process is one way to make sure that the
     // latest value prevail.
-    process.env.TYPEORM_DATABASE = path.join(tmpDir, 'landing.db');
+    await prepareDatabase(tmpDir);
-    const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
-    execFileSync('node', [seed, 'init'], {
-      env: process.env,
-      stdio: 'inherit'
-    });
   });
 
   after(() => {
@@ -109,7 +110,11 @@ describe('DocApi', function() {
 
   describe("should work with a merged server", async () => {
     setup('merged', async () => {
-      home = docs = await startServer('home,docs');
+      const additionalEnvConfiguration = {
+        ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+        GRIST_DATA_DIR: dataDir
+      };
+      home = docs = await TestServer.startServer('home,docs', tmpDir, suitename, additionalEnvConfiguration);
       homeUrl = serverUrl = home.serverUrl;
       hasHomeApi = true;
     });
@@ -120,8 +125,13 @@ describe('DocApi', function() {
   if (process.env.TEST_REDIS_URL) {
     describe("should work with a home server and a docworker", async () => {
       setup('separated', async () => {
-        home = await startServer('home');
+        const additionalEnvConfiguration = {
-        docs = await startServer('docs', home.serverUrl);
+          ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+          GRIST_DATA_DIR: dataDir
+        };
+
+        home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
+        docs = await TestServer.startServer('docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl);
         homeUrl = serverUrl = home.serverUrl;
         hasHomeApi = true;
       });
@@ -130,8 +140,12 @@ describe('DocApi', function() {
 
     describe("should work directly with a docworker", async () => {
       setup('docs', async () => {
-        home = await startServer('home');
+        const additionalEnvConfiguration = {
-        docs = await startServer('docs', home.serverUrl);
+          ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+          GRIST_DATA_DIR: dataDir
+        };
+        home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
+        docs = await TestServer.startServer('docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl);
         homeUrl = home.serverUrl;
         serverUrl = docs.serverUrl;
         hasHomeApi = false;
@@ -857,7 +871,8 @@ function testDocApi() {
       const query = "filter=" + encodeURIComponent(JSON.stringify(filters));
       return axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data?${query}`, chimpy);
     }
-    function checkResults(resp: AxiosResponse<any>, expectedData: any) {
+
+    function checkResults(resp: AxiosResponse, expectedData: any) {
       assert.equal(resp.status, 200);
       assert.deepEqual(resp.data, expectedData);
     }
@@ -911,15 +926,24 @@ function testDocApi() {
         const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
         const config = configForUser('chimpy');
         if (mode === 'url') {
-          if (sort) { url.searchParams.append('sort', sort.join(',')); }
+          if (sort) {
-          if (limit) { url.searchParams.append('limit', String(limit)); }
+            url.searchParams.append('sort', sort.join(','));
+          }
+          if (limit) {
+            url.searchParams.append('limit', String(limit));
+          }
         } else {
-          if (sort)  { config.headers['x-sort'] = sort.join(','); }
+          if (sort) {
-          if (limit) { config.headers['x-limit'] = String(limit); }
+            config.headers['x-sort'] = sort.join(',');
+          }
+          if (limit) {
+            config.headers['x-limit'] = String(limit);
+          }
         }
         return axios.get(url.href, config);
       }
-      function checkResults(resp: AxiosResponse<any>, expectedData: any) {
+
+      function checkResults(resp: AxiosResponse, expectedData: any) {
         assert.equal(resp.status, 200);
         assert.deepEqual(resp.data, expectedData);
       }
@@ -1109,7 +1133,7 @@ function testDocApi() {
     });
   });
 
-  function checkError(status: number, test: RegExp|object, resp: AxiosResponse<any>, message?: string) {
+  function checkError(status: number, test: RegExp | object, resp: AxiosResponse, message?: string) {
     assert.equal(resp.status, status);
     if (test instanceof RegExp) {
       assert.match(resp.data.error, test, message);
@@ -1446,13 +1470,15 @@ function testDocApi() {
       await test({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
       // All column types are allowed, except Arrays (or objects) without correct code.
       const testField = async (A: any) => {
-        await test({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
+        await test({records: [{id: 1, fields: {A}}]}, {
+          error: 'Invalid payload', details:
             'Error: body.records[0] is not a NewRecord; ' +
             'body.records[0].fields.A is not a CellValue; ' +
             'body.records[0].fields.A is none of number, ' +
             'string, boolean, null, 1 more; body.records[0].' +
             'fields.A[0] is not a GristObjCode; body.records[0]' +
-                    '.fields.A[0] is not a valid enum value'});
+            '.fields.A[0] is not a valid enum value'
+        });
       };
       // test no code at all
       await testField([]);
@@ -1601,6 +1627,7 @@ function testDocApi() {
 
     it("validates request schema", async function () {
       const url = `${serverUrl}/api/docs/${docIds.TestDoc}/tables/Foo/records`;
+
       async function failsWithError(payload: any, error: { error: string, details?: string }) {
         const resp = await axios.patch(url, payload, chimpy);
         checkError(400, error, resp);
@@ -1610,18 +1637,24 @@ function testDocApi() {
 
       await failsWithError({records: 1}, {error: 'Invalid payload', details: 'Error: body.records is not an array'});
 
-      await failsWithError({records: []}, {error: 'Invalid payload', details:
+      await failsWithError({records: []}, {
-                  'Error: body.records[0] is not a Record; body.records[0] is not an object'});
+        error: 'Invalid payload', details:
+          'Error: body.records[0] is not a Record; body.records[0] is not an object'
+      });
 
-      await failsWithError({records: [{}]}, {error: 'Invalid payload', details:
+      await failsWithError({records: [{}]}, {
+        error: 'Invalid payload', details:
           'Error: body.records[0] is not a Record\n    ' +
           'body.records[0].id is missing\n    ' +
-                  'body.records[0].fields is missing'});
+          'body.records[0].fields is missing'
+      });
 
-      await failsWithError({records: [{id: "1"}]}, {error: 'Invalid payload', details:
+      await failsWithError({records: [{id: "1"}]}, {
+        error: 'Invalid payload', details:
           'Error: body.records[0] is not a Record\n' +
           '    body.records[0].id is not a number\n' +
-                  '    body.records[0].fields is missing'});
+          '    body.records[0].fields is missing'
+      });
 
       await failsWithError(
         {records: [{id: 1, fields: {A: 1}}, {id: 2, fields: {B: 3}}]},
@@ -1629,13 +1662,15 @@ function testDocApi() {
 
       // Test invalid object codes
       const fieldIsNotValid = async (A: any) => {
-        await failsWithError({records: [{ id: 1, fields: { A } }]}, {error: 'Invalid payload', details:
+        await failsWithError({records: [{id: 1, fields: {A}}]}, {
+          error: 'Invalid payload', details:
             'Error: body.records[0] is not a Record; ' +
             'body.records[0].fields.A is not a CellValue; ' +
             'body.records[0].fields.A is none of number, ' +
             'string, boolean, null, 1 more; body.records[0].' +
             'fields.A[0] is not a GristObjCode; body.records[0]' +
-                    '.fields.A[0] is not a valid enum value'});
+            '.fields.A[0] is not a valid enum value'
+        });
       };
       await fieldIsNotValid([]);
       await fieldIsNotValid(['ZZ']);
@@ -2232,7 +2267,9 @@ function testDocApi() {
   });
 
   it('POST /workspaces/{wid}/import handles empty filenames', async function () {
-    if (!process.env.TEST_REDIS_URL) { this.skip(); }
+    if (!process.env.TEST_REDIS_URL) {
+      this.skip();
+    }
     const worker1 = await userApi.getWorkerAPI('import');
     const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
     const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
@@ -2246,7 +2283,9 @@ function testDocApi() {
   });
 
   it("document is protected during upload-and-import sequence", async function () {
-    if (!process.env.TEST_REDIS_URL) { this.skip(); }
+    if (!process.env.TEST_REDIS_URL) {
+      this.skip();
+    }
     // Prepare an API for a different user.
     const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
       headers: {Authorization: 'Bearer api_key_for_kiwi'},
@@ -2332,15 +2371,15 @@ function testDocApi() {
     const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid1'}, ws1);
     try {
       // Make sure an edit made by docId is visible when accessed via docId or urlId
-      let resp = await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
+      await axios.post(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, {
         A: ['Apple'], B: [99]
       }, chimpy);
-      resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
+      let resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
       assert.equal(resp.data.A[0], 'Apple');
       resp = await axios.get(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, chimpy);
       assert.equal(resp.data.A[0], 'Apple');
       // Make sure an edit made by urlId is visible when accessed via docId or urlId
-      resp = await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
+      await axios.post(`${serverUrl}/api/docs/urlid1/tables/Table1/data`, {
         A: ['Orange'], B: [42]
       }, chimpy);
       resp = await axios.get(`${serverUrl}/api/docs/${doc1}/tables/Table1/data`, chimpy);
@@ -2564,7 +2603,8 @@ function testDocApi() {
       {tableRenames: [], tableDeltas: {}});
     const addA1: ActionSummary = {
       tableRenames: [],
-      tableDeltas: { Table1: {
+      tableDeltas: {
+        Table1: {
           updateRows: [],
           removeRows: [],
           addRows: [2],
@@ -2573,7 +2613,8 @@ function testDocApi() {
             manualSort: {[2]: [null, [2]]},
           },
           columnRenames: [],
-      } }
+        }
+      }
     };
     assert.deepEqual(comp.details!.leftChanges, addA1);
 
@@ -2617,7 +2658,8 @@ function testDocApi() {
       {tableRenames: [], tableDeltas: {}});
     const addA2: ActionSummary = {
       tableRenames: [],
-      tableDeltas: { Table1: {
+      tableDeltas: {
+        Table1: {
           updateRows: [],
           removeRows: [],
           addRows: [3],
@@ -2626,7 +2668,8 @@ function testDocApi() {
             manualSort: {[3]: [null, [3]]},
           },
           columnRenames: [],
-      } }
+        }
+      }
     };
     assert.deepEqual(comp.details!.rightChanges, addA2);
   });
@@ -2701,8 +2744,10 @@ function testDocApi() {
           removeRows: [],
           addRows: [2],
           columnDeltas: {
-            A: { [1]: [['a1'], ['A1']],
+            A: {
-                 [2]: [null, ['a2']] },
+              [1]: [['a1'], ['A1']],
+              [2]: [null, ['a2']]
+            },
             B: {[2]: [null, ['b2']]},
             manualSort: {[2]: [null, [2]]},
           },
@@ -2748,7 +2793,7 @@ function testDocApi() {
     await check({eventTypes: 0}, 400, /url is missing/, /eventTypes is not an array/);
     await check({eventTypes: []}, 400, /url is missing/);
     await check({eventTypes: [], url: "https://example.com"}, 400, /eventTypes must be a non-empty array/);
-    await check({eventTypes: ["foo"], url: "https://example.com"}, 400, /eventTypes\[0\] is none of "add", "update"/);
+    await check({eventTypes: ["foo"], url: "https://example.com"}, 400, /eventTypes\[0] is none of "add", "update"/);
     await check({eventTypes: ["add"]}, 400, /url is missing/);
     await check({eventTypes: ["add"], url: "https://evil.com"}, 403, /Provided url is forbidden/);
     await check({eventTypes: ["add"], url: "http://example.com"}, 403, /Provided url is forbidden/);  // not https
@@ -2825,7 +2870,9 @@ function testDocApi() {
     let redisClient: RedisClient;
 
     before(async function () {
-      if (!process.env.TEST_REDIS_URL) { this.skip(); }
+      if (!process.env.TEST_REDIS_URL) {
+        this.skip();
+      }
       redisClient = createClient(process.env.TEST_REDIS_URL);
     });
 
@@ -2951,6 +2998,7 @@ function testDocApi() {
       assert.equal(nextHour, `doc-myDocId-periodicApiUsage-2000-01-01T00`);
 
       const usage = new LRUCache<string, number>({max: 1024});
+
       function check(expected: string[] | undefined) {
         assert.deepEqual(getDocApiUsageKeysToIncr(docId, usage, dailyMax, m), expected);
       }
@@ -2979,7 +3027,9 @@ function testDocApi() {
     });
 
     after(async function () {
-      if (!process.env.TEST_REDIS_URL) { this.skip(); }
+      if (!process.env.TEST_REDIS_URL) {
+        this.skip();
+      }
       await redisClient.quitAsync();
     });
   });
@@ -3194,7 +3244,9 @@ function testDocApi() {
       before(async function () {
         this.timeout(30000);
         // We rely on the REDIS server in this test.
-        if (!process.env.TEST_REDIS_URL) { this.skip(); }
+        if (!process.env.TEST_REDIS_URL) {
+          this.skip();
+        }
         requests = {
           "add,update": [],
           "add": [],
@@ -3210,7 +3262,9 @@ function testDocApi() {
       });
 
       after(async function () {
-        if (!process.env.TEST_REDIS_URL) { this.skip(); }
+        if (!process.env.TEST_REDIS_URL) {
+          this.skip();
+        }
         await redisMonitor.quitAsync();
       });
 
@@ -3847,7 +3901,8 @@ function testDocApi() {
         const addRowProm = doc.addRows("Table1", {
           A: arrayRepeat(5, 100), // there are 2 webhooks, so 5 events per webhook.
           B: arrayRepeat(5, true)
-        }).catch(() => {});
+        }).catch(() => {
+        });
         // WARNING: we can't wait for it, as the Webhooks will literally stop the document, and wait
         // for the queue to drain. So we will carefully go further, and wait for the queue to drain.
 
@@ -3941,7 +3996,9 @@ function testDocApi() {
               stats = await readStats(docId);
               assert.equal(stats.length, 1);
               assert.equal(stats[0].id, webhook.webhookId);
-              if (expectedFieldsCallback) { expectedFieldsCallback(expectedFields); }
+              if (expectedFieldsCallback) {
+                expectedFieldsCallback(expectedFields);
+              }
               assert.deepEqual(stats[0].fields, {...expectedFields, ...fields});
               if (fields.tableId) {
                 savedTableId = fields.tableId;
@@ -3974,7 +4031,7 @@ function testDocApi() {
 
           await check({eventTypes: ['add', 'update']}, 200);
           await check({eventTypes: []}, 400, "eventTypes must be a non-empty array");
-          await check({eventTypes: ["foo"]}, 400, /eventTypes\[0\] is none of "add", "update"/);
+          await check({eventTypes: ["foo"]}, 400, /eventTypes\[0] is none of "add", "update"/);
 
           await check({isReadyColumn: null}, 200);
           await check({isReadyColumn: "bar"}, 404, `Column not found "bar"`);
@@ -4087,6 +4144,7 @@ interface WebhookRequests {
 }
 
 const ORG_NAME = 'docs-1';
+
 function setup(name: string, cb: () => Promise<void>) {
   let api: UserAPIImpl;
 
@@ -4128,129 +4186,9 @@ async function getWorkspaceId(api: UserAPIImpl, name: string) {
   return workspaces.find((w) => w.name === name)!.id;
 }
 
-async function startServer(serverTypes: string, _homeUrl?: string): Promise<TestServer> {
+// TODO: deal with safe port allocation
-  const server = new TestServer(serverTypes);
-  await server.start(_homeUrl);
-  return server;
-}
-
 const webhooksTestPort = 34365;
 
-class TestServer {
-  public testingSocket: string;
-  public testingHooks: TestingHooksClient;
-  public serverUrl: string;
-  public stopped = false;
-
-  private _server: ChildProcess;
-  private _exitPromise: Promise<number|string>;
-
-  constructor(private _serverTypes: string) {}
-
-  public async start(_homeUrl?: string) {
-
-    // put node logs into files with meaningful name that relate to the suite name and server type
-    const fixedName = this._serverTypes.replace(/,/, '_');
-    const nodeLogPath = path.join(tmpDir, `${suitename}-${fixedName}-node.log`);
-    const nodeLogFd = await fse.open(nodeLogPath, 'a');
-    const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd;
-
-    // use a path for socket that relates to suite name and server types
-    this.testingSocket = path.join(tmpDir, `${suitename}-${fixedName}.socket`);
-
-    // env
-    const env = {
-      GRIST_DATA_DIR: dataDir,
-      GRIST_INST_DIR: tmpDir,
-      GRIST_SERVERS: this._serverTypes,
-      // with port '0' no need to hard code a port number (we can use testing hooks to find out what
-      // port server is listening on).
-      GRIST_PORT: '0',
-      GRIST_TESTING_SOCKET: this.testingSocket,
-      GRIST_DISABLE_S3: 'true',
-      REDIS_URL: process.env.TEST_REDIS_URL,
-      APP_HOME_URL: _homeUrl,
-      ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
-      GRIST_ALLOWED_HOSTS: `example.com,localhost`,
-      GRIST_TRIGGER_WAIT_DELAY: '100',
-      // this is calculated value, some tests expect 4 attempts and some will try 3 times
-      GRIST_TRIGGER_MAX_ATTEMPTS: '4',
-      GRIST_MAX_QUEUE_SIZE: '10',
-      ...process.env
-    };
-
-    const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
-    this._server = spawn('node', [main, '--testingHooks'], {
-      env,
-      stdio: ['inherit', serverLog, serverLog]
-    });
-
-    this._exitPromise = exitPromise(this._server);
-
-    // Try to be more helpful when server exits by printing out the tail of its log.
-    this._exitPromise.then((code) => {
-        if (this._server.killed) { return; }
-        log.error("Server died unexpectedly, with code", code);
-        const output = execFileSync('tail', ['-30', nodeLogPath]);
-        log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`);
-      })
-      .catch(() => undefined);
-
-    await this._waitServerReady(30000);
-    log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
-  }
-
-  public async stop() {
-    if (this.stopped) { return; }
-    log.info("Stopping node server: " + this._serverTypes);
-    this.stopped = true;
-    this._server.kill();
-    this.testingHooks.close();
-    await this._exitPromise;
-  }
-
-  public async isServerReady(): Promise<boolean> {
-    // Let's wait for the testingSocket to be created, then get the port the server is listening on,
-    // and then do an api check. This approach allow us to start server with GRIST_PORT set to '0',
-    // which will listen on first available port, removing the need to hard code a port number.
-    try {
-
-      // wait for testing socket
-      while (!(await fse.pathExists(this.testingSocket))) {
-        await delay(200);
-      }
-
-      // create testing hooks and get own port
-      this.testingHooks = await connectTestingHooks(this.testingSocket);
-      const port: number = await this.testingHooks.getOwnPort();
-      this.serverUrl = `http://localhost:${port}`;
-
-      // wait for check
-      return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
-    } catch (err) {
-      return false;
-    }
-  }
-
-
-  private async _waitServerReady(ms: number) {
-    // It's important to clear the timeout, because it can prevent node from exiting otherwise,
-    // which is annoying when running only this test for debugging.
-    let timeout: any;
-    const maxDelay = new Promise((resolve) => {
-      timeout = setTimeout(resolve, 30000);
-    });
-    try {
-      await Promise.race([
-        this.isServerReady(),
-        this._exitPromise.then(() => { throw new Error("Server exited while waiting for it"); }),
-        maxDelay,
-      ]);
-    } finally {
-      clearTimeout(timeout);
-    }
-  }
-}
 
 async function setupDataDir(dir: string) {
   // we'll be serving Hello.grist content for various document ids, so let's make copies of it in
@@ -4263,42 +4201,3 @@ async function setupDataDir(dir: string) {
     'ApiDataRecordsTest.grist',
     path.resolve(dir, docIds.ApiDataRecordsTest + '.grist'));
 }
-
-/**
- * Helper that creates a promise that can be resolved from outside.
- */
-function signal() {
-  let resolve: null | ((data: any) => void) = null;
-  let promise: null | Promise<any> = null;
-  let called = false;
-  return {
-    emit(data: any) {
-      if (!resolve) {
-        throw new Error("signal.emit() called before signal.reset()");
-      }
-      called = true;
-      resolve(data);
-    },
-    async wait() {
-      if (!promise) {
-        throw new Error("signal.wait() called before signal.reset()");
-      }
-      const proms = Promise.race([promise, delay(2000).then(() => { throw new Error("signal.wait() timed out"); })]);
-      return await proms;
-    },
-    async waitAndReset() {
-      try {
-        return await this.wait();
-      } finally {
-        this.reset();
-      }
-    },
-    called() {
-      return called;
-    },
-    reset() {
-      called = false;
-      promise = new Promise((res) => { resolve = res; });
-    }
-  };
-}

test/server/lib/ProxyAgent.ts

@@ -0,0 +1,38 @@
+import {proxyAgent} from "app/server/utils/ProxyAgent";
+import {assert} from "chai";
+import {HttpsProxyAgent} from "https-proxy-agent";
+import {HttpProxyAgent} from "http-proxy-agent";
+import {EnvironmentSnapshot} from "test/server/testUtils";
+
+
+describe("ProxyAgent", function () {
+  let oldEnv: EnvironmentSnapshot;
+  before(() => {
+    oldEnv = new EnvironmentSnapshot();
+  });
+
+  after(() => {
+    oldEnv.restore();
+  });
+
+  it("should be undefined if no proxy is configured", async function () {
+    delete process.env.GRIST_HTTPS_PROXY;
+    const httpProxy = proxyAgent(new URL("http://localhost:3000"));
+    const httpsProxy = proxyAgent(new URL("https://localhost:3000"));
+
+    assert.equal(httpProxy, undefined);
+    assert.equal(httpsProxy, undefined);
+  });
+
+  it("should be https proxy if proxy is configured and address is https", async function () {
+    process.env.GRIST_HTTPS_PROXY = "https://localhost:9000";
+    const httpsProxy = proxyAgent(new URL("https://localhost:3000"));
+    assert.instanceOf(httpsProxy, HttpsProxyAgent);
+  });
+
+  it("should be https proxy if proxy is configured and address is https", async function () {
+    process.env.GRIST_HTTPS_PROXY = "https://localhost:9000";
+    const httpsProxy = proxyAgent(new URL("http://localhost:3000"));
+    assert.instanceOf(httpsProxy, HttpProxyAgent);
+  });
+});

test/server/lib/Webhooks-Proxy.ts

@@ -0,0 +1,341 @@
+import {UserAPIImpl} from 'app/common/UserAPI';
+import {WebhookSubscription} from 'app/server/lib/DocApi';
+import log from 'app/server/lib/log';
+import axios from 'axios';
+import * as bodyParser from 'body-parser';
+import {assert} from 'chai';
+import FormData from 'form-data';
+import fetch from 'node-fetch';
+import {tmpdir} from 'os';
+import * as path from 'path';
+import {createClient} from 'redis';
+import {configForUser} from 'test/gen-server/testUtils';
+import {serveSomething, Serving} from 'test/server/customUtil';
+import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
+import {prepareFilesystemDirectoryForTests} from 'test/server/lib/helpers/PrepareFilesystemDirectoryForTests';
+import {signal} from 'test/server/lib/helpers/Signal';
+import {TestProxyServer} from 'test/server/lib/helpers/TestProxyServer';
+import {TestServer} from 'test/server/lib/helpers/TestServer';
+import * as testUtils from 'test/server/testUtils';
+import clone = require('lodash/clone');
+
+const chimpy = configForUser('Chimpy');
+
+
+// some doc ids
+const docIds: { [name: string]: string } = {
+  ApiDataRecordsTest: 'sample_7',
+  Timesheets: 'sample_13',
+  Bananas: 'sample_6',
+  Antartic: 'sample_11'
+};
+
+let dataDir: string;
+let suitename: string;
+let serverUrl: string;
+let userApi: UserAPIImpl;
+
+async function cleanRedisDatabase() {
+  const cli = createClient(process.env.TEST_REDIS_URL);
+  await cli.flushdbAsync();
+  await cli.quitAsync();
+}
+
+function backupEnvironmentVariables() {
+  let oldEnv: NodeJS.ProcessEnv;
+  before(() => {
+    oldEnv = clone(process.env);
+  });
+  after(() => {
+    Object.assign(process.env, oldEnv);
+  });
+}
+
+/*
+TODO: this hardcoded port numbers might cause conflicts in parallel tests executions. replace with someone more generic
+*/
+const webhooksTestPort = 34365;
+const webhooksTestProxyPort = 22335;
+
+describe('Webhooks proxy configuration', function () {
+  // A testDir of the form grist_test_{USER}_{SERVER_NAME}
+  // - its a directory that will be base for all test related files and activities
+  const username = process.env.USER || "nobody";
+  const tmpDir = path.join(tmpdir(), `grist_test_${username}_docapi_webhooks_proxy`);
+  let home: TestServer;
+  let docs: TestServer;
+
+  this.timeout(30000);
+  testUtils.setTmpLogLevel('debug');
+  // test might override environment values, therefore we need to backup current ones to restore them later
+  backupEnvironmentVariables();
+
+  function setupMockServers(name: string, tmpDir: string, cb: () => Promise<void>) {
+    let api: UserAPIImpl;
+
+    before(async function () {
+      suitename = name;
+      await cb();
+
+      // create TestDoc as an empty doc into Private workspace
+      userApi = api = makeUserApi(ORG_NAME, home.serverUrl);
+      const wid = await getWorkspaceId(api, 'Private');
+      docIds.TestDoc = await api.newDoc({name: 'TestDoc'}, wid);
+    });
+
+    after(async function () {
+      // remove TestDoc
+      await api.deleteDoc(docIds.TestDoc);
+      delete docIds.TestDoc;
+
+      // stop all servers
+      await home.stop();
+      await docs.stop();
+    });
+  }
+
+
+  describe('Proxy is configured', function () {
+    runServerConfigurations({GRIST_HTTPS_PROXY:`http://localhost:${webhooksTestProxyPort}`}, ()=>testWebhookProxy(true));
+  });
+
+  describe('Proxy not configured', function () {
+    runServerConfigurations({GRIST_HTTPS_PROXY:undefined}, ()=>testWebhookProxy(false));
+
+  });
+
+
+  function runServerConfigurations(additionaEnvConfiguration: object, subTestCall: Function) {
+    additionaEnvConfiguration = {
+      ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+      GRIST_DATA_DIR: dataDir,
+      ...additionaEnvConfiguration
+    };
+
+    before(async function () {
+      // Clear redis test database if redis is in use.
+      if (process.env.TEST_REDIS_URL) {
+        await cleanRedisDatabase();
+      }
+      await prepareFilesystemDirectoryForTests(tmpDir);
+      await prepareDatabase(tmpDir);
+    });
+    /**
+     * Doc api tests are run against three different setup:
+     *  - a merged server: a single server serving both as a home and doc worker
+     *  - two separated servers: requests are sent to a home server which then forward them to a doc worker
+     *  - a doc worker: request are sent directly to the doc worker (note that even though it is not
+     *    used for testing we starts anyway a home server, needed for setting up the test cases)
+     *
+     *  Future tests must be added within the testDocApi() function.
+     */
+    describe("should work with a merged server", async () => {
+      setupMockServers('merged', tmpDir, async () => {
+        home = docs = await TestServer.startServer('home,docs', tmpDir, suitename, additionaEnvConfiguration);
+        serverUrl = home.serverUrl;
+      });
+      subTestCall();
+    });
+
+    // the way these tests are written, non-merged server requires redis.
+    if (process.env.TEST_REDIS_URL) {
+      describe("should work with a home server and a docworker", async () => {
+        setupMockServers('separated', tmpDir, async () => {
+          home = await TestServer.startServer('home', tmpDir, suitename, additionaEnvConfiguration);
+          docs = await TestServer.startServer('docs', tmpDir, suitename, additionaEnvConfiguration, home.serverUrl);
+          serverUrl = home.serverUrl;
+        });
+        subTestCall();
+      });
+
+
+      describe("should work directly with a docworker", async () => {
+        setupMockServers('docs', tmpDir, async () => {
+          home = await TestServer.startServer('home', tmpDir, suitename, additionaEnvConfiguration);
+          docs = await TestServer.startServer('docs', tmpDir, suitename, additionaEnvConfiguration, home.serverUrl);
+          serverUrl = docs.serverUrl;
+        });
+        subTestCall();
+      });
+    }
+  }
+
+  function testWebhookProxy(shouldProxyBeCalled: boolean) {
+    describe('calling registered webhooks after data update', function () {
+
+      let serving: Serving;  // manages the test webhook server
+      let testProxyServer: TestProxyServer;  // manages the test webhook server
+
+
+      let redisMonitor: any;
+
+
+      // Create couple of promises that can be used to monitor
+      // if the endpoint was called.
+      const successCalled = signal();
+      const notFoundCalled = signal();
+
+
+      async function autoSubscribe(
+        endpoint: string, docId: string, options?: {
+          tableId?: string,
+          isReadyColumn?: string | null,
+          eventTypes?: string[]
+        }) {
+        // Subscribe helper that returns a method to unsubscribe.
+        const data = await subscribe(endpoint, docId, options);
+        return () => unsubscribe(docId, data, options?.tableId ?? 'Table1');
+      }
+
+      function unsubscribe(docId: string, data: any, tableId = 'Table1') {
+        return axios.post(
+          `${serverUrl}/api/docs/${docId}/tables/${tableId}/_unsubscribe`,
+          data, chimpy
+        );
+      }
+
+      async function subscribe(endpoint: string, docId: string, options?: {
+        tableId?: string,
+        isReadyColumn?: string | null,
+        eventTypes?: string[]
+      }) {
+        // Subscribe helper that returns a method to unsubscribe.
+        const {data, status} = await axios.post(
+          `${serverUrl}/api/docs/${docId}/tables/${options?.tableId ?? 'Table1'}/_subscribe`,
+          {
+            eventTypes: options?.eventTypes ?? ['add', 'update'],
+            url: `${serving.url}/${endpoint}`,
+            isReadyColumn: options?.isReadyColumn === undefined ? 'B' : options?.isReadyColumn
+          }, chimpy
+        );
+        assert.equal(status, 200);
+        return data as WebhookSubscription;
+      }
+
+      async function clearQueue(docId: string) {
+        const deleteResult = await axios.delete(
+          `${serverUrl}/api/docs/${docId}/webhooks/queue`, chimpy
+        );
+        assert.equal(deleteResult.status, 200);
+      }
+
+
+      before(async function () {
+        this.timeout(30000);
+        serving = await serveSomething(app => {
+
+          app.use(bodyParser.json());
+          app.post('/200', ({body}, res) => {
+            successCalled.emit(body[0].A);
+            res.sendStatus(200);
+            res.end();
+          });
+          app.post('/404', ({body}, res) => {
+            notFoundCalled.emit(body[0].A);
+            res.sendStatus(404); // Webhooks treats it as an error and will retry. Probably it shouldn't work this way.
+            res.end();
+          });
+        }, webhooksTestPort);
+        testProxyServer = await TestProxyServer.Prepare(webhooksTestProxyPort);
+      });
+
+      after(async function () {
+        await serving.shutdown();
+        await testProxyServer.dispose();
+      });
+
+      before(async function () {
+        this.timeout(30000);
+
+        if (process.env.TEST_REDIS_URL) {
+
+          redisMonitor = createClient(process.env.TEST_REDIS_URL);
+        }
+      });
+
+      after(async function () {
+        if (process.env.TEST_REDIS_URL) {
+          await redisMonitor.quitAsync();
+        }
+      });
+
+      if (shouldProxyBeCalled) {
+        it("Should call proxy", async function () {
+          //Run standard subscribe-modify data-check response - unsubscribe scenario, we are not mutch
+          // intrested in it, only want to check if proxy was used
+          await runTestCase();
+          assert.isTrue(testProxyServer.wasProxyCalled());
+        });
+      } else {
+        it("Should not call proxy", async function () {
+          //Run standard subscribe-modify data-check response - unsubscribe scenario, we are not mutch
+          // intrested in it, only want to check if proxy was used
+          await runTestCase();
+          assert.isFalse(testProxyServer.wasProxyCalled());
+        });
+      }
+
+
+      async function runTestCase() {
+        //Create a test document.
+        const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
+        const docId = await userApi.newDoc({name: 'testdoc2'}, ws1);
+        const doc = userApi.getDocAPI(docId);
+        await axios.post(`${serverUrl}/api/docs/${docId}/apply`, [
+          ['ModifyColumn', 'Table1', 'B', {type: 'Bool'}],
+        ], chimpy);
+
+        // Try to clear the queue, even if it is empty.
+        await clearQueue(docId);
+
+        const cleanup: (() => Promise<any>)[] = [];
+
+        // Subscribe a valid webhook endpoint.
+        cleanup.push(await autoSubscribe('200', docId));
+        // Subscribe an invalid webhook endpoint.
+        cleanup.push(await autoSubscribe('404', docId));
+
+        // Prepare signals, we will be waiting for those two to be called.
+        successCalled.reset();
+        notFoundCalled.reset();
+        // Trigger both events.
+        await doc.addRows("Table1", {
+          A: [1],
+          B: [true],
+        });
+
+        // Wait for both of them to be called (this is correct order)
+        await successCalled.waitAndReset();
+        await notFoundCalled.waitAndReset();
+
+        // Broken endpoint will be called multiple times here, and any subsequent triggers for working
+        // endpoint won't be called.
+        await notFoundCalled.waitAndReset();
+
+        // But the working endpoint won't be called more then once.
+        assert.isFalse(successCalled.called());
+
+        //Cleanup all
+        await Promise.all(cleanup.map(fn => fn())).finally(() => cleanup.length = 0);
+        await clearQueue(docId);
+      }
+    });
+  }
+});
+
+const ORG_NAME = 'docs-1';
+
+
+function makeUserApi(org: string, homeServerUrl: string, user?: string) {
+  return new UserAPIImpl(`${homeServerUrl}/o/${org}`, {
+    headers: {Authorization: `Bearer api_key_for_${user || 'chimpy'}`},
+    fetch: fetch as any,
+    newFormData: () => new FormData() as any,
+    logger: log
+  });
+}
+
+async function getWorkspaceId(api: UserAPIImpl, name: string) {
+  const workspaces = await api.getOrgWorkspaces('current');
+  return workspaces.find((w) => w.name === name)!.id;
+}

test/server/lib/helpers/PrepareDatabase.ts

@@ -0,0 +1,17 @@
+import path from "path";
+import * as testUtils from "test/server/testUtils";
+import {execFileSync} from "child_process";
+
+export async function prepareDatabase(tempDirectory: string) {
+  // Let's create a sqlite db that we can share with servers that run in other processes, hence
+  // not an in-memory db. Running seed.ts directly might not take in account the most recent value
+  // for TYPEORM_DATABASE, because ormconfig.js may already have been loaded with a different
+  // configuration (in-memory for instance). Spawning a process is one way to make sure that the
+  // latest value prevail.
+  process.env.TYPEORM_DATABASE = path.join(tempDirectory, 'landing.db');
+  const seed = await testUtils.getBuildFile('test/gen-server/seed.js');
+  execFileSync('node', [seed, 'init'], {
+    env: process.env,
+    stdio: 'inherit'
+  });
+}

test/server/lib/helpers/PrepareFilesystemDirectoryForTests.ts

@@ -0,0 +1,9 @@
+import * as fse from "fs-extra";
+import log from "app/server/lib/log";
+
+export async function prepareFilesystemDirectoryForTests(directory: string) {
+  // Create the tmp dir removing any previous one
+  await fse.remove(directory);
+  await fse.mkdirs(directory);
+  log.warn(`Test logs and data are at: ${directory}/`);
+}

test/server/lib/helpers/Signal.ts

@@ -0,0 +1,44 @@
+import {delay} from "bluebird";
+
+/**
+ * Helper that creates a promise that can be resolved from outside.
+ */
+export function signal() {
+    let resolve: null | ((data: any) => void) = null;
+    let promise: null | Promise<any> = null;
+    let called = false;
+    return {
+        emit(data: any) {
+            if (!resolve) {
+                throw new Error("signal.emit() called before signal.reset()");
+            }
+            called = true;
+            resolve(data);
+        },
+        async wait() {
+            if (!promise) {
+                throw new Error("signal.wait() called before signal.reset()");
+            }
+            const proms = Promise.race([promise, delay(2000).then(() => {
+                throw new Error("signal.wait() timed out");
+            })]);
+            return await proms;
+        },
+        async waitAndReset() {
+            try {
+                return await this.wait();
+            } finally {
+                this.reset();
+            }
+        },
+        called() {
+            return called;
+        },
+        reset() {
+            called = false;
+            promise = new Promise((res) => {
+                resolve = res;
+            });
+        }
+    };
+}

test/server/lib/helpers/TestProxyServer.ts

@@ -0,0 +1,46 @@
+import {serveSomething, Serving} from "test/server/customUtil";
+import * as bodyParser from "body-parser";
+import {Request, Response} from "express-serve-static-core";
+import axios from "axios";
+
+export class TestProxyServer {
+  public static async Prepare(portNumber: number): Promise<TestProxyServer> {
+    const server = new TestProxyServer();
+    await server._prepare(portNumber);
+    return server;
+
+  }
+
+  private _proxyCallsCounter: number = 0;
+  private _proxyServing: Serving;
+
+  private constructor() {
+  }
+
+  public wasProxyCalled(): boolean {
+    return this._proxyCallsCounter > 0;
+  }
+
+  public async dispose() {
+    await this._proxyServing.shutdown();
+  }
+
+  private async _prepare(portNumber: number) {
+    this._proxyServing = await serveSomething(app => {
+      app.use(bodyParser.json());
+      app.all('*', async (req: Request, res: Response) => {
+        this._proxyCallsCounter += 1;
+        let responseCode;
+        try {
+          const axiosResponse = await axios.post(req.url, req.body);
+          responseCode = axiosResponse.status;
+        } catch (error: any) {
+          responseCode = error.response.status;
+        }
+        res.sendStatus(responseCode);
+        res.end();
+        //next();
+      });
+    }, portNumber);
+  }
+}

test/server/lib/helpers/TestServer.ts

@@ -0,0 +1,143 @@
+import {connectTestingHooks, TestingHooksClient} from "app/server/lib/TestingHooks";
+import {ChildProcess, execFileSync, spawn} from "child_process";
+import path from "path";
+import * as fse from "fs-extra";
+import * as testUtils from "test/server/testUtils";
+import {exitPromise} from "app/server/lib/serverUtils";
+import log from "app/server/lib/log";
+import {delay} from "bluebird";
+import fetch from "node-fetch";
+
+export class TestServer {
+  public static async startServer
+  (serverTypes: string,
+   tempDirectory: string,
+   suitename: string,
+   additionalConfig?: Object,
+   _homeUrl?: string): Promise<TestServer> {
+    const server = new TestServer(serverTypes, tempDirectory, suitename);
+    // Override some env variables in server configuration to serve our test purpose:
+    const customEnv = {
+      ...additionalConfig};
+    await server.start(_homeUrl, customEnv);
+    return server;
+  }
+
+  public testingSocket: string;
+  public testingHooks: TestingHooksClient;
+  public serverUrl: string;
+  public stopped = false;
+
+  private _server: ChildProcess;
+  private _exitPromise: Promise<number | string>;
+
+  private readonly _defaultEnv;
+
+  constructor(private _serverTypes: string, private _tmpDir: string, private _suiteName: string) {
+    this._defaultEnv = {
+      GRIST_INST_DIR: this._tmpDir,
+      GRIST_SERVERS: this._serverTypes,
+      // with port '0' no need to hard code a port number (we can use testing hooks to find out what
+      // port server is listening on).
+      GRIST_PORT: '0',
+      GRIST_DISABLE_S3: 'true',
+      REDIS_URL: process.env.TEST_REDIS_URL,
+      GRIST_ALLOWED_HOSTS: `example.com,localhost`,
+      GRIST_TRIGGER_WAIT_DELAY: '100',
+      // this is calculated value, some tests expect 4 attempts and some will try 3 times
+      GRIST_TRIGGER_MAX_ATTEMPTS: '4',
+      GRIST_MAX_QUEUE_SIZE: '10',
+      ...process.env
+    };
+  }
+  public async start(_homeUrl?: string, customEnv?: object) {
+    // put node logs into files with meaningful name that relate to the suite name and server type
+    const fixedName = this._serverTypes.replace(/,/, '_');
+    const nodeLogPath = path.join(this._tmpDir, `${this._suiteName}-${fixedName}-node.log`);
+    const nodeLogFd = await fse.open(nodeLogPath, 'a');
+    const serverLog = process.env.VERBOSE ? 'inherit' : nodeLogFd;
+    // use a path for socket that relates to suite name and server types
+    this.testingSocket = path.join(this._tmpDir, `${this._suiteName}-${fixedName}.socket`);
+    const env = {
+      APP_HOME_URL: _homeUrl,
+      GRIST_TESTING_SOCKET: this.testingSocket,
+      ...this._defaultEnv,
+      ...customEnv
+    };
+    const main = await testUtils.getBuildFile('app/server/mergedServerMain.js');
+    this._server = spawn('node', [main, '--testingHooks'], {
+      env,
+      stdio: ['inherit', serverLog, serverLog]
+    });
+    this._exitPromise = exitPromise(this._server);
+
+    // Try to be more helpful when server exits by printing out the tail of its log.
+    this._exitPromise.then((code) => {
+      if (this._server.killed) {
+        return;
+      }
+      log.error("Server died unexpectedly, with code", code);
+      const output = execFileSync('tail', ['-30', nodeLogPath]);
+      log.info(`\n===== BEGIN SERVER OUTPUT ====\n${output}\n===== END SERVER OUTPUT =====`);
+    })
+      .catch(() => undefined);
+
+    await this._waitServerReady();
+    log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
+  }
+
+  public async stop() {
+    if (this.stopped) {
+      return;
+    }
+    log.info("Stopping node server: " + this._serverTypes);
+    this.stopped = true;
+    this._server.kill();
+    this.testingHooks.close();
+    await this._exitPromise;
+  }
+
+  public async isServerReady(): Promise<boolean> {
+    // Let's wait for the testingSocket to be created, then get the port the server is listening on,
+    // and then do an api check. This approach allow us to start server with GRIST_PORT set to '0',
+    // which will listen on first available port, removing the need to hard code a port number.
+    try {
+
+      // wait for testing socket
+      while (!(await fse.pathExists(this.testingSocket))) {
+        await delay(200);
+      }
+
+      // create testing hooks and get own port
+      this.testingHooks = await connectTestingHooks(this.testingSocket);
+      const port: number = await this.testingHooks.getOwnPort();
+      this.serverUrl = `http://localhost:${port}`;
+
+      // wait for check
+      return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
+    } catch (err) {
+      return false;
+    }
+  }
+
+
+  private async _waitServerReady() {
+    // It's important to clear the timeout, because it can prevent node from exiting otherwise,
+    // which is annoying when running only this test for debugging.
+    let timeout: any;
+    const maxDelay = new Promise((resolve) => {
+      timeout = setTimeout(resolve, 30000);
+    });
+    try {
+      await Promise.race([
+        this.isServerReady(),
+        this._exitPromise.then(() => {
+          throw new Error("Server exited while waiting for it");
+        }),
+        maxDelay,
+      ]);
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+}