Add tests

app/server/lib/uploads.ts

@@ -420,13 +420,7 @@ export async function fetchDoc(
   template: boolean
 ): Promise<UploadResult> {
   // Prepare headers that preserve credentials of current user.
-  const headers = getTransitiveHeaders(req, { includeOrigin: false });
-
-  // Passing the Origin header would serve no purpose here, as we are
-  // constructing an internal request to fetch from our own doc worker
-  // URL. Indeed, it may interfere, as it could incur a CORS check in
-  // `trustOrigin`, which we do not need.
-  delete headers.Origin;
+  const headers = getTransitiveHeaders(req, { includeOrigin: false }); // NO EFFECT
 
   // Find the doc worker responsible for the document we wish to copy.
   // The backend needs to be well configured for this to work.

test/server/lib/DocApi.ts

@@ -35,7 +35,7 @@ import {serveSomething, Serving} from 'test/server/customUtil';
 import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
 import {prepareFilesystemDirectoryForTests} from 'test/server/lib/helpers/PrepareFilesystemDirectoryForTests';
 import {signal} from 'test/server/lib/helpers/Signal';
-import {TestServer} from 'test/server/lib/helpers/TestServer';
+import {TestServer, TestServerProxy} from 'test/server/lib/helpers/TestServer';
 import * as testUtils from 'test/server/testUtils';
 import {waitForIt} from 'test/server/wait';
 import defaultsDeep = require('lodash/defaultsDeep');
@@ -48,6 +48,8 @@ const charon = configForUser('Charon');
 const nobody = configForUser('Anonymous');
 const support = configForUser('support');
 
+const accounts = {chimpy, kiwi, charon, nobody, support};
+
 // some doc ids
 const docIds: { [name: string]: string } = {
   ApiDataRecordsTest: 'sampledocid_7',
@@ -158,6 +160,51 @@ describe('DocApi', function () {
       testDocApi();
     });
 
+    describe("should work behind a proxy", async () => {
+      let proxy: TestServerProxy;
+
+      const originalHeaders = new WeakMap<AxiosRequestConfig, AxiosRequestConfig<any>["headers"]>();
+      function iterateOverAccountHeaders (
+        cb: (account: AxiosRequestConfig) => AxiosRequestConfig<any>["headers"]
+      ) {
+        for (const account of Object.values(accounts)) {
+          if (account.headers) {
+            account.headers = cb(account);
+          }
+        }
+      }
+      setup('separated', async () => {
+        proxy = new TestServerProxy();
+        const additionalEnvConfiguration = {
+          ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+          GRIST_DATA_DIR: dataDir,
+          APP_HOME_URL: await proxy.getServerUrl()
+        };
+        home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
+        docs = await TestServer.startServer('docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl);
+
+        proxy.start(home, docs);
+
+        homeUrl = serverUrl = await proxy.getServerUrl();
+        iterateOverAccountHeaders(account => {
+          originalHeaders.set(account, account.headers);
+          const newHeaders = _.clone(account.headers)!;
+          newHeaders.Origin = serverUrl;
+          return newHeaders;
+        });
+        hasHomeApi = true;
+      });
+
+      after(() => {
+        proxy.stop();
+        iterateOverAccountHeaders((account) => {
+          return originalHeaders.get(account)!;
+        });
+      });
+
+      testDocApi();
+    });
+
     describe("should work directly with a docworker", async () => {
       setup('docs', async () => {
         const additionalEnvConfiguration = {
@@ -2615,6 +2662,18 @@ function testDocApi() {
     await worker1.copyDoc(docId, undefined, 'copy');
   });
 
+  it("POST /docs/{did} with sourceDocId copies a document", async function () {
+    const chimpyWs = await userApi.newWorkspace({name: "Chimpy's Workspace"}, ORG_NAME);
+    const resp = await axios.post(`${serverUrl}/api/docs`, {
+      sourceDocumentId: docIds.TestDoc,
+      documentName: 'copy of TestDoc',
+      asTemplate: false,
+      workspaceId: chimpyWs
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    assert.isString(resp.data);
+  });
+
   it("GET /docs/{did}/download/csv serves CSV-encoded document", async function () {
     const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
     assert.equal(resp.status, 200);
@@ -2859,7 +2918,7 @@ function testDocApi() {
       this.skip();
     }
     // Prepare an API for a different user.
-    const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
+    const kiwiApi = new UserAPIImpl(`${homeUrl}/o/Fish`, {
       headers: {Authorization: 'Bearer api_key_for_kiwi'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
@@ -2966,7 +3025,7 @@ function testDocApi() {
     // Make two documents with same urlId
     const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
     const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
-    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
+    const nasaApi = new UserAPIImpl(`${homeUrl}/o/nasa`, {
       headers: {Authorization: 'Bearer api_key_for_chimpy'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
@@ -2997,7 +3056,7 @@ function testDocApi() {
     // Make two documents
     const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
     const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
-    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
+    const nasaApi = new UserAPIImpl(`${homeUrl}/o/nasa`, {
       headers: {Authorization: 'Bearer api_key_for_chimpy'},
       fetch: fetch as any,
       newFormData: () => new FormData() as any,
@@ -3125,11 +3184,16 @@ function testDocApi() {
   });
 
   it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function () {
-    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
-    const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
-    const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
-    const doc1 = userApi.getDocAPI(docId1);
-    const doc2 = userApi.getDocAPI(docId2);
+    // Pass kiwi's headers as it contains both Authorization and Origin headers
+    // if run behind a proxy, so we can ensure that the Origin header check is not made.
+    const chimpyApi = home.makeUserApi(
+      ORG_NAME, 'chimpy', { serverUrl, headers: chimpy.headers as Record<string, string> }
+    );
+    const ws1 = (await chimpyApi.getOrgWorkspaces('current'))[0].id;
+    const docId1 = await chimpyApi.newDoc({name: 'testdoc1'}, ws1);
+    const docId2 = await chimpyApi.newDoc({name: 'testdoc2'}, ws1);
+    const doc1 = chimpyApi.getDocAPI(docId1);
+    const doc2 = chimpyApi.getDocAPI(docId2);
 
     // Stick some content in column A so it has a defined type
     // so diffs are smaller and simpler.
@@ -3327,6 +3391,9 @@ function testDocApi() {
   });
 
   it('doc worker endpoints ignore any /dw/.../ prefix', async function () {
+    if (docs.proxiedServer) {
+      this.skip();
+    }
     const docWorkerUrl = docs.serverUrl;
     let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
     assert.equal(resp.status, 200);

test/server/lib/helpers/TestServer.ts

@@ -1,5 +1,6 @@
 import {connectTestingHooks, TestingHooksClient} from "app/server/lib/TestingHooks";
 import {ChildProcess, execFileSync, spawn} from "child_process";
+import * as http from "http";
 import FormData from 'form-data';
 import path from "path";
 import * as fse from "fs-extra";
@@ -10,6 +11,8 @@ import log from "app/server/lib/log";
 import {delay} from "bluebird";
 import fetch from "node-fetch";
 import {Writable} from "stream";
+import express from "express";
+import { AddressInfo } from "net";
 
 /**
  * This starts a server in a separate process.
@@ -24,18 +27,26 @@ export class TestServer {
     options: {output?: Writable} = {},      // Pipe server output to the given stream
   ): Promise<TestServer> {
 
-    const server = new TestServer(serverTypes, tempDirectory, suitename);
+    const server = new this(serverTypes, tempDirectory, suitename);
     await server.start(_homeUrl, customEnv, options);
     return server;
   }
 
   public testingSocket: string;
   public testingHooks: TestingHooksClient;
-  public serverUrl: string;
   public stopped = false;
+  public get serverUrl() {
+    if (this._proxiedServer) {
+      throw new Error('Direct access to this test server is disallowed');
+    }
+    return this._serverUrl;
+  }
+  public get proxiedServer() { return this._proxiedServer; }
 
   private _server: ChildProcess;
   private _exitPromise: Promise<number | string>;
+  private _serverUrl: string;
+  private _proxiedServer: boolean = false;
 
   private readonly _defaultEnv;
 
@@ -70,6 +81,7 @@ export class TestServer {
     }
     const env = {
       APP_HOME_URL: _homeUrl,
+      APP_HOME_INTERNAL_URL: _homeUrl,
       GRIST_TESTING_SOCKET: this.testingSocket,
       ...this._defaultEnv,
       ...customEnv
@@ -98,7 +110,7 @@ export class TestServer {
       .catch(() => undefined);
 
     await this._waitServerReady();
-    log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
+    log.info(`server ${this._serverTypes} up and listening on ${this._serverUrl}`);
   }
 
   public async stop() {
@@ -126,10 +138,10 @@ export class TestServer {
       // create testing hooks and get own port
       this.testingHooks = await connectTestingHooks(this.testingSocket);
       const port: number = await this.testingHooks.getOwnPort();
-      this.serverUrl = `http://localhost:${port}`;
+      this._serverUrl = `http://localhost:${port}`;
 
       // wait for check
-      return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
+      return (await fetch(`${this._serverUrl}/status/hooks`, {timeout: 1000})).ok;
     } catch (err) {
       log.warn("Failed to initialize server", err);
       return false;
@@ -142,14 +154,28 @@ export class TestServer {
   // Returns the promise for the ChildProcess's signal or exit code.
   public getExitPromise(): Promise<string|number> { return this._exitPromise; }
 
-  public makeUserApi(org: string, user: string = 'chimpy'): UserAPIImpl {
-    return new UserAPIImpl(`${this.serverUrl}/o/${org}`, {
-      headers: {Authorization: `Bearer api_key_for_${user}`},
+  public makeUserApi(
+    org: string,
+    user: string = 'chimpy',
+    {
+      headers = {Authorization: `Bearer api_key_for_${user}`},
+      serverUrl = this._serverUrl,
+    }: {
+      headers?: Record<string, string>
+      serverUrl?: string,
+    } = { headers: undefined, serverUrl: undefined },
+  ): UserAPIImpl {
+    return new UserAPIImpl(`${serverUrl}/o/${org}`, {
+      headers,
       fetch: fetch as unknown as typeof globalThis.fetch,
       newFormData: () => new FormData() as any,
     });
   }
 
+  public disallowDirectAccess() {
+    this._proxiedServer = true;
+  }
+
   private async _waitServerReady() {
     // It's important to clear the timeout, because it can prevent node from exiting otherwise,
     // which is annoying when running only this test for debugging.
@@ -170,3 +196,105 @@ export class TestServer {
     }
   }
 }
+
+// FIXME: found that TestProxyServer exist, what should I do? :'(
+export class TestServerProxy {
+  public static readonly HOSTNAME: string = 'grist-test-proxy.localhost';
+
+  private _stopped: boolean = false;
+  private _app = express();
+  private _server: http.Server;
+  private _address: Promise<AddressInfo>;
+
+  public get stopped() { return this._stopped; }
+
+  public constructor() {
+    this._address = new Promise(resolve => {
+      this._server = this._app.listen(0, TestServerProxy.HOSTNAME, () => {
+        resolve(this._server.address() as AddressInfo);
+      });
+    });
+  }
+
+  public start(homeServer: TestServer, docServer: TestServer) {
+    this._app.all(['/dw/dw1', '/dw/dw1/*'], (oreq, ores) => this._getRequestHandlerFor(docServer));
+    this._app.all('/*', this._getRequestHandlerFor(homeServer));
+    // Forbid now the use of serverUrl property
+    homeServer.disallowDirectAccess();
+    docServer.disallowDirectAccess();
+  }
+
+  public async getAddress() {
+    return this._address;
+  }
+
+  public async getServerUrl() {
+    const address = await this.getAddress();
+    return `http://${TestServerProxy.HOSTNAME}:${address.port}`;
+  }
+
+  public stop() {
+    if (this._stopped) {
+      return;
+    }
+    log.info("Stopping node TestServerProxy");
+    this._stopped = true;
+    this._server.close();
+  }
+
+  private _getRequestHandlerFor(server: TestServer) {
+    const serverUrl = new URL(server.serverUrl);
+
+    return (oreq: express.Request, ores: express.Response) => {
+      const options = {
+        host: serverUrl.hostname,
+        port: serverUrl.port,
+        path: oreq.url,
+        method: oreq.method,
+        headers: oreq.headers,
+      };
+
+      log.debug('[proxy] Requesting: ' + oreq.url);
+
+      const creq = http
+      .request(options, pres => {
+        log.debug('[proxy] Received response for ' + pres.url);
+
+        // set encoding, required?
+        pres.setEncoding('utf8');
+
+        // set http status code based on proxied response
+        ores.writeHead(pres.statusCode ?? 200, pres.statusMessage, pres.headers);
+
+        // wait for data
+        pres.on('data', chunk => {
+          ores.write(chunk);
+        });
+
+        pres.on('close', () => {
+          // closed, let's end client request as well
+          ores.end();
+        });
+
+        pres.on('end', () => {
+          // finished, let's finish client request as well
+          ores.end();
+        });
+      })
+      .on('error', e => {
+        // we got an error
+        console.log(e.message);
+        try {
+          // attempt to set error message and http status
+          ores.writeHead(500);
+          ores.write(e.message);
+        } catch (e) {
+          // ignore
+        }
+        ores.end();
+      });
+
+      oreq.pipe(creq).on('end', () => creq.end());
+    };
+  }
+}