Introduce APP_HOME_INTERNAL_URL and fix duplicate docs (#915)

Context:

On self-hosted instances, some places in the code rely on the fact that we resolves public domains while being behind reverse proxies. This leads to cases where features are not available, such as the "Duplicate document" one.

Bugs that are solved - n self-hosted instances:

Impossible to open templates and tutorials right after having converted them;
Impossible to submit forms since version 1.1.13;
Impossible to restore a previous version of a document (snapshot);
Impossible to copy a document;

Solution:

Introduce the APP_HOME_INTERNAL_URL env variable, which is quite the same as APP_DOC_INTERNAL_URL except that it may point to any home worker;
Make /api/worker/:assignmentId([^/]+)/?* return not only the doc worker public url but also the internal one, and adapt the call points like fetchDocs;
Ensure that the home and doc worker internal urls are trusted by trustOrigin;

---------

Co-authored-by: jordigh <jordigh@octave.org>

app/common/UserAPI.ts

@@ -773,11 +773,11 @@ export class UserAPIImpl extends BaseAPI implements UserAPI {
   }
 
   public async getWorker(key: string): Promise<string> {
-    const json = await this.requestJson(`${this._url}/api/worker/${key}`, {
+    const json = (await this.requestJson(`${this._url}/api/worker/${key}`, {
       method: 'GET',
       credentials: 'include'
-    });
-    return getDocWorkerUrl(this._homeUrl, json);
+    })) as PublicDocWorkerUrlInfo;
+    return getPublicDocWorkerUrl(this._homeUrl, json);
   }
 
   public async getWorkerAPI(key: string): Promise<DocWorkerAPI> {
@@ -1156,6 +1156,27 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
   }
 }
 
+/**
+ * Represents information to build public doc worker url.
+ *
+ * Structure that may contain either **exclusively**:
+ *  - a selfPrefix when no pool of doc worker exist.
+ *  - a public doc worker url otherwise.
+ */
+export type PublicDocWorkerUrlInfo = {
+  selfPrefix: string;
+  docWorkerUrl: null;
+} | {
+  selfPrefix: null;
+  docWorkerUrl: string;
+}
+
+export function getUrlFromPrefix(homeUrl: string, prefix: string) {
+  const url = new URL(homeUrl);
+  url.pathname = prefix + url.pathname;
+  return url.href;
+}
+
 /**
  * Get a docWorkerUrl from information returned from backend. When the backend
  * is fully configured, and there is a pool of workers, this is straightforward,
@@ -1164,19 +1185,13 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
  * use the homeUrl of the backend, with extra path prefix information
  * given by selfPrefix. At the time of writing, the selfPrefix contains a
  * doc-worker id, and a tag for the codebase (used in consistency checks).
+ *
+ * @param {string} homeUrl
+ * @param {string} docWorkerInfo The information to build the public doc worker url
+ *                               (result of the call to /api/worker/:docId)
  */
-export function getDocWorkerUrl(homeUrl: string, docWorkerInfo: {
-  docWorkerUrl: string|null,
-  selfPrefix?: string,
-}): string {
-  if (!docWorkerInfo.docWorkerUrl) {
-    if (!docWorkerInfo.selfPrefix) {
-      // This should never happen.
-      throw new Error('missing selfPrefix for docWorkerUrl');
-    }
-    const url = new URL(homeUrl);
-    url.pathname = docWorkerInfo.selfPrefix + url.pathname;
-    return url.href;
-  }
-  return docWorkerInfo.docWorkerUrl;
+export function getPublicDocWorkerUrl(homeUrl: string, docWorkerInfo: PublicDocWorkerUrlInfo) {
+  return docWorkerInfo.selfPrefix !== null ?
+    getUrlFromPrefix(homeUrl, docWorkerInfo.selfPrefix) :
+    docWorkerInfo.docWorkerUrl;
 }

app/common/gristUrls.ts

@@ -186,10 +186,23 @@ export interface OrgUrlInfo {
   orgInPath?: string;     // If /o/{orgInPath} should be used to access the requested org.
 }
 
-function isDocInternalUrl(host: string) {
-  if (!process.env.APP_DOC_INTERNAL_URL) { return false; }
-  const internalUrl = new URL('/', process.env.APP_DOC_INTERNAL_URL);
-  return internalUrl.host === host;
+function hostMatchesUrl(host?: string, url?: string) {
+  return host !== undefined && url !== undefined && new URL(url).host === host;
+}
+
+/**
+ * Returns true if:
+ *  - the server is a home worker and the host matches APP_HOME_INTERNAL_URL;
+ *  - or the server is a doc worker and the host matches APP_DOC_INTERNAL_URL;
+ *
+ * @param {string?} host The host to check
+ */
+function isOwnInternalUrlHost(host?: string) {
+  // Note: APP_HOME_INTERNAL_URL may also be defined in doc worker as well as in home worker
+  if (process.env.APP_HOME_INTERNAL_URL && hostMatchesUrl(host, process.env.APP_HOME_INTERNAL_URL)) {
+    return true;
+  }
+  return Boolean(process.env.APP_DOC_INTERNAL_URL) && hostMatchesUrl(host, process.env.APP_DOC_INTERNAL_URL);
 }
 
 /**
@@ -209,7 +222,11 @@ export function getHostType(host: string, options: {
 
   const hostname = host.split(":")[0];
   if (!options.baseDomain) { return 'native'; }
-  if (hostname === 'localhost' || isDocInternalUrl(host) || hostname.endsWith(options.baseDomain)) {
+  if (
+    hostname === 'localhost' ||
+    isOwnInternalUrlHost(host) ||
+    hostname.endsWith(options.baseDomain)
+  ) {
     return 'native';
   }
   return 'custom';

app/gen-server/lib/DocApiForwarder.ts

@@ -104,7 +104,11 @@ export class DocApiForwarder {
     url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
 
     const headers: {[key: string]: string} = {
-      ...getTransitiveHeaders(req),
+      // At this point, we have already checked and trusted the origin of the request.
+      // See FlexServer#addApiMiddleware(). So don't include the "Origin" header.
+      // Including this header also would break features like form submissions,
+      // as the "Host" header is not retrieved when calling getTransitiveHeaders().
+      ...getTransitiveHeaders(req, { includeOrigin: false }),
       'Content-Type': req.get('Content-Type') || 'application/json',
     };
     for (const key of ['X-Sort', 'X-Limit']) {

app/server/lib/AppEndpoint.ts

@@ -9,17 +9,18 @@ import {ApiError} from 'app/common/ApiError';
 import {getSlugIfNeeded, parseUrlId, SHARE_KEY_PREFIX} from 'app/common/gristUrls';
 import {LocalPlugin} from "app/common/plugin";
 import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
-import {Document as APIDocument} from 'app/common/UserAPI';
+import {Document as APIDocument, PublicDocWorkerUrlInfo} from 'app/common/UserAPI';
 import {Document} from "app/gen-server/entity/Document";
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
-import {customizeDocWorkerUrl, getWorker, useWorkerPool} from 'app/server/lib/DocWorkerUtils';
+import {
+  customizeDocWorkerUrl, getDocWorkerInfoOrSelfPrefix, getWorker, useWorkerPool
+} from 'app/server/lib/DocWorkerUtils';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
 import {getCookieDomain} from 'app/server/lib/gristSessions';
-import {getAssignmentId} from 'app/server/lib/idUtils';
 import log from 'app/server/lib/log';
 import {addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions} from 'app/server/lib/sendAppPage';
@@ -48,32 +49,18 @@ export function attachAppEndpoint(options: AttachOptions): void {
   app.get('/apiconsole', expressWrap(async (req, res) =>
     sendAppPage(req, res, {path: 'apiconsole.html', status: 200, config: {}})));
 
-  app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => {
-    if (!useWorkerPool()) {
-      // Let the client know there is not a separate pool of workers,
-      // so they should continue to use the same base URL for accessing
-      // documents. For consistency, return a prefix to add into that
-      // URL, as there would be for a pool of workers. It would be nice
-      // to go ahead and provide the full URL, but that requires making
-      // more assumptions about how Grist is configured.
-      // Alternatives could be: have the client to send their base URL
-      // in the request; or use headers commonly added by reverse proxies.
-      const selfPrefix =  "/dw/self/v/" + gristServer.getTag();
-      res.json({docWorkerUrl: null, selfPrefix});
-      return;
-    }
+  app.get('/api/worker/:docId([^/]+)/?*', expressWrap(async (req, res) => {
     if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); }
     res.header("Access-Control-Allow-Credentials", "true");
 
-    if (!docWorkerMap) {
-      return res.status(500).json({error: 'no worker map'});
-    }
-    const assignmentId = getAssignmentId(docWorkerMap, req.params.assignmentId);
-    const {docStatus} = await getWorker(docWorkerMap, assignmentId, '/status');
-    if (!docStatus) {
-      return res.status(500).json({error: 'no worker'});
-    }
-    res.json({docWorkerUrl: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)});
+    const {selfPrefix, docWorker} = await getDocWorkerInfoOrSelfPrefix(
+      req.params.docId, docWorkerMap, gristServer.getTag()
+    );
+    const info: PublicDocWorkerUrlInfo = selfPrefix ?
+      { docWorkerUrl: null, selfPrefix } :
+      { docWorkerUrl: customizeDocWorkerUrl(docWorker!.publicUrl, req), selfPrefix: null };
+
+    return res.json(info);
   }));
 
   // Handler for serving the document landing pages.  Expects the following parameters:
@@ -160,7 +147,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
       // TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
       const headers = {
         Accept: 'application/json',
-        ...getTransitiveHeaders(req),
+        ...getTransitiveHeaders(req, { includeOrigin: true }),
       };
       const workerInfo = await getWorker(docWorkerMap, docId, `/${docId}/app.html`, {headers});
       docStatus = workerInfo.docStatus;
@@ -206,10 +193,16 @@ export function attachAppEndpoint(options: AttachOptions): void {
       });
     }
 
+    // Without a public URL, we're in single server mode.
+    // Use a null workerPublicURL, to signify that the URL prefix serving the
+    // current endpoint is the only one available.
+    const publicUrl = docStatus?.docWorker?.publicUrl;
+    const workerPublicUrl = publicUrl !== undefined ? customizeDocWorkerUrl(publicUrl, req) : null;
+
     await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
                                  googleTagManager: 'anon', config: {
       assignmentId: docId,
-      getWorker: {[docId]: customizeDocWorkerUrl(docStatus?.docWorker?.publicUrl, req)},
+      getWorker: {[docId]: workerPublicUrl },
       getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)},
       plugins
     }});

app/server/lib/Authorizer.ts

@@ -677,7 +677,10 @@ export function assertAccess(
  * Pull out headers to pass along to a proxied service.  Focused primarily on
  * authentication.
  */
-export function getTransitiveHeaders(req: Request): {[key: string]: string} {
+export function getTransitiveHeaders(
+  req: Request,
+  { includeOrigin }: { includeOrigin: boolean }
+): {[key: string]: string} {
   const Authorization = req.get('Authorization');
   const Cookie = req.get('Cookie');
   const PermitHeader = req.get('Permit');
@@ -685,13 +688,14 @@ export function getTransitiveHeaders(req: Request): {[key: string]: string} {
   const XRequestedWith = req.get('X-Requested-With');
   const Origin = req.get('Origin');  // Pass along the original Origin since it may
                                      // play a role in granular access control.
+
   const result: Record<string, string> = {
     ...(Authorization ? { Authorization } : undefined),
     ...(Cookie ? { Cookie } : undefined),
     ...(Organization ? { Organization } : undefined),
     ...(PermitHeader ? { Permit: PermitHeader } : undefined),
     ...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined),
-    ...(Origin ? { Origin } : undefined),
+    ...((includeOrigin && Origin) ? { Origin } : undefined),
   };
   const extraHeader = process.env.GRIST_FORWARD_AUTH_HEADER;
   const extraHeaderValue = extraHeader && req.get(extraHeader);

app/server/lib/BootProbes.ts

@@ -77,7 +77,7 @@ const _homeUrlReachableProbe: Probe = {
   id: 'reachable',
   name: 'Grist is reachable',
   apply: async (server, req) => {
-    const url = server.getHomeUrl(req);
+    const url = server.getHomeInternalUrl();
     try {
       const resp = await fetch(url);
       if (resp.status !== 200) {
@@ -102,7 +102,7 @@ const _statusCheckProbe: Probe = {
   id: 'health-check',
   name: 'Built-in Health check',
   apply: async (server, req) => {
-    const baseUrl = server.getHomeUrl(req);
+    const baseUrl = server.getHomeInternalUrl();
     const url = new URL(baseUrl);
     url.pathname = removeTrailingSlash(url.pathname) + '/status';
     try {

app/server/lib/DocApi.ts

@@ -1098,10 +1098,11 @@ export class DocWorkerApi {
       if (req.body.sourceDocId) {
         options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId));
         // Make sure that if we wanted to download the full source, we would be allowed.
-        const result = await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/download?dryrun=1`), {
+        const homeUrl = this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/download?dryrun=1`);
+        const result = await fetch(homeUrl, {
           method: 'GET',
           headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
             'Content-Type': 'application/json',
           }
         });
@@ -1111,10 +1112,10 @@ export class DocWorkerApi {
         }
         // We should make sure the source document has flushed recently.
         // It may not be served by the same worker, so work through the api.
-        await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/flush`), {
+        await fetch(this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/flush`), {
           method: 'POST',
           headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
             'Content-Type': 'application/json',
           }
         });
@@ -1170,12 +1171,16 @@ export class DocWorkerApi {
       const showDetails = isAffirmative(req.query.detail);
       const docSession = docSessionFromRequest(req);
       const {states} = await this._getStates(docSession, activeDoc);
-      const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), {
+      const ref = await fetch(this._grist.getHomeInternalUrl(`/api/docs/${req.params.docId2}/states`), {
         headers: {
-          ...getTransitiveHeaders(req),
+          ...getTransitiveHeaders(req, { includeOrigin: false }),
           'Content-Type': 'application/json',
         }
       });
+      if (!ref.ok) {
+        res.status(ref.status).send(await ref.text());
+        return;
+      }
       const states2: DocState[] = (await ref.json()).states;
       const left = states[0];
       const right = states2[0];
@@ -1199,9 +1204,9 @@ export class DocWorkerApi {
 
         // Calculate changes from the (common) parent to the current version of the other document.
         const url = `/api/docs/${req.params.docId2}/compare?left=${parent.h}`;
-        const rightChangesReq = await fetch(this._grist.getHomeUrl(req, url), {
+        const rightChangesReq = await fetch(this._grist.getHomeInternalUrl(url), {
           headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
             'Content-Type': 'application/json',
           }
         });
@@ -1644,7 +1649,7 @@ export class DocWorkerApi {
     let uploadResult;
     try {
       const accessId = makeAccessId(req, getAuthorizedUserId(req));
-      uploadResult = await fetchDoc(this._grist, sourceDocumentId, req, accessId, asTemplate);
+      uploadResult = await fetchDoc(this._grist, this._docWorkerMap, sourceDocumentId, req, accessId, asTemplate);
       globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, `${documentName}.grist`);
     } catch (err) {
       if ((err as ApiError).status === 403) {

app/server/lib/DocWorkerUtils.ts

@@ -1,11 +1,12 @@
 import {ApiError} from 'app/common/ApiError';
 import {parseSubdomainStrictly} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
-import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
+import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import log from 'app/server/lib/log';
 import {adaptServerUrl} from 'app/server/lib/requestUtils';
 import * as express from 'express';
 import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
+import {getAssignmentId} from './idUtils';
 
 /**
  * This method transforms a doc worker's public url as needed based on the request.
@@ -35,16 +36,7 @@ import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
  * TODO: doc worker registration could be redesigned to remove the assumption
  * of a fixed base domain.
  */
-export function customizeDocWorkerUrl(
-  docWorkerUrlSeed: string|undefined,
-  req: express.Request
-): string|null {
-  if (!docWorkerUrlSeed) {
-    // When no doc worker seed, we're in single server mode.
-    // Return null, to signify that the URL prefix serving the
-    // current endpoint is the only one available.
-    return null;
-  }
+export function customizeDocWorkerUrl( docWorkerUrlSeed: string, req: express.Request): string {
   const docWorkerUrl = new URL(docWorkerUrlSeed);
   const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org;
   adaptServerUrl(docWorkerUrl, req);
@@ -152,6 +144,43 @@ export async function getWorker(
   }
 }
 
+export type DocWorkerInfoOrSelfPrefix = {
+  docWorker: DocWorkerInfo,
+  selfPrefix?: never,
+} | {
+  docWorker?: never,
+  selfPrefix: string
+};
+
+export async function getDocWorkerInfoOrSelfPrefix(
+  docId: string,
+  docWorkerMap?: IDocWorkerMap | null,
+  tag?: string
+): Promise<DocWorkerInfoOrSelfPrefix> {
+  if (!useWorkerPool()) {
+    // Let the client know there is not a separate pool of workers,
+    // so they should continue to use the same base URL for accessing
+    // documents. For consistency, return a prefix to add into that
+    // URL, as there would be for a pool of workers. It would be nice
+    // to go ahead and provide the full URL, but that requires making
+    // more assumptions about how Grist is configured.
+    // Alternatives could be: have the client to send their base URL
+    // in the request; or use headers commonly added by reverse proxies.
+    const selfPrefix = "/dw/self/v/" + tag;
+    return { selfPrefix };
+  }
+
+  if (!docWorkerMap) {
+    throw new Error('no worker map');
+  }
+  const assignmentId = getAssignmentId(docWorkerMap, docId);
+  const { docStatus } = await getWorker(docWorkerMap, assignmentId, '/status');
+  if (!docStatus) {
+    throw new Error('no worker');
+  }
+  return { docWorker: docStatus.docWorker };
+}
+
 // Return true if document related endpoints are served by separate workers.
 export function useWorkerPool() {
   return process.env.GRIST_SINGLE_PORT !== 'true';

app/server/lib/FlexServer.ts

@@ -295,6 +295,13 @@ export class FlexServer implements GristServer {
     return homeUrl;
   }
 
+  /**
+   * Same as getDefaultHomeUrl, but for internal use.
+   */
+  public getDefaultHomeInternalUrl(): string {
+    return process.env.APP_HOME_INTERNAL_URL || this.getDefaultHomeUrl();
+  }
+
   /**
    * Get a url for the home server api, adapting it to match the base domain in the
    * requested url.  This adaptation is important for cookie-based authentication.
@@ -309,6 +316,14 @@ export class FlexServer implements GristServer {
     return homeUrl.href;
   }
 
+  /**
+   * Same as getHomeUrl, but for requesting internally.
+   */
+  public getHomeInternalUrl(relPath: string = ''): string {
+    const homeUrl = new URL(relPath, this.getDefaultHomeInternalUrl());
+    return homeUrl.href;
+  }
+
   /**
    * Get a home url that is appropriate for the given document.  For now, this
    * returns a default that works for all documents.  That could change in future,
@@ -316,7 +331,7 @@ export class FlexServer implements GristServer {
    * based on domain).
    */
   public async getHomeUrlByDocId(docId: string, relPath: string = ''): Promise<string> {
-    return new URL(relPath, this.getDefaultHomeUrl()).href;
+    return new URL(relPath, this.getDefaultHomeInternalUrl()).href;
   }
 
   // Get the port number the server listens on.  This may be different from the port
@@ -1429,12 +1444,12 @@ export class FlexServer implements GristServer {
       return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
     }));
 
-    const createDoom = async (req: express.Request) => {
+    const createDoom = async () => {
       const dbManager = this.getHomeDBManager();
       const permitStore = this.getPermitStore();
       const notifier = this.getNotifier();
       const loginSystem = await this.resolveLoginSystem();
-      const homeUrl = this.getHomeUrl(req).replace(/\/$/, '');
+      const homeUrl = this.getHomeInternalUrl().replace(/\/$/, '');
       return new Doom(dbManager, permitStore, notifier, loginSystem, homeUrl);
     };
 
@@ -1458,7 +1473,7 @@ export class FlexServer implements GristServer {
 
         // Reuse Doom cli tool for account deletion. It won't allow to delete account if it has access
         // to other (not public) team sites.
-        const doom = await createDoom(req);
+        const doom = await createDoom();
         await doom.deleteUser(userId);
         this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedAccount');
         return resp.status(200).json(true);
@@ -1491,7 +1506,7 @@ export class FlexServer implements GristServer {
         }
 
         // Reuse Doom cli tool for org deletion. Note, this removes everything as a super user.
-        const doom = await createDoom(req);
+        const doom = await createDoom();
         await doom.deleteOrg(org.id);
 
         this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedSite', {
@@ -1980,7 +1995,7 @@ export class FlexServer implements GristServer {
     // Add the handling for the /upload route. Most uploads are meant for a DocWorker: they are put
     // in temporary files, and the DocWorker needs to be on the same machine to have access to them.
     // This doesn't check for doc access permissions because the request isn't tied to a document.
-    addUploadRoute(this, this.app, this._trustOriginsMiddleware, ...basicMiddleware);
+    addUploadRoute(this, this.app, this._docWorkerMap, this._trustOriginsMiddleware, ...basicMiddleware);
 
     this.app.get('/attachment', ...docAccessMiddleware,
       expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)));
@@ -2418,10 +2433,10 @@ export class FlexServer implements GristServer {
     const workspace = workspaces.find(w => w.name === 'Home');
     if (!workspace) { throw new Error('Home workspace not found'); }
 
-    const copyDocUrl = this.getHomeUrl(req, '/api/docs');
+    const copyDocUrl = this.getHomeInternalUrl('/api/docs');
     const response = await fetch(copyDocUrl, {
       headers: {
-        ...getTransitiveHeaders(req),
+        ...getTransitiveHeaders(req, { includeOrigin: false }),
         'Content-Type': 'application/json',
       },
       method: 'POST',

app/server/lib/GristServer.ts

@@ -35,6 +35,7 @@ export interface GristServer {
   settings?: Readonly<Record<string, unknown>>;
   getHost(): string;
   getHomeUrl(req: express.Request, relPath?: string): string;
+  getHomeInternalUrl(relPath?: string): string;
   getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
   getOwnUrl(): string;
   getOrgUrl(orgKey: string|number): Promise<string>;
@@ -127,6 +128,7 @@ export function createDummyGristServer(): GristServer {
     settings: {},
     getHost() { return 'localhost:4242'; },
     getHomeUrl() { return 'http://localhost:4242'; },
+    getHomeInternalUrl() { return 'http://localhost:4242'; },
     getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
     getMergedOrgUrl() { return 'http://localhost:4242'; },
     getOwnUrl() { return 'http://localhost:4242'; },

app/server/lib/ITestingHooks-ti.ts

@@ -11,7 +11,6 @@ export const ClientJsonMemoryLimits = t.iface([], {
 });
 
 export const ITestingHooks = t.iface([], {
-  "getOwnPort": t.func("number"),
   "getPort": t.func("number"),
   "setLoginSessionProfile": t.func("void", t.param("gristSidCookie", "string"), t.param("profile", t.union("UserProfile", "null")), t.param("org", "string", true)),
   "setServerVersion": t.func("void", t.param("version", t.union("string", "null"))),

app/server/lib/ITestingHooks.ts

@@ -7,7 +7,6 @@ export interface ClientJsonMemoryLimits {
 }
 
 export interface ITestingHooks {
-  getOwnPort(): Promise<number>;
   getPort(): Promise<number>;
   setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise<void>;
   setServerVersion(version: string|null): Promise<void>;

app/server/lib/TestingHooks.ts

@@ -68,11 +68,6 @@ export class TestingHooks implements ITestingHooks {
     private _workerServers: FlexServer[]
   ) {}
 
-  public async getOwnPort(): Promise<number> {
-    log.info("TestingHooks.getOwnPort called");
-    return this._server.getOwnPort();
-  }
-
   public async getPort(): Promise<number> {
     log.info("TestingHooks.getPort called");
     return this._port;

app/server/lib/requestUtils.ts

@@ -1,5 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
-import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
+import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
 import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';

app/server/lib/uploads.ts

@@ -1,7 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
-import {getDocWorkerUrl} from 'app/common/UserAPI';
+import {getUrlFromPrefix} from 'app/common/UserAPI';
 import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -21,6 +21,8 @@ import * as multiparty from 'multiparty';
 import fetch, {Response as FetchResponse} from 'node-fetch';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import {IDocWorkerMap} from './DocWorkerMap';
+import {getDocWorkerInfoOrSelfPrefix} from './DocWorkerUtils';
 
 // After some time of inactivity, clean up the upload. We give an hour, which seems generous,
 // except that if one is toying with import options, and leaves the upload in an open browser idle
@@ -39,7 +41,12 @@ export interface FormResult {
 /**
  * Adds an upload route to the given express app, listening for POST requests at UPLOAD_URL_PATH.
  */
-export function addUploadRoute(server: GristServer, expressApp: Application, ...handlers: RequestHandler[]): void {
+export function addUploadRoute(
+  server: GristServer,
+  expressApp: Application,
+  docWorkerMap: IDocWorkerMap,
+  ...handlers: RequestHandler[]
+): void {
 
   // When doing a cross-origin post, the browser will check for access with options prior to posting.
   // We need to reassure it that the request will be accepted before it will go ahead and post.
@@ -72,7 +79,7 @@ export function addUploadRoute(server: GristServer, expressApp: Application, ...
     if (!docId) { throw new Error('doc must be specified'); }
     const accessId = makeAccessId(req, getAuthorizedUserId(req));
     try {
-      const uploadResult: UploadResult = await fetchDoc(server, docId, req, accessId,
+      const uploadResult: UploadResult = await fetchDoc(server, docWorkerMap, docId, req, accessId,
                                                         req.query.template === '1');
       if (name) {
         globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, name);
@@ -404,24 +411,21 @@ async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlO
  * Fetches a Grist doc potentially managed by a different doc worker.  Passes on credentials
  * supplied in the current request.
  */
-export async function fetchDoc(server: GristServer, docId: string, req: Request, accessId: string|null,
-                        template: boolean): Promise<UploadResult> {
+export async function fetchDoc(
+  server: GristServer,
+  docWorkerMap: IDocWorkerMap,
+  docId: string,
+  req: Request,
+  accessId: string|null,
+  template: boolean
+): Promise<UploadResult> {
   // Prepare headers that preserve credentials of current user.
-  const headers = getTransitiveHeaders(req);
-
-  // Passing the Origin header would serve no purpose here, as we are
-  // constructing an internal request to fetch from our own doc worker
-  // URL. Indeed, it may interfere, as it could incur a CORS check in
-  // `trustOrigin`, which we do not need.
-  delete headers.Origin;
+  const headers = getTransitiveHeaders(req, { includeOrigin: false });
 
   // Find the doc worker responsible for the document we wish to copy.
   // The backend needs to be well configured for this to work.
-  const homeUrl = server.getHomeUrl(req);
-  const fetchUrl = new URL(`/api/worker/${docId}`, homeUrl);
-  const response: FetchResponse = await Deps.fetch(fetchUrl.href, {headers});
-  await _checkForError(response);
-  const docWorkerUrl = getDocWorkerUrl(server.getOwnUrl(), await response.json());
+  const { selfPrefix, docWorker } = await getDocWorkerInfoOrSelfPrefix(docId, docWorkerMap, server.getTag());
+  const docWorkerUrl = docWorker ? docWorker.internalUrl : getUrlFromPrefix(server.getHomeInternalUrl(), selfPrefix);
   // Download the document, in full or as a template.
   const url = new URL(`api/docs/${docId}/download?template=${Number(template)}`,
                       docWorkerUrl.replace(/\/*$/, '/'));