(core) updates from grist-core

2024-10-27 20:44:07 +00:00 · 2024-05-23 13:27:59 -04:00 · 2024-05-23 13:27:59 -04:00 · 76a43129f1
commit 76a43129f1
parent 31722de2eb 53cfc6148d
42 changed files with 2647 additions and 205 deletions
--- a/.github/workflows/self-hosted.yml
+++ b/.github/workflows/self-hosted.yml
@ -0,0 +1,18 @@
+name: Add self-hosting issues to the self-hosting project
+
+on:
+  issues:
+    types:
+      - opened
+      - labeled
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v1.0.1
+        with:
+          project-url: https://github.com/orgs/gristlabs/projects/2
+          github-token: ${{ secrets.SELF_HOSTED_PROJECT }}
+          labeled: self-hosting
--- a/README.md
+++ b/README.md
@ -235,10 +235,11 @@ Grist can be configured in many ways. Here are the main environment variables it

 | Variable                           | Purpose                                                                                                                                                                                                                                                                                                                                                       |
 |------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| ALLOWED_WEBHOOK_DOMAINS            | comma-separated list of permitted domains to use in webhooks (e.g. webhook.site,zapier.com). You can set this to `*` to allow all domains, but if doing so, we recommend using a carefully locked-down proxy (see `GRIST_HTTPS_PROXY`) if you do not entirely trust users. Otherwise services on your internal network may become vulnerable to manipulation. |
+| ALLOWED_WEBHOOK_DOMAINS            | comma-separated list of permitted domains to use in webhooks (e.g. webhook.site,zapier.com). You can set this to `*` to allow all domains, but if doing so, we recommend using a carefully locked-down proxy (see `GRIST_HTTPS_PROXY`) if you do not entirely trust users. Otherwise services on your internal network may become vulnerable to manipulation.   |
 | APP_DOC_URL                        | doc worker url, set when starting an individual doc worker (other servers will find doc worker urls via redis)                                                                                                                                                                                                                                                |
-| APP_DOC_INTERNAL_URL               | like `APP_DOC_URL` but used by the home server to reach the server using an internal domain name resolution (like in a docker environment). Defaults to `APP_DOC_URL`                                                                                                                                                                                         |
+| APP_DOC_INTERNAL_URL               | like `APP_DOC_URL` but used by the home server to reach the server using an internal domain name resolution (like in a docker environment). It only makes sense to define this value in the doc worker. Defaults to `APP_DOC_URL`.                                                                                                                                     |
 | APP_HOME_URL                       | url prefix for home api (home and doc servers need this)                                                                                                                                                                                                                                                                                                      |
+| APP_HOME_INTERNAL_URL              | like `APP_HOME_URL` but used by the home and the doc servers to reach any home workers using an internal domain name resolution (like in a docker environment). Defaults to `APP_HOME_URL`                                                                                                                                                                      |
 | APP_STATIC_URL                     | url prefix for static resources                                                                                                                                                                                                                                                                                                                               |
 | APP_STATIC_INCLUDE_CUSTOM_CSS      | set to "true" to include custom.css (from APP_STATIC_URL) in static pages                                                                                                                                                                                                                                                                                     |
 | APP_UNTRUSTED_URL                  | URL at which to serve/expect plugin content.                                                                                                                                                                                                                                                                                                                  |
@ -280,6 +281,7 @@ Grist can be configured in many ways. Here are the main environment variables it
 | GRIST_SINGLE_ORG                   | set to an org "domain" to pin client to that org                                                                                                                                                                                                                                                                                                              |
 | GRIST_TEMPLATE_ORG                 | set to an org "domain" to show public docs from that org                                                                                                                                                                                                                                                                                                      |
 | GRIST_HELP_CENTER                  | set the help center link ref                                                                                                                                                                                                                                                                                                                                  |
+| GRIST_TERMS_OF_SERVICE_URL         | if set, adds terms of service link                                                                                                                                                                                                                                                                                                                            |
 | FREE_COACHING_CALL_URL             | set the link to the human help (example: email adress or meeting scheduling tool)                                                                                                                                                                                                                                                                             |
 | GRIST_CONTACT_SUPPORT_URL          | set the link to contact support on error pages (example: email adress or online form)                                                                                                                                                                                                                                                                         |
 | GRIST_SUPPORT_ANON                 | if set to 'true', show UI for anonymous access (not shown by default)                                                                                                                                                                                                                                                                                         |
--- a/app/client/models/entities/ViewSectionRec.ts
+++ b/app/client/models/entities/ViewSectionRec.ts
@ -460,7 +460,9 @@ export function createViewSectionRec(this: ViewSectionRec, docModel: DocModel):
  // - Widget type description (if not grid)
  // All concatenated separated by space.
  this.defaultWidgetTitle = this.autoDispose(ko.pureComputed(() => {
-    const widgetTypeDesc = this.parentKey() !== 'record' ? `${getWidgetTypes(this.parentKey.peek() as any).label}` : '';
+    const widgetTypeDesc = this.parentKey() !== 'record'
+      ? `${getWidgetTypes(this.parentKey.peek() as any).getLabel()}`
+      : '';
    const table = this.table();
    return [
      table.tableNameDef()?.toUpperCase(), // Due to ACL this can be null.
--- a/app/client/ui/AdminPanel.ts
+++ b/app/client/ui/AdminPanel.ts
@ -105,6 +105,13 @@ export class AdminPanel extends Disposable {
          value: this._buildSandboxingDisplay(owner),
          expandedContent: this._buildSandboxingNotice(),
        }),
+        dom.create(AdminSectionItem, {
+          id: 'authentication',
+          name: t('Authentication'),
+          description: t('Current authentication method'),
+          value: this._buildAuthenticationDisplay(owner),
+          expandedContent: this._buildAuthenticationNotice(owner),
+        })
      ]),

      dom.create(AdminSection, t('Version'), [
@ -156,6 +163,37 @@ isolated from other documents and isolated from the network.'),
    ];
  }

+  private _buildAuthenticationDisplay(owner: IDisposableOwner) {
+    return dom.domComputed(
+      use => {
+        const req = this._checks.requestCheckById(use, 'authentication');
+        const result = req ? use(req.result) : undefined;
+        if (!result) {
+          return cssValueLabel(cssErrorText('unavailable'));
+        }
+
+        const { success, details } = result;
+        const loginSystemId = details?.loginSystemId;
+
+        if (!success || !loginSystemId) {
+          return cssValueLabel(cssErrorText('auth error'));
+        }
+
+        if (loginSystemId === 'no-logins') {
+          return cssValueLabel(cssDangerText('no authentication'));
+        }
+
+        return cssValueLabel(cssHappyText(loginSystemId));
+      }
+    );
+  }
+
+  private _buildAuthenticationNotice(owner: IDisposableOwner) {
+    return t('Grist allows different types of authentication to be configured, including SAML and OIDC. \
+    We recommend enabling one of these if Grist is accessible over the network or being made available \
+    to multiple people.');
+  }
+
  private _buildUpdates(owner: MultiHolder) {
    // We can be in those states:
    enum State {
@ -446,6 +484,10 @@ const cssErrorText = styled('span', `
  color: ${theme.errorText};
 `);

+export const cssDangerText = styled('div', `
+  color: ${theme.dangerText};
+`);
+
 const cssHappyText = styled('span', `
  color: ${theme.controlFg};
 `);
--- a/app/client/ui/AdminPanelCss.ts
+++ b/app/client/ui/AdminPanelCss.ts
@ -124,6 +124,7 @@ const cssItemName = styled('div', `
  font-weight: bold;
  display: flex;
  align-items: center;
+  margin-right: 14px;
  font-size: ${vars.largeFontSize};
  padding-left: 24px;
  &-prefixed {
--- a/app/client/ui/ColumnFilterMenu.ts
+++ b/app/client/ui/ColumnFilterMenu.ts
@ -130,7 +130,10 @@ export function columnFilterMenu(owner: IDisposableOwner, opts: IFilterMenuOptio
    dom.onDispose(() => cancel ? doCancel() : doSave()),
    dom.onKeyDown({
      Enter: () => onClose(),
-      Escape: () => onClose(),
+      Escape: () => {
+        cancel = true;
+        onClose();
+      },
    }),

    // Filter by range
--- a/app/client/ui/HomeLeftPane.ts
+++ b/app/client/ui/HomeLeftPane.ts
@ -146,6 +146,14 @@ export function createHomeLeftPane(leftPanelOpen: Observable<boolean>, home: Hom
          ) : null
        ),
        createHelpTools(home.app),
+        (commonUrls.termsOfService ?
+          cssPageEntry(
+            cssPageLink(cssPageIcon('Memo'), cssLinkText(t("Terms of service")),
+              { href: commonUrls.termsOfService, target: '_blank' },
+              testId('dm-tos'),
+            ),
+          ) : null
+        ),
      )
    )
  );
--- a/app/client/ui/PageWidgetPicker.ts
+++ b/app/client/ui/PageWidgetPicker.ts
@ -205,7 +205,7 @@ export function buildPageWidgetPicker(
      // If savePromise throws an error, before or after timeout, we let the error propagate as it
      // should be handle by the caller.
      if (await isLongerThan(savePromise, DELAY_BEFORE_SPINNER_MS)) {
-        const label = getWidgetTypes(type).label;
+        const label = getWidgetTypes(type).getLabel();
        await spinnerModal(t("Building {{- label}} widget", { label }), savePromise);
      }
    }
@ -317,12 +317,12 @@ export class PageWidgetSelect extends Disposable {
        cssPanel(
          header(t("Select Widget")),
          sectionTypes.map((value) => {
-            const {label, icon: iconName} = getWidgetTypes(value);
+            const widgetInfo = getWidgetTypes(value);
            const disabled = computed(this._value.table, (use, tid) => this._isTypeDisabled(value, tid));
            return cssEntry(
              dom.autoDispose(disabled),
-              cssTypeIcon(iconName),
-              label,
+              cssTypeIcon(widgetInfo.icon),
+              widgetInfo.getLabel(),
              dom.on('click', () => !disabled.get() && this._selectType(value)),
              cssEntry.cls('-selected', (use) => use(this._value.type) === value),
              cssEntry.cls('-disabled', disabled),
--- a/app/client/ui/RightPanel.ts
+++ b/app/client/ui/RightPanel.ts
@ -38,7 +38,7 @@ import {PredefinedCustomSectionConfig} from "app/client/ui/PredefinedCustomSecti
 import {cssLabel} from 'app/client/ui/RightPanelStyles';
 import {linkId, NoLink, selectBy} from 'app/client/ui/selectBy';
 import {VisibleFieldsConfig} from 'app/client/ui/VisibleFieldsConfig';
-import {getTelemetryWidgetTypeFromVS, widgetTypesMap} from "app/client/ui/widgetTypesMap";
+import {getTelemetryWidgetTypeFromVS, getWidgetTypes} from "app/client/ui/widgetTypesMap";
 import {basicButton, primaryButton} from 'app/client/ui2018/buttons';
 import {buttonSelect} from 'app/client/ui2018/buttonSelect';
 import {labeledSquareCheckbox} from 'app/client/ui2018/checkbox';
@ -220,10 +220,10 @@ export class RightPanel extends Disposable {

  private _buildStandardHeader() {
    return dom.maybe(this._pageWidgetType, (type) => {
-      const widgetInfo = widgetTypesMap.get(type) || {label: 'Table', icon: 'TypeTable'};
+      const widgetInfo = getWidgetTypes(type);
      const fieldInfo = getFieldType(type);
      return [
-        cssTopBarItem(cssTopBarIcon(widgetInfo.icon), widgetInfo.label,
+        cssTopBarItem(cssTopBarIcon(widgetInfo.icon), widgetInfo.getLabel(),
          cssTopBarItem.cls('-selected', (use) => use(this._topTab) === 'pageWidget'),
          dom.on('click', () => this._topTab.set("pageWidget")),
          testId('right-tab-pagewidget')),
--- a/app/client/ui/widgetTypesMap.ts
+++ b/app/client/ui/widgetTypesMap.ts
@ -3,21 +3,25 @@ import {ViewSectionRec} from "app/client/models/entities/ViewSectionRec";
 import {IPageWidget} from "app/client/ui/PageWidgetPicker";
 import {IconName} from "app/client/ui2018/IconList";
 import {IWidgetType} from "app/common/widgetTypes";
+import {makeT} from 'app/client/lib/localization';
+
+const t = makeT('widgetTypesMap');

 export const widgetTypesMap = new Map<IWidgetType, IWidgetTypeInfo>([
-  ['record', {label: 'Table', icon: 'TypeTable'}],
-  ['single', {label: 'Card', icon: 'TypeCard'}],
-  ['detail', {label: 'Card List', icon: 'TypeCardList'}],
-  ['chart', {label: 'Chart', icon: 'TypeChart'}],
-  ['form', {label: 'Form', icon: 'Board'}],
-  ['custom', {label: 'Custom', icon: 'TypeCustom'}],
-  ['custom.calendar', {label: 'Calendar', icon: 'TypeCalendar'}],
+  ['record', {name: 'Table', icon: 'TypeTable', getLabel: () => t('Table')}],
+  ['single', {name: 'Card', icon: 'TypeCard', getLabel: () => t('Card')}],
+  ['detail', {name: 'Card List', icon: 'TypeCardList', getLabel: () => t('Card List')}],
+  ['chart', {name: 'Chart', icon: 'TypeChart', getLabel: () => t('Chart')}],
+  ['form', {name: 'Form', icon: 'Board', getLabel: () => t('Form')}],
+  ['custom', {name: 'Custom', icon: 'TypeCustom', getLabel: () => t('Custom')}],
+  ['custom.calendar', {name: 'Calendar', icon: 'TypeCalendar', getLabel: () => t('Calendar')}],
 ]);

 // Widget type info.
 export interface IWidgetTypeInfo {
-  label: string;
+  name: string;
  icon: IconName;
+  getLabel: () => string;
 }

 // Returns the widget type info for sectionType, or the one for 'record' if sectionType is null.
@ -46,7 +50,7 @@ export function getTelemetryWidgetTypeFromPageWidget(widget: IPageWidget) {
 }

 function getTelemetryWidgetType(type: IWidgetType, options: GetTelemetryWidgetTypeOptions = {}) {
-  let telemetryWidgetType: string | undefined = widgetTypesMap.get(type)?.label;
+  let telemetryWidgetType: string | undefined = widgetTypesMap.get(type)?.name;
  if (!telemetryWidgetType) { return undefined; }

  if (options.isNewTable) {
--- a/app/common/BootProbe.ts
+++ b/app/common/BootProbe.ts
@ -6,7 +6,8 @@ export type BootProbeIds =
    'reachable' |
    'host-header' |
    'sandboxing' |
-    'system-user'
+    'system-user' |
+    'authentication'
 ;

 export interface BootProbeResult {
--- a/app/common/UserAPI.ts
+++ b/app/common/UserAPI.ts
@ -780,11 +780,11 @@ export class UserAPIImpl extends BaseAPI implements UserAPI {
  }

  public async getWorker(key: string): Promise<string> {
-    const json = await this.requestJson(`${this._url}/api/worker/${key}`, {
+    const json = (await this.requestJson(`${this._url}/api/worker/${key}`, {
      method: 'GET',
      credentials: 'include'
-    });
-    return getDocWorkerUrl(this._homeUrl, json);
+    })) as PublicDocWorkerUrlInfo;
+    return getPublicDocWorkerUrl(this._homeUrl, json);
  }

  public async getWorkerAPI(key: string): Promise<DocWorkerAPI> {
@ -1163,6 +1163,27 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
  }
 }

+/**
+ * Represents information to build public doc worker url.
+ *
+ * Structure that may contain either **exclusively**:
+ *  - a selfPrefix when no pool of doc worker exist.
+ *  - a public doc worker url otherwise.
+ */
+export type PublicDocWorkerUrlInfo = {
+  selfPrefix: string;
+  docWorkerUrl: null;
+} | {
+  selfPrefix: null;
+  docWorkerUrl: string;
+}
+
+export function getUrlFromPrefix(homeUrl: string, prefix: string) {
+  const url = new URL(homeUrl);
+  url.pathname = prefix + url.pathname;
+  return url.href;
+}
+
 /**
 * Get a docWorkerUrl from information returned from backend. When the backend
 * is fully configured, and there is a pool of workers, this is straightforward,
@ -1171,19 +1192,13 @@ export class DocAPIImpl extends BaseAPI implements DocAPI {
 * use the homeUrl of the backend, with extra path prefix information
 * given by selfPrefix. At the time of writing, the selfPrefix contains a
 * doc-worker id, and a tag for the codebase (used in consistency checks).
+ *
+ * @param {string} homeUrl
+ * @param {string} docWorkerInfo The information to build the public doc worker url
+ *                               (result of the call to /api/worker/:docId)
 */
-export function getDocWorkerUrl(homeUrl: string, docWorkerInfo: {
-  docWorkerUrl: string|null,
-  selfPrefix?: string,
-}): string {
-  if (!docWorkerInfo.docWorkerUrl) {
-    if (!docWorkerInfo.selfPrefix) {
-      // This should never happen.
-      throw new Error('missing selfPrefix for docWorkerUrl');
-    }
-    const url = new URL(homeUrl);
-    url.pathname = docWorkerInfo.selfPrefix + url.pathname;
-    return url.href;
-  }
-  return docWorkerInfo.docWorkerUrl;
+export function getPublicDocWorkerUrl(homeUrl: string, docWorkerInfo: PublicDocWorkerUrlInfo) {
+  return docWorkerInfo.selfPrefix !== null ?
+    getUrlFromPrefix(homeUrl, docWorkerInfo.selfPrefix) :
+    docWorkerInfo.docWorkerUrl;
 }
--- a/app/common/gristUrls.ts
+++ b/app/common/gristUrls.ts
@ -91,6 +91,7 @@ export const commonUrls = {
  helpAPI: 'https://support.getgrist.com/api',
  freeCoachingCall: getFreeCoachingCallUrl(),
  contactSupport: getContactSupportUrl(),
+  termsOfService: getTermsOfServiceUrl(),
  plans: "https://www.getgrist.com/pricing",
  sproutsProgram: "https://www.getgrist.com/sprouts-program",
  contact: "https://www.getgrist.com/contact",
@ -187,10 +188,23 @@ export interface OrgUrlInfo {
  orgInPath?: string;     // If /o/{orgInPath} should be used to access the requested org.
 }

-function isDocInternalUrl(host: string) {
-  if (!process.env.APP_DOC_INTERNAL_URL) { return false; }
-  const internalUrl = new URL('/', process.env.APP_DOC_INTERNAL_URL);
-  return internalUrl.host === host;
+function hostMatchesUrl(host?: string, url?: string) {
+  return host !== undefined && url !== undefined && new URL(url).host === host;
+}
+
+/**
+ * Returns true if:
+ *  - the server is a home worker and the host matches APP_HOME_INTERNAL_URL;
+ *  - or the server is a doc worker and the host matches APP_DOC_INTERNAL_URL;
+ *
+ * @param {string?} host The host to check
+ */
+function isOwnInternalUrlHost(host?: string) {
+  // Note: APP_HOME_INTERNAL_URL may also be defined in doc worker as well as in home worker
+  if (process.env.APP_HOME_INTERNAL_URL && hostMatchesUrl(host, process.env.APP_HOME_INTERNAL_URL)) {
+    return true;
+  }
+  return Boolean(process.env.APP_DOC_INTERNAL_URL) && hostMatchesUrl(host, process.env.APP_DOC_INTERNAL_URL);
 }

 /**
@ -210,7 +224,11 @@ export function getHostType(host: string, options: {

  const hostname = host.split(":")[0];
  if (!options.baseDomain) { return 'native'; }
-  if (hostname === 'localhost' || isDocInternalUrl(host) || hostname.endsWith(options.baseDomain)) {
+  if (
+    hostname === 'localhost' ||
+    isOwnInternalUrlHost(host) ||
+    hostname.endsWith(options.baseDomain)
+  ) {
    return 'native';
  }
  return 'custom';
@ -676,6 +694,9 @@ export interface GristLoadConfig {
  // Url for support for the browser client to use.
  helpCenterUrl?: string;

+  // Url for terms of service for the browser client to use
+  termsOfServiceUrl?: string;
+
  // Url for free coaching call scheduling for the browser client to use.
  freeCoachingCallUrl?: string;

@ -887,6 +908,15 @@ export function getHelpCenterUrl(): string {
  }
 }

+export function getTermsOfServiceUrl(): string|undefined {
+  if(isClient()) {
+    const gristConfig: GristLoadConfig = (window as any).gristConfig;
+    return gristConfig && gristConfig.termsOfServiceUrl || undefined;
+  } else {
+    return process.env.GRIST_TERMS_OF_SERVICE_URL || undefined;
+  }
+}
+
 export function getFreeCoachingCallUrl(): string {
  const defaultUrl = "https://calendly.com/grist-team/grist-free-coaching-call";
  if(isClient()) {
--- a/app/gen-server/lib/DocApiForwarder.ts
+++ b/app/gen-server/lib/DocApiForwarder.ts
@ -104,7 +104,11 @@ export class DocApiForwarder {
    url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;

    const headers: {[key: string]: string} = {
-      ...getTransitiveHeaders(req),
+      // At this point, we have already checked and trusted the origin of the request.
+      // See FlexServer#addApiMiddleware(). So don't include the "Origin" header.
+      // Including this header also would break features like form submissions,
+      // as the "Host" header is not retrieved when calling getTransitiveHeaders().
+      ...getTransitiveHeaders(req, { includeOrigin: false }),
      'Content-Type': req.get('Content-Type') || 'application/json',
    };
    for (const key of ['X-Sort', 'X-Limit']) {
--- a/app/server/lib/AppEndpoint.ts
+++ b/app/server/lib/AppEndpoint.ts
@ -9,17 +9,18 @@ import {ApiError} from 'app/common/ApiError';
 import {getSlugIfNeeded, parseUrlId, SHARE_KEY_PREFIX} from 'app/common/gristUrls';
 import {LocalPlugin} from "app/common/plugin";
 import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
-import {Document as APIDocument} from 'app/common/UserAPI';
+import {Document as APIDocument, PublicDocWorkerUrlInfo} from 'app/common/UserAPI';
 import {Document} from "app/gen-server/entity/Document";
 import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
 import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
        RequestWithLogin} from 'app/server/lib/Authorizer';
 import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
-import {customizeDocWorkerUrl, getWorker, useWorkerPool} from 'app/server/lib/DocWorkerUtils';
+import {
+  customizeDocWorkerUrl, getDocWorkerInfoOrSelfPrefix, getWorker, useWorkerPool
+} from 'app/server/lib/DocWorkerUtils';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
 import {getCookieDomain} from 'app/server/lib/gristSessions';
-import {getAssignmentId} from 'app/server/lib/idUtils';
 import log from 'app/server/lib/log';
 import {addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
 import {ISendAppPageOptions} from 'app/server/lib/sendAppPage';
@ -48,32 +49,18 @@ export function attachAppEndpoint(options: AttachOptions): void {
  app.get('/apiconsole', expressWrap(async (req, res) =>
    sendAppPage(req, res, {path: 'apiconsole.html', status: 200, config: {}})));

-  app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => {
-    if (!useWorkerPool()) {
-      // Let the client know there is not a separate pool of workers,
-      // so they should continue to use the same base URL for accessing
-      // documents. For consistency, return a prefix to add into that
-      // URL, as there would be for a pool of workers. It would be nice
-      // to go ahead and provide the full URL, but that requires making
-      // more assumptions about how Grist is configured.
-      // Alternatives could be: have the client to send their base URL
-      // in the request; or use headers commonly added by reverse proxies.
-      const selfPrefix =  "/dw/self/v/" + gristServer.getTag();
-      res.json({docWorkerUrl: null, selfPrefix});
-      return;
-    }
+  app.get('/api/worker/:docId([^/]+)/?*', expressWrap(async (req, res) => {
    if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); }
    res.header("Access-Control-Allow-Credentials", "true");

-    if (!docWorkerMap) {
-      return res.status(500).json({error: 'no worker map'});
-    }
-    const assignmentId = getAssignmentId(docWorkerMap, req.params.assignmentId);
-    const {docStatus} = await getWorker(docWorkerMap, assignmentId, '/status');
-    if (!docStatus) {
-      return res.status(500).json({error: 'no worker'});
-    }
-    res.json({docWorkerUrl: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)});
+    const {selfPrefix, docWorker} = await getDocWorkerInfoOrSelfPrefix(
+      req.params.docId, docWorkerMap, gristServer.getTag()
+    );
+    const info: PublicDocWorkerUrlInfo = selfPrefix ?
+      { docWorkerUrl: null, selfPrefix } :
+      { docWorkerUrl: customizeDocWorkerUrl(docWorker!.publicUrl, req), selfPrefix: null };
+
+    return res.json(info);
  }));

  // Handler for serving the document landing pages.  Expects the following parameters:
@ -160,7 +147,7 @@ export function attachAppEndpoint(options: AttachOptions): void {
      // TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
      const headers = {
        Accept: 'application/json',
-        ...getTransitiveHeaders(req),
+        ...getTransitiveHeaders(req, { includeOrigin: true }),
      };
      const workerInfo = await getWorker(docWorkerMap, docId, `/${docId}/app.html`, {headers});
      docStatus = workerInfo.docStatus;
@ -206,10 +193,16 @@ export function attachAppEndpoint(options: AttachOptions): void {
      });
    }

+    // Without a public URL, we're in single server mode.
+    // Use a null workerPublicURL, to signify that the URL prefix serving the
+    // current endpoint is the only one available.
+    const publicUrl = docStatus?.docWorker?.publicUrl;
+    const workerPublicUrl = publicUrl !== undefined ? customizeDocWorkerUrl(publicUrl, req) : null;
+
    await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
                                 googleTagManager: 'anon', config: {
      assignmentId: docId,
-      getWorker: {[docId]: customizeDocWorkerUrl(docStatus?.docWorker?.publicUrl, req)},
+      getWorker: {[docId]: workerPublicUrl },
      getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)},
      plugins
    }});
--- a/app/server/lib/Authorizer.ts
+++ b/app/server/lib/Authorizer.ts
@ -677,7 +677,10 @@ export function assertAccess(
 * Pull out headers to pass along to a proxied service.  Focused primarily on
 * authentication.
 */
-export function getTransitiveHeaders(req: Request): {[key: string]: string} {
+export function getTransitiveHeaders(
+  req: Request,
+  { includeOrigin }: { includeOrigin: boolean }
+): {[key: string]: string} {
  const Authorization = req.get('Authorization');
  const Cookie = req.get('Cookie');
  const PermitHeader = req.get('Permit');
@ -685,13 +688,14 @@ export function getTransitiveHeaders(req: Request): {[key: string]: string} {
  const XRequestedWith = req.get('X-Requested-With');
  const Origin = req.get('Origin');  // Pass along the original Origin since it may
                                     // play a role in granular access control.
+
  const result: Record<string, string> = {
    ...(Authorization ? { Authorization } : undefined),
    ...(Cookie ? { Cookie } : undefined),
    ...(Organization ? { Organization } : undefined),
    ...(PermitHeader ? { Permit: PermitHeader } : undefined),
    ...(XRequestedWith ? { 'X-Requested-With': XRequestedWith } : undefined),
-    ...(Origin ? { Origin } : undefined),
+    ...((includeOrigin && Origin) ? { Origin } : undefined),
  };
  const extraHeader = process.env.GRIST_FORWARD_AUTH_HEADER;
  const extraHeaderValue = extraHeader && req.get(extraHeader);
--- a/app/server/lib/BootProbes.ts
+++ b/app/server/lib/BootProbes.ts
@ -58,6 +58,7 @@ export class BootProbes {
    this._probes.push(_bootProbe);
    this._probes.push(_hostHeaderProbe);
    this._probes.push(_sandboxingProbe);
+    this._probes.push(_authenticationProbe);
    this._probeById = new Map(this._probes.map(p => [p.id, p]));
  }
 }
@ -77,7 +78,7 @@ const _homeUrlReachableProbe: Probe = {
  id: 'reachable',
  name: 'Grist is reachable',
  apply: async (server, req) => {
-    const url = server.getHomeUrl(req);
+    const url = server.getHomeInternalUrl();
    try {
      const resp = await fetch(url);
      if (resp.status !== 200) {
@ -102,7 +103,7 @@ const _statusCheckProbe: Probe = {
  id: 'health-check',
  name: 'Built-in Health check',
  apply: async (server, req) => {
-    const baseUrl = server.getHomeUrl(req);
+    const baseUrl = server.getHomeInternalUrl();
    const url = new URL(baseUrl);
    url.pathname = removeTrailingSlash(url.pathname) + '/status';
    try {
@ -202,3 +203,17 @@ const _sandboxingProbe: Probe = {
    };
  },
 };
+
+const _authenticationProbe: Probe = {
+  id: 'authentication',
+  name: 'Authentication system',
+  apply: async(server, req) => {
+    const loginSystemId = server.getInfo('loginMiddlewareComment');
+    return {
+      success: loginSystemId != undefined,
+      details: {
+        loginSystemId,
+      }
+    };
+  },
+};
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@ -1098,10 +1098,11 @@ export class DocWorkerApi {
      if (req.body.sourceDocId) {
        options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId));
        // Make sure that if we wanted to download the full source, we would be allowed.
-        const result = await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/download?dryrun=1`), {
+        const homeUrl = this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/download?dryrun=1`);
+        const result = await fetch(homeUrl, {
          method: 'GET',
          headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
            'Content-Type': 'application/json',
          }
        });
@ -1111,10 +1112,10 @@ export class DocWorkerApi {
        }
        // We should make sure the source document has flushed recently.
        // It may not be served by the same worker, so work through the api.
-        await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/flush`), {
+        await fetch(this._grist.getHomeInternalUrl(`/api/docs/${options.sourceDocId}/flush`), {
          method: 'POST',
          headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
            'Content-Type': 'application/json',
          }
        });
@ -1170,12 +1171,16 @@ export class DocWorkerApi {
      const showDetails = isAffirmative(req.query.detail);
      const docSession = docSessionFromRequest(req);
      const {states} = await this._getStates(docSession, activeDoc);
-      const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), {
+      const ref = await fetch(this._grist.getHomeInternalUrl(`/api/docs/${req.params.docId2}/states`), {
        headers: {
-          ...getTransitiveHeaders(req),
+          ...getTransitiveHeaders(req, { includeOrigin: false }),
          'Content-Type': 'application/json',
        }
      });
+      if (!ref.ok) {
+        res.status(ref.status).send(await ref.text());
+        return;
+      }
      const states2: DocState[] = (await ref.json()).states;
      const left = states[0];
      const right = states2[0];
@ -1199,9 +1204,9 @@ export class DocWorkerApi {

        // Calculate changes from the (common) parent to the current version of the other document.
        const url = `/api/docs/${req.params.docId2}/compare?left=${parent.h}`;
-        const rightChangesReq = await fetch(this._grist.getHomeUrl(req, url), {
+        const rightChangesReq = await fetch(this._grist.getHomeInternalUrl(url), {
          headers: {
-            ...getTransitiveHeaders(req),
+            ...getTransitiveHeaders(req, { includeOrigin: false }),
            'Content-Type': 'application/json',
          }
        });
@ -1644,7 +1649,7 @@ export class DocWorkerApi {
    let uploadResult;
    try {
      const accessId = makeAccessId(req, getAuthorizedUserId(req));
-      uploadResult = await fetchDoc(this._grist, sourceDocumentId, req, accessId, asTemplate);
+      uploadResult = await fetchDoc(this._grist, this._docWorkerMap, sourceDocumentId, req, accessId, asTemplate);
      globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, `${documentName}.grist`);
    } catch (err) {
      if ((err as ApiError).status === 403) {
--- a/app/server/lib/DocWorkerUtils.ts
+++ b/app/server/lib/DocWorkerUtils.ts
@ -1,11 +1,12 @@
 import {ApiError} from 'app/common/ApiError';
 import {parseSubdomainStrictly} from 'app/common/gristUrls';
 import {removeTrailingSlash} from 'app/common/gutil';
-import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
+import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import log from 'app/server/lib/log';
 import {adaptServerUrl} from 'app/server/lib/requestUtils';
 import * as express from 'express';
 import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
+import {getAssignmentId} from './idUtils';

 /**
 * This method transforms a doc worker's public url as needed based on the request.
@ -35,16 +36,7 @@ import fetch, {Response as FetchResponse, RequestInit} from 'node-fetch';
 * TODO: doc worker registration could be redesigned to remove the assumption
 * of a fixed base domain.
 */
-export function customizeDocWorkerUrl(
-  docWorkerUrlSeed: string|undefined,
-  req: express.Request
-): string|null {
-  if (!docWorkerUrlSeed) {
-    // When no doc worker seed, we're in single server mode.
-    // Return null, to signify that the URL prefix serving the
-    // current endpoint is the only one available.
-    return null;
-  }
+export function customizeDocWorkerUrl( docWorkerUrlSeed: string, req: express.Request): string {
  const docWorkerUrl = new URL(docWorkerUrlSeed);
  const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org;
  adaptServerUrl(docWorkerUrl, req);
@ -152,6 +144,43 @@ export async function getWorker(
  }
 }

+export type DocWorkerInfoOrSelfPrefix = {
+  docWorker: DocWorkerInfo,
+  selfPrefix?: never,
+} | {
+  docWorker?: never,
+  selfPrefix: string
+};
+
+export async function getDocWorkerInfoOrSelfPrefix(
+  docId: string,
+  docWorkerMap?: IDocWorkerMap | null,
+  tag?: string
+): Promise<DocWorkerInfoOrSelfPrefix> {
+  if (!useWorkerPool()) {
+    // Let the client know there is not a separate pool of workers,
+    // so they should continue to use the same base URL for accessing
+    // documents. For consistency, return a prefix to add into that
+    // URL, as there would be for a pool of workers. It would be nice
+    // to go ahead and provide the full URL, but that requires making
+    // more assumptions about how Grist is configured.
+    // Alternatives could be: have the client to send their base URL
+    // in the request; or use headers commonly added by reverse proxies.
+    const selfPrefix = "/dw/self/v/" + tag;
+    return { selfPrefix };
+  }
+
+  if (!docWorkerMap) {
+    throw new Error('no worker map');
+  }
+  const assignmentId = getAssignmentId(docWorkerMap, docId);
+  const { docStatus } = await getWorker(docWorkerMap, assignmentId, '/status');
+  if (!docStatus) {
+    throw new Error('no worker');
+  }
+  return { docWorker: docStatus.docWorker };
+}
+
 // Return true if document related endpoints are served by separate workers.
 export function useWorkerPool() {
  return process.env.GRIST_SINGLE_PORT !== 'true';
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@ -295,6 +295,13 @@ export class FlexServer implements GristServer {
    return homeUrl;
  }

+  /**
+   * Same as getDefaultHomeUrl, but for internal use.
+   */
+  public getDefaultHomeInternalUrl(): string {
+    return process.env.APP_HOME_INTERNAL_URL || this.getDefaultHomeUrl();
+  }
+
  /**
   * Get a url for the home server api, adapting it to match the base domain in the
   * requested url.  This adaptation is important for cookie-based authentication.
@ -309,6 +316,14 @@ export class FlexServer implements GristServer {
    return homeUrl.href;
  }

+  /**
+   * Same as getHomeUrl, but for requesting internally.
+   */
+  public getHomeInternalUrl(relPath: string = ''): string {
+    const homeUrl = new URL(relPath, this.getDefaultHomeInternalUrl());
+    return homeUrl.href;
+  }
+
  /**
   * Get a home url that is appropriate for the given document.  For now, this
   * returns a default that works for all documents.  That could change in future,
@ -316,7 +331,7 @@ export class FlexServer implements GristServer {
   * based on domain).
   */
  public async getHomeUrlByDocId(docId: string, relPath: string = ''): Promise<string> {
-    return new URL(relPath, this.getDefaultHomeUrl()).href;
+    return new URL(relPath, this.getDefaultHomeInternalUrl()).href;
  }

  // Get the port number the server listens on.  This may be different from the port
@ -1411,6 +1426,11 @@ export class FlexServer implements GristServer {
    return this._sandboxInfo;
  }

+  public getInfo(key: string): any {
+    const infoPair = this.info.find(([keyToCheck]) => key === keyToCheck);
+    return infoPair?.[1];
+  }
+
  public disableExternalStorage() {
    if (this.deps.has('doc')) {
      throw new Error('disableExternalStorage called too late');
@ -1429,12 +1449,12 @@ export class FlexServer implements GristServer {
      return this._sendAppPage(req, resp, {path: 'app.html', status: 200, config: {}});
    }));

-    const createDoom = async (req: express.Request) => {
+    const createDoom = async () => {
      const dbManager = this.getHomeDBManager();
      const permitStore = this.getPermitStore();
      const notifier = this.getNotifier();
      const loginSystem = await this.resolveLoginSystem();
-      const homeUrl = this.getHomeUrl(req).replace(/\/$/, '');
+      const homeUrl = this.getHomeInternalUrl().replace(/\/$/, '');
      return new Doom(dbManager, permitStore, notifier, loginSystem, homeUrl);
    };

@ -1458,7 +1478,7 @@ export class FlexServer implements GristServer {

        // Reuse Doom cli tool for account deletion. It won't allow to delete account if it has access
        // to other (not public) team sites.
-        const doom = await createDoom(req);
+        const doom = await createDoom();
        await doom.deleteUser(userId);
        this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedAccount');
        return resp.status(200).json(true);
@ -1491,7 +1511,7 @@ export class FlexServer implements GristServer {
        }

        // Reuse Doom cli tool for org deletion. Note, this removes everything as a super user.
-        const doom = await createDoom(req);
+        const doom = await createDoom();
        await doom.deleteOrg(org.id);

        this.getTelemetry().logEvent(req as RequestWithLogin, 'deletedSite', {
@ -1980,7 +2000,7 @@ export class FlexServer implements GristServer {
    // Add the handling for the /upload route. Most uploads are meant for a DocWorker: they are put
    // in temporary files, and the DocWorker needs to be on the same machine to have access to them.
    // This doesn't check for doc access permissions because the request isn't tied to a document.
-    addUploadRoute(this, this.app, this._trustOriginsMiddleware, ...basicMiddleware);
+    addUploadRoute(this, this.app, this._docWorkerMap, this._trustOriginsMiddleware, ...basicMiddleware);

    this.app.get('/attachment', ...docAccessMiddleware,
      expressWrap(async (req, res) => this._docWorker.getAttachment(req, res)));
@ -2418,10 +2438,10 @@ export class FlexServer implements GristServer {
    const workspace = workspaces.find(w => w.name === 'Home');
    if (!workspace) { throw new Error('Home workspace not found'); }

-    const copyDocUrl = this.getHomeUrl(req, '/api/docs');
+    const copyDocUrl = this.getHomeInternalUrl('/api/docs');
    const response = await fetch(copyDocUrl, {
      headers: {
-        ...getTransitiveHeaders(req),
+        ...getTransitiveHeaders(req, { includeOrigin: false }),
        'Content-Type': 'application/json',
      },
      method: 'POST',
--- a/app/server/lib/GristServer.ts
+++ b/app/server/lib/GristServer.ts
@ -35,6 +35,7 @@ export interface GristServer {
  settings?: Readonly<Record<string, unknown>>;
  getHost(): string;
  getHomeUrl(req: express.Request, relPath?: string): string;
+  getHomeInternalUrl(relPath?: string): string;
  getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
  getOwnUrl(): string;
  getOrgUrl(orgKey: string|number): Promise<string>;
@ -66,6 +67,7 @@ export interface GristServer {
  getBundledWidgets(): ICustomWidget[];
  hasBoot(): boolean;
  getSandboxInfo(): SandboxInfo|undefined;
+  getInfo(key: string): any;
 }

 export interface GristLoginSystem {
@ -127,6 +129,7 @@ export function createDummyGristServer(): GristServer {
    settings: {},
    getHost() { return 'localhost:4242'; },
    getHomeUrl() { return 'http://localhost:4242'; },
+    getHomeInternalUrl() { return 'http://localhost:4242'; },
    getHomeUrlByDocId() { return Promise.resolve('http://localhost:4242'); },
    getMergedOrgUrl() { return 'http://localhost:4242'; },
    getOwnUrl() { return 'http://localhost:4242'; },
@ -157,6 +160,7 @@ export function createDummyGristServer(): GristServer {
    getBundledWidgets() { return []; },
    hasBoot() { return false; },
    getSandboxInfo() { return undefined; },
+    getInfo(key: string) { return undefined; }
  };
 }

--- a/app/server/lib/ITestingHooks-ti.ts
+++ b/app/server/lib/ITestingHooks-ti.ts
@ -11,7 +11,6 @@ export const ClientJsonMemoryLimits = t.iface([], {
 });

 export const ITestingHooks = t.iface([], {
-  "getOwnPort": t.func("number"),
  "getPort": t.func("number"),
  "setLoginSessionProfile": t.func("void", t.param("gristSidCookie", "string"), t.param("profile", t.union("UserProfile", "null")), t.param("org", "string", true)),
  "setServerVersion": t.func("void", t.param("version", t.union("string", "null"))),
--- a/app/server/lib/ITestingHooks.ts
+++ b/app/server/lib/ITestingHooks.ts
@ -7,7 +7,6 @@ export interface ClientJsonMemoryLimits {
 }

 export interface ITestingHooks {
-  getOwnPort(): Promise<number>;
  getPort(): Promise<number>;
  setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise<void>;
  setServerVersion(version: string|null): Promise<void>;
--- a/app/server/lib/TestingHooks.ts
+++ b/app/server/lib/TestingHooks.ts
@ -68,11 +68,6 @@ export class TestingHooks implements ITestingHooks {
    private _workerServers: FlexServer[]
  ) {}

-  public async getOwnPort(): Promise<number> {
-    log.info("TestingHooks.getOwnPort called");
-    return this._server.getOwnPort();
-  }
-
  public async getPort(): Promise<number> {
    log.info("TestingHooks.getPort called");
    return this._port;
--- a/app/server/lib/requestUtils.ts
+++ b/app/server/lib/requestUtils.ts
@ -1,5 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
-import {DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail} from 'app/common/gristUrls';
+import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
 import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
--- a/app/server/lib/sendAppPage.ts
+++ b/app/server/lib/sendAppPage.ts
@ -4,6 +4,7 @@ import {
  getFreeCoachingCallUrl,
  getHelpCenterUrl,
  getPageTitleSuffix,
+  getTermsOfServiceUrl,
  GristLoadConfig,
  IFeature
 } from 'app/common/gristUrls';
@ -62,6 +63,7 @@ export function makeGristConfig(options: MakeGristConfigOptions): GristLoadConfi
    baseDomain,
    singleOrg: process.env.GRIST_SINGLE_ORG,
    helpCenterUrl: getHelpCenterUrl(),
+    termsOfServiceUrl: getTermsOfServiceUrl(),
    freeCoachingCallUrl: getFreeCoachingCallUrl(),
    contactSupportUrl: getContactSupportUrl(),
    pathOnly,
--- a/app/server/lib/uploads.ts
+++ b/app/server/lib/uploads.ts
@ -1,7 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
-import {getDocWorkerUrl} from 'app/common/UserAPI';
+import {getUrlFromPrefix} from 'app/common/UserAPI';
 import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
        RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
@ -21,6 +21,8 @@ import * as multiparty from 'multiparty';
 import fetch, {Response as FetchResponse} from 'node-fetch';
 import * as path from 'path';
 import * as tmp from 'tmp';
+import {IDocWorkerMap} from './DocWorkerMap';
+import {getDocWorkerInfoOrSelfPrefix} from './DocWorkerUtils';

 // After some time of inactivity, clean up the upload. We give an hour, which seems generous,
 // except that if one is toying with import options, and leaves the upload in an open browser idle
@ -39,7 +41,12 @@ export interface FormResult {
 /**
 * Adds an upload route to the given express app, listening for POST requests at UPLOAD_URL_PATH.
 */
-export function addUploadRoute(server: GristServer, expressApp: Application, ...handlers: RequestHandler[]): void {
+export function addUploadRoute(
+  server: GristServer,
+  expressApp: Application,
+  docWorkerMap: IDocWorkerMap,
+  ...handlers: RequestHandler[]
+): void {

  // When doing a cross-origin post, the browser will check for access with options prior to posting.
  // We need to reassure it that the request will be accepted before it will go ahead and post.
@ -72,7 +79,7 @@ export function addUploadRoute(server: GristServer, expressApp: Application, ...
    if (!docId) { throw new Error('doc must be specified'); }
    const accessId = makeAccessId(req, getAuthorizedUserId(req));
    try {
-      const uploadResult: UploadResult = await fetchDoc(server, docId, req, accessId,
+      const uploadResult: UploadResult = await fetchDoc(server, docWorkerMap, docId, req, accessId,
                                                        req.query.template === '1');
      if (name) {
        globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, name);
@ -404,24 +411,21 @@ async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlO
 * Fetches a Grist doc potentially managed by a different doc worker.  Passes on credentials
 * supplied in the current request.
 */
-export async function fetchDoc(server: GristServer, docId: string, req: Request, accessId: string|null,
-                        template: boolean): Promise<UploadResult> {
+export async function fetchDoc(
+  server: GristServer,
+  docWorkerMap: IDocWorkerMap,
+  docId: string,
+  req: Request,
+  accessId: string|null,
+  template: boolean
+): Promise<UploadResult> {
  // Prepare headers that preserve credentials of current user.
-  const headers = getTransitiveHeaders(req);
-
-  // Passing the Origin header would serve no purpose here, as we are
-  // constructing an internal request to fetch from our own doc worker
-  // URL. Indeed, it may interfere, as it could incur a CORS check in
-  // `trustOrigin`, which we do not need.
-  delete headers.Origin;
+  const headers = getTransitiveHeaders(req, { includeOrigin: false });

  // Find the doc worker responsible for the document we wish to copy.
  // The backend needs to be well configured for this to work.
-  const homeUrl = server.getHomeUrl(req);
-  const fetchUrl = new URL(`/api/worker/${docId}`, homeUrl);
-  const response: FetchResponse = await Deps.fetch(fetchUrl.href, {headers});
-  await _checkForError(response);
-  const docWorkerUrl = getDocWorkerUrl(server.getOwnUrl(), await response.json());
+  const { selfPrefix, docWorker } = await getDocWorkerInfoOrSelfPrefix(docId, docWorkerMap, server.getTag());
+  const docWorkerUrl = docWorker ? docWorker.internalUrl : getUrlFromPrefix(server.getHomeInternalUrl(), selfPrefix);
  // Download the document, in full or as a template.
  const url = new URL(`api/docs/${docId}/download?template=${Number(template)}`,
                      docWorkerUrl.replace(/\/*$/, '/'));
--- a/package.json
+++ b/package.json
@ -1,6 +1,6 @@
 {
  "name": "grist-core",
-  "version": "1.1.13",
+  "version": "1.1.14",
  "license": "Apache-2.0",
  "description": "Grist is the evolution of spreadsheets",
  "homepage": "https://github.com/gristlabs/grist-core",
--- a/sandbox/grist/gencode.py
+++ b/sandbox/grist/gencode.py
@ -17,7 +17,7 @@ The schema for grist data is:
 """
 import logging
 import re
-import imp
+import types
 from collections import OrderedDict

 import six
@ -207,7 +207,7 @@ def _is_special_table(table_id):


 def exec_module_text(module_text):
-  mod = imp.new_module(codebuilder.code_filename)
+  mod = types.ModuleType(codebuilder.code_filename)
  codebuilder.save_to_linecache(module_text)
  code_obj = compile(module_text, codebuilder.code_filename, "exec")
  # pylint: disable=exec-used
--- a/static/locales/bg.client.json
+++ b/static/locales/bg.client.json
--- a/static/locales/de.client.json
+++ b/static/locales/de.client.json
@ -308,7 +308,34 @@
        "Ok": "OK",
        "Webhooks": "Webhaken",
        "Manage Webhooks": "Webhaken verwalten",
-        "API Console": "API-Konsole"
+        "API Console": "API-Konsole",
+        "Coming soon": "Demnächst verfügbar",
+        "For number and date formats": "Für Zahlen- und Datumsformate",
+        "Formula times": "Formelzeiten",
+        "Locale": "Lokale",
+        "Time Zone": "Zeitzone",
+        "Find slow formulas": "Langsame Formeln finden",
+        "Manage webhooks": "Webhooks verwalten",
+        "For currency columns": "Für Währungsspalten",
+        "Notify other services on doc changes": "Benachrichtigen Sie andere Dienstleistungen bei doc Änderungen",
+        "Hard reset of data engine": "Hartes Zurücksetzen der Datenmaschine",
+        "Python": "Python",
+        "ID for API use": "ID für API-Verwendung",
+        "Python version used": "Verwendete Python-Version",
+        "Reload": "Neu laden",
+        "Try API calls from the browser": "Versuchen Sie API-Aufrufe über den Browser",
+        "python2 (legacy)": "python2 (veraltet)",
+        "python3 (recommended)": "python3 (empfohlen)",
+        "API URL copied to clipboard": "API-URL in die Zwischenablage kopiert",
+        "Base doc URL: {{docApiUrl}}": "URL des Basisdokuments: {{docApiUrl}}",
+        "API console": "API-Konsole",
+        "API documentation.": "API Dokumentation.",
+        "Copy to clipboard": "In die Zwischenablage kopieren",
+        "Currency": "Währung",
+        "Data Engine": "Datenmaschine",
+        "Default for DateTime columns": "Standard für DateTime-Spalten",
+        "Document ID": "Dokument-ID",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "Dokument-ID, die bei Aufrufen der REST-API für {{docId}} zu verwenden ist. Siehe {{apiURL}}"
    },
    "DocumentUsage": {
        "Attachments Size": "Größe der Anhänge",
@ -607,7 +634,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Beenden",
-        "Next": "Weiter"
+        "Next": "Weiter",
+        "Previous": "Vorherige"
    },
    "OpenVideoTour": {
        "Grist Video Tour": "Grist Video Tour",
@ -981,7 +1009,8 @@
        "Dismiss": "Ablehnen",
        "Don't ask again.": "Frag nicht mehr.",
        "Don't show again.": "Zeig nicht mehr.",
-        "Don't show tips": "Keine Tipps anzeigen"
+        "Don't show tips": "Keine Tipps anzeigen",
+        "TIP": "TIPP"
    },
    "pages": {
        "Duplicate Page": "Seite duplizieren",
@ -1607,5 +1636,20 @@
    },
    "DropdownConditionEditor": {
        "Enter condition.": "Bedingung eingeben."
+    },
+    "FormRenderer": {
+        "Reset": "Zurücksetzen",
+        "Submit": "Einreichen",
+        "Search": "Suchen",
+        "Select...": "Wählen Sie..."
+    },
+    "widgetTypesMap": {
+        "Calendar": "Kalender",
+        "Card": "Karte",
+        "Card List": "Kartenliste",
+        "Form": "Formular",
+        "Table": "Tabelle",
+        "Chart": "Diagramm",
+        "Custom": "Benutzerdefiniert"
    }
 }
--- a/static/locales/en.client.json
+++ b/static/locales/en.client.json
@ -300,7 +300,34 @@
        "Ok": "OK",
        "Manage Webhooks": "Manage Webhooks",
        "Webhooks": "Webhooks",
-        "API Console": "API Console"
+        "API Console": "API Console",
+        "API URL copied to clipboard": "API URL copied to clipboard",
+        "API console": "API console",
+        "API documentation.": "API documentation.",
+        "Base doc URL: {{docApiUrl}}": "Base doc URL: {{docApiUrl}}",
+        "Coming soon": "Coming soon",
+        "Copy to clipboard": "Copy to clipboard",
+        "Currency": "Currency",
+        "Data Engine": "Data Engine",
+        "Default for DateTime columns": "Default for DateTime columns",
+        "Document ID": "Document ID",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}",
+        "Find slow formulas": "Find slow formulas",
+        "For currency columns": "For currency columns",
+        "For number and date formats": "For number and date formats",
+        "Formula times": "Formula times",
+        "Hard reset of data engine": "Hard reset of data engine",
+        "ID for API use": "ID for API use",
+        "Locale": "Locale",
+        "Manage webhooks": "Manage webhooks",
+        "Notify other services on doc changes": "Notify other services on doc changes",
+        "Python": "Python",
+        "Python version used": "Python version used",
+        "Reload": "Reload",
+        "Time Zone": "Time Zone",
+        "Try API calls from the browser": "Try API calls from the browser",
+        "python2 (legacy)": "python2 (legacy)",
+        "python3 (recommended)": "python3 (recommended)"
    },
    "DocumentUsage": {
        "Attachments Size": "Size of Attachments",
@ -567,7 +594,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Finish",
-        "Next": "Next"
+        "Next": "Next",
+        "Previous": "Previous"
    },
    "OpenVideoTour": {
        "Grist Video Tour": "Grist Video Tour",
@ -921,7 +949,8 @@
        "Don't show tips": "Don't show tips",
        "Undo to restore": "Undo to restore",
        "Got it": "Got it",
-        "Don't show again": "Don't show again"
+        "Don't show again": "Don't show again",
+        "TIP": "TIP"
    },
    "pages": {
        "Duplicate Page": "Duplicate Page",
@ -1543,5 +1572,20 @@
        "Error in dropdown condition": "Error in dropdown condition",
        "No choices matching condition": "No choices matching condition",
        "No choices to select": "No choices to select"
+    },
+    "FormRenderer": {
+        "Reset": "Reset",
+        "Search": "Search",
+        "Select...": "Select...",
+        "Submit": "Submit"
+    },
+    "widgetTypesMap": {
+        "Calendar": "Calendar",
+        "Card": "Card",
+        "Card List": "Card List",
+        "Chart": "Chart",
+        "Custom": "Custom",
+        "Form": "Form",
+        "Table": "Table"
    }
 }
--- a/static/locales/es.client.json
+++ b/static/locales/es.client.json
@ -250,7 +250,34 @@
        "API": "API",
        "Webhooks": "Ganchos Web",
        "Manage Webhooks": "Administrar los ganchos web",
-        "API Console": "Consola de la API"
+        "API Console": "Consola de la API",
+        "API URL copied to clipboard": "URL de API copiada en el portapapeles",
+        "API documentation.": "Documentación de la API.",
+        "Base doc URL: {{docApiUrl}}": "URL del doc base: {{docApiUrl}}",
+        "Currency": "Moneda",
+        "Data Engine": "Motor de datos",
+        "Formula times": "Tiempo de fórmula",
+        "Hard reset of data engine": "Reinicio completo del motor de datos",
+        "ID for API use": "ID para uso de API",
+        "Locale": "Configuración regional",
+        "API console": "Consola de la API",
+        "Coming soon": "Próximamente",
+        "Copy to clipboard": "Copiar al portapapeles",
+        "Default for DateTime columns": "Predeterminado para las columnas DateTime",
+        "Document ID": "ID del documento",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID de documento para usar cuando la API REST solicite {{docId}}. Véase {{apiURL}}",
+        "Find slow formulas": "Encontrar fórmulas lentas",
+        "For currency columns": "Para columnas de moneda",
+        "For number and date formats": "Para formatos de número y fecha",
+        "Manage webhooks": "Administrar los ganchos web",
+        "Python version used": "Versión Python utilizada",
+        "Time Zone": "Huso horario",
+        "Try API calls from the browser": "Pruebe las llamadas API desde el navegador",
+        "Reload": "Recargar",
+        "python2 (legacy)": "python2 (legado)",
+        "Notify other services on doc changes": "Notificar a otros servicios los cambios de documentos",
+        "Python": "Python",
+        "python3 (recommended)": "python3 (recomendado)"
    },
    "DuplicateTable": {
        "Copy all data in addition to the table structure.": "Copiar todos los datos además de la estructura de la tabla.",
@ -503,7 +530,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Finalizar",
-        "Next": "Siguiente"
+        "Next": "Siguiente",
+        "Previous": "Anterior"
    },
    "OpenVideoTour": {
        "Grist Video Tour": "Recorrido en video de Grist",
@ -986,7 +1014,8 @@
        "Don't ask again.": "No preguntes de nuevo.",
        "Don't show again.": "No vuelvas a mostrarlo.",
        "Don't show again": "No volver a mostrar",
-        "Are you sure you want to delete these records?": "¿Seguro que quieres borrar estos registros?"
+        "Are you sure you want to delete these records?": "¿Seguro que quieres borrar estos registros?",
+        "TIP": "TIP"
    },
    "pages": {
        "Duplicate Page": "Duplicar página",
@ -1597,5 +1626,20 @@
    },
    "DropdownConditionEditor": {
        "Enter condition.": "Introduzca la condición."
+    },
+    "FormRenderer": {
+        "Select...": "Seleccione...",
+        "Reset": "Restablecer",
+        "Search": "Búsqueda",
+        "Submit": "Enviar"
+    },
+    "widgetTypesMap": {
+        "Calendar": "Calendario",
+        "Card": "Tarjeta",
+        "Card List": "Lista de tarjetas",
+        "Chart": "Gráfico",
+        "Custom": "Personalizado",
+        "Form": "Formulario",
+        "Table": "Tabla"
    }
 }
--- a/static/locales/fr.client.json
+++ b/static/locales/fr.client.json
@ -296,7 +296,9 @@
        "Ok": "OK",
        "Manage Webhooks": "Gérer les points d’ancrage Web",
        "Webhooks": "Points d’ancrage Web",
-        "API Console": "Console de l'API"
+        "API Console": "Console de l'API",
+        "Reload": "Recharger",
+        "Python": "Python"
    },
    "DocumentUsage": {
        "Usage statistics are only available to users with full access to the document data.": "Les statistiques d'utilisation ne sont disponibles qu'aux utilisateurs ayant un accès complet aux données du document.",
@ -495,7 +497,8 @@
        "Workspace will be moved to Trash.": "Le dossier va être mis à la corbeille.",
        "Manage Users": "Gérer les utilisateurs",
        "Access Details": "Détails d'accès",
-        "Tutorial": "Tutoriel"
+        "Tutorial": "Tutoriel",
+        "Terms of service": "CGU"
    },
    "Importer": {
        "Update existing records": "Mettre à jour les enregistrements existants",
@ -564,7 +567,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Terminer",
-        "Next": "Suivant"
+        "Next": "Suivant",
+        "Previous": "Précédent"
    },
    "OpenVideoTour": {
        "YouTube video player": "Lecteur vidéo YouTube",
@ -1146,7 +1150,9 @@
        "Learn more": "En savoir plus",
        "Build simple forms right in Grist and share in a click with our new widget. {{learnMoreButton}}": "Créez des formulaires simples directement dans Grist et partagez-les en un clic avec notre nouveau widget. {{learnMoreButton}}",
        "Forms are here!": "Les formulaires sont là !",
-        "These rules are applied after all column rules have been processed, if applicable.": "Ces règles sont appliquées après le traitement de toutes les règles de la colonne, le cas échéant."
+        "These rules are applied after all column rules have been processed, if applicable.": "Ces règles sont appliquées après le traitement de toutes les règles de la colonne, le cas échéant.",
+        "Example: {{example}}": "Exemple : {{example}}",
+        "Filter displayed dropdown values with a condition.": "Filtrer les valeurs affichées dans la liste déroulante en fonction d'une condition."
    },
    "ColumnTitle": {
        "Add description": "Ajouter une description",
@ -1473,7 +1479,24 @@
        "Admin Panel": "Panneau d'administration",
        "Sponsor": "Parrainage",
        "Support Grist": "Soutenir Grist",
-        "Version": "Version"
+        "Version": "Version",
+        "Check now": "Vérifier",
+        "Checking for updates...": "Vérifier les mises à jour...",
+        "Error": "Erreur",
+        "Error checking for updates": "Erreur lors de la vérification des mises à jour",
+        "Grist is up to date": "Grist est à jour",
+        "Last checked {{time}}": "Dernière vérification {{time}}",
+        "Learn more.": "En savoir plus.",
+        "No information available": "Aucune information disponible",
+        "OK": "OK",
+        "Security Settings": "Paramètres de sécurité",
+        "Grist releases are at ": "Les releases de Grist sont disponibles sur ",
+        "Newer version available": "Nouvelle version disponible",
+        "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist permet d'utiliser des formules très puissantes, en utilisant Python. Nous recommandons de définir la variable d'environnement GRIST_SANDBOX_FLAVOR sur gvisor si votre machine le supporte (la plupart le supportent), afin d'exécuter des formules dans chaque document à l'intérieur d'une sandbox isolée des autres documents et du réseau.",
+        "Updates": "Mises à jour",
+        "unconfigured": "non configuré",
+        "unknown": "inconnu",
+        "Auto-check when this page loads": "Vérification automatique au chargement de cette page"
    },
    "Field": {
        "No choices configured": "Aucun choix configuré",
@ -1500,5 +1523,43 @@
    },
    "Columns": {
        "Remove Column": "Supprimer la colonne"
+    },
+    "ChoiceEditor": {
+        "Error in dropdown condition": "Erreur dans la condition de la liste déroulante",
+        "No choices to select": "Aucun choix à sélectionner",
+        "No choices matching condition": "Aucun choix correspondant à la condition"
+    },
+    "ChoiceListEditor": {
+        "Error in dropdown condition": "Erreur dans la condition de la liste déroulante",
+        "No choices to select": "Aucun choix à sélectionner",
+        "No choices matching condition": "Aucun choix correspondant à la condition"
+    },
+    "DropdownConditionConfig": {
+        "Invalid columns: {{colIds}}": "Colonnes invalides : {{colIds}}",
+        "Set dropdown condition": "Définir la condition de la liste déroulante",
+        "Dropdown Condition": "Condition de la liste déroulante"
+    },
+    "DropdownConditionEditor": {
+        "Enter condition.": "Saisir la condition."
+    },
+    "ReferenceUtils": {
+        "Error in dropdown condition": "Erreur dans la condition de la liste déroulant",
+        "No choices matching condition": "Aucun choix correspondant à la condition",
+        "No choices to select": "Aucun choix à sélectionner"
+    },
+    "FormRenderer": {
+        "Submit": "Soumettre",
+        "Select...": "Sélectionner...",
+        "Search": "Rechercher",
+        "Reset": "Réinitialiser"
+    },
+    "widgetTypesMap": {
+        "Table": "Table",
+        "Form": "Formulaire",
+        "Custom": "Personnalisée",
+        "Chart": "Graphique",
+        "Card List": "Liste de fiches",
+        "Card": "Fiche",
+        "Calendar": "Calendrier"
    }
 }
--- a/static/locales/it.client.json
+++ b/static/locales/it.client.json
@ -699,7 +699,34 @@
        "Ok": "OK",
        "Manage Webhooks": "Gestisci web hook",
        "Webhooks": "Web hook",
-        "API Console": "Tavola delle API"
+        "API Console": "Tavola delle API",
+        "API console": "Console Api",
+        "Coming soon": "In arrivo",
+        "Copy to clipboard": "Copia negli appunti",
+        "Data Engine": "Motore dati",
+        "Default for DateTime columns": "Default per le colonne Data/Ora",
+        "Document ID": "ID documento",
+        "Find slow formulas": "Trova le formule lente",
+        "For currency columns": "Per le colonne di valuta",
+        "For number and date formats": "Per i formati di numeri e date",
+        "Formula times": "Tempi delle formule",
+        "Hard reset of data engine": "Reset forzato del motore dati",
+        "ID for API use": "ID per l'uso con le Api",
+        "Locale": "Locale",
+        "Manage webhooks": "Gestisci i webhooks",
+        "Notify other services on doc changes": "Notifica altri servizi dei cambiamenti nel documento",
+        "Python": "Python",
+        "Reload": "Ricarica",
+        "Time Zone": "Fuso orario",
+        "python2 (legacy)": "Python 2 (superato)",
+        "python3 (recommended)": "Python 3 (raccomandato)",
+        "API URL copied to clipboard": "Url della Api copiata negli appunti",
+        "API documentation.": "Documentazione Api.",
+        "Base doc URL: {{docApiUrl}}": "Url documento base: {{docApiUrl}}",
+        "Currency": "Valuta",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID Documento da usare quando le Api REST chiedono {{docId}}. Vedi {{apiURL}}",
+        "Python version used": "Versione di Python in uso",
+        "Try API calls from the browser": "Prova le chiamate Api nel browser"
    },
    "DocumentUsage": {
        "Data Size": "Dimensione dei dati",
@ -860,7 +887,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Termina",
-        "Next": "Prossimo"
+        "Next": "Prossimo",
+        "Previous": "Precedente"
    },
    "OpenVideoTour": {
        "Grist Video Tour": "Video tour di Grist",
@ -1041,7 +1069,8 @@
        "Dismiss": "Ignora",
        "Don't ask again.": "Non chiedere più.",
        "Don't show again.": "Non mostrare più.",
-        "Don't show tips": "Non mostrare i suggerimenti"
+        "Don't show tips": "Non mostrare i suggerimenti",
+        "TIP": "TIP"
    },
    "pages": {
        "Duplicate Page": "Duplica pagina",
@ -1145,7 +1174,9 @@
        "These rules are applied after all column rules have been processed, if applicable.": "Queste regole sono applicate dopo che tutte le regole delle colonne sono state applicate, se possibile.",
        "Build simple forms right in Grist and share in a click with our new widget. {{learnMoreButton}}": "Costruisci del semplici moduli e condividili rapidamente con il nostro nuovo widget. {{learnMoreButton}}",
        "Forms are here!": "Sono arrivati i moduli!",
-        "Learn more": "Approfondisci"
+        "Learn more": "Approfondisci",
+        "Example: {{example}}": "Esempio: {{example}}",
+        "Filter displayed dropdown values with a condition.": "Il filtro mostrava i valori nella tendina con una condizione."
    },
    "DescriptionConfig": {
        "DESCRIPTION": "DESCRIZIONE"
@ -1416,7 +1447,26 @@
        "Sponsor": "Sponsor",
        "Current": "Attuale",
        "Telemetry": "Telemetria",
-        "Version": "Versione"
+        "Version": "Versione",
+        "Auto-check when this page loads": "Controlla automaticamente quando questa pagina è caricata",
+        "Grist is up to date": "Grist è aggiornato",
+        "Error checking for updates": "Errore nel controllo degli aggiornamenti",
+        "Grist allows for very powerful formulas, using Python. We recommend setting the environment variable GRIST_SANDBOX_FLAVOR to gvisor if your hardware supports it (most will), to run formulas in each document within a sandbox isolated from other documents and isolated from the network.": "Grist supporta formule molto potenti, grazie a Python. Raccomandiamo di impostare la variabile d'ambiente GRIST_SANDBOX_FLAVOR a gvisor se il vostro hardware lo permette (in genere sì), così che le formule di un documento agiscano in una sandbox isolata dagli altri documenti e dalla rete.",
+        "Last checked {{time}}": "Ultimo controllo {{time}}",
+        "Learn more.": "Per saperne di più.",
+        "Newer version available": "Disponibile una versione più recente",
+        "No information available": "Nessuna informazione disponibile",
+        "OK": "OK",
+        "Sandboxing": "Uso della sandbox",
+        "Security Settings": "Impostazioni di sicurezza",
+        "Updates": "Aggiornamenti",
+        "unconfigured": "non configurato",
+        "unknown": "sconosciuto",
+        "Error": "Errore",
+        "Check now": "Controlla adesso",
+        "Checking for updates...": "Controllo gli aggiornamenti...",
+        "Grist releases are at ": "Le release di Grist sono a ",
+        "Sandbox settings for data engine": "Impostazione della sandbox per il motore dati"
    },
    "WelcomeCoachingCall": {
        "Maybe Later": "Forse più tardi",
@ -1499,5 +1549,43 @@
    },
    "FormErrorPage": {
        "Error": "Errore"
+    },
+    "ChoiceEditor": {
+        "Error in dropdown condition": "Errore nella condizione della tendina",
+        "No choices matching condition": "Nessuna scelta soddisfa la condizione",
+        "No choices to select": "Nessuna scelta da selezionare"
+    },
+    "ReferenceUtils": {
+        "No choices to select": "Nessuna scelta da selezionare",
+        "Error in dropdown condition": "Errore nella condizione della tendina",
+        "No choices matching condition": "Nessuna scelta soddisfa la condizione"
+    },
+    "DropdownConditionConfig": {
+        "Set dropdown condition": "Imposta la condizione della tendina",
+        "Dropdown Condition": "Condizione della tendina",
+        "Invalid columns: {{colIds}}": "Colonne non valide: {{colIds}}"
+    },
+    "DropdownConditionEditor": {
+        "Enter condition.": "Inserisci la condizione."
+    },
+    "ChoiceListEditor": {
+        "Error in dropdown condition": "Errore nella condizione della tendina",
+        "No choices matching condition": "Nessuna scelta soddisfa la condizione",
+        "No choices to select": "Nessuna scelta da selezionare"
+    },
+    "FormRenderer": {
+        "Reset": "Reset",
+        "Search": "Cerca",
+        "Select...": "Seleziona...",
+        "Submit": "Invia"
+    },
+    "widgetTypesMap": {
+        "Card": "Scheda",
+        "Table": "Tabella",
+        "Calendar": "Calendario",
+        "Card List": "Lista di schede",
+        "Chart": "Grafico",
+        "Custom": "Personalizzato",
+        "Form": "Modulo"
    }
 }
--- a/static/locales/pt_BR.client.json
+++ b/static/locales/pt_BR.client.json
@ -308,7 +308,34 @@
        "API": "API",
        "Manage Webhooks": "Gerenciar ganchos web",
        "Webhooks": "Ganchos Web",
-        "API Console": "Consola API"
+        "API Console": "Consola API",
+        "API documentation.": "Documentação de API.",
+        "Currency": "Moeda",
+        "Data Engine": "Motor de dados",
+        "Find slow formulas": "Encontrar fórmulas lentas",
+        "For currency columns": "Para colunas de moeda",
+        "Hard reset of data engine": "Reinicialização total do motor de dados",
+        "Manage webhooks": "Gerenciar webhooks",
+        "Python": "Python",
+        "Python version used": "Versão Python usada",
+        "Try API calls from the browser": "Experimente chamadas de API do navegador",
+        "python2 (legacy)": "python2 (legado)",
+        "API URL copied to clipboard": "URL da API copiado para a área de transferência",
+        "Coming soon": "Em breve",
+        "API console": "Consola API",
+        "Base doc URL: {{docApiUrl}}": "URL do documento base: {{docApiUrl}}",
+        "Copy to clipboard": "Copiar para a área de transferência",
+        "Default for DateTime columns": "Padrão para colunas DataHorário",
+        "Document ID": "ID do documento",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID do documento a ser usado sempre que a API REST fizer uma chamada para {{docId}}. Veja {{apiURL}}",
+        "For number and date formats": "Para formatos de número e data",
+        "Formula times": "Tempos de fórmula",
+        "Locale": "Localização",
+        "ID for API use": "ID para uso da API",
+        "Notify other services on doc changes": "Notifique outros serviços em alterações doc",
+        "Reload": "Recarregar",
+        "Time Zone": "Fuso horário",
+        "python3 (recommended)": "python3 (recomendado)"
    },
    "DocumentUsage": {
        "Attachments Size": "Tamanho dos Anexos",
@ -607,7 +634,8 @@
    },
    "OnBoardingPopups": {
        "Finish": "Terminar",
-        "Next": "Próximo"
+        "Next": "Próximo",
+        "Previous": "Anterior"
    },
    "OpenVideoTour": {
        "Grist Video Tour": "Tour de Vídeo Grist",
@ -981,7 +1009,8 @@
        "Got it": "Entendido",
        "Don't show again": "Não mostrar novamente",
        "Don't show again.": "Não mostrar novamente.",
-        "Don't show tips": "Não mostrar dicas"
+        "Don't show tips": "Não mostrar dicas",
+        "TIP": "DICA"
    },
    "pages": {
        "Duplicate Page": "Duplicar a Página",
@ -1607,5 +1636,20 @@
        "Error in dropdown condition": "Erro na condição do menu suspenso",
        "No choices matching condition": "Nenhuma opção que corresponda à condição",
        "No choices to select": "Não há opções para selecionar"
+    },
+    "FormRenderer": {
+        "Reset": "Redefinir",
+        "Search": "Pesquisar",
+        "Select...": "Selecionar...",
+        "Submit": "Enviar"
+    },
+    "widgetTypesMap": {
+        "Calendar": "Calendário",
+        "Card": "Cartão",
+        "Card List": "Lista de cartões",
+        "Chart": "Gráfico",
+        "Custom": "Personalizado",
+        "Table": "Tabela",
+        "Form": "Formulário"
    }
 }
--- a/static/locales/ru.client.json
+++ b/static/locales/ru.client.json
@ -1315,7 +1315,7 @@
        "Opt out of Telemetry": "Отказаться от телеметрии",
        "Sponsor Grist Labs on GitHub": "Спонсор Grist Labs на GitHub",
        "This instance is opted in to telemetry. Only the site administrator has permission to change this.": "В этом экземпляре включена телеметрия. Только администратор сайта может изменить это.",
-        "We only collect usage statistics, as detailed in our {{link}}, never document contents.": "Мы собираем только статистику использования, как описано в нашей {{link}}, никогда содержимое документов.",
+        "We only collect usage statistics, as detailed in our {{link}}, never document contents.": "Мы собираем только статистику использования, как описано в {{link}} и никогда не собираем содержимое документов.",
        "Sponsor": "Спонсор"
    },
    "buildViewSectionDom": {
--- a/static/locales/sl.client.json
+++ b/static/locales/sl.client.json
@ -497,7 +497,22 @@
        "Manage Webhooks": "Upravljanje spletnih kljuk",
        "Webhooks": "Spletne kljuke",
        "Engine (experimental {{span}} change at own risk):": "Pogon (eksperimentalno {{span}} spreminjanje na lastno odgovornost):",
-        "API Console": "API Konzola"
+        "API Console": "API Konzola",
+        "API console": "API konzola",
+        "API documentation.": "API dokumentacija.",
+        "Base doc URL: {{docApiUrl}}": "URL osnovnega dokumenta: {{docApiUrl}}",
+        "Coming soon": "Prihaja kmalu",
+        "Copy to clipboard": "Kopiraj v odložišče",
+        "Currency": "Valuta",
+        "Data Engine": "Podatkovna mašina",
+        "Document ID": "ID dokumenta",
+        "Document ID to use whenever the REST API calls for {{docId}}. See {{apiURL}}": "ID dokumenta, ki se uporabi vsakič, ko REST API  pokliče {{docId}}. Oglej si {{apiURL}}",
+        "Find slow formulas": "Poišči počasne formule",
+        "For currency columns": "Za valutne stolpce",
+        "For number and date formats": "Za format števila in datuma",
+        "Formula times": "Časi formule",
+        "API URL copied to clipboard": "URL API-ja kopiran v odložišče",
+        "Default for DateTime columns": "Privzeto za stolpce DateTime"
    },
    "GridOptions": {
        "Horizontal Gridlines": "Vodoravne linije",
--- a/test/nbrowser/TermsOfService.ts
+++ b/test/nbrowser/TermsOfService.ts
@ -0,0 +1,37 @@
+import { assert, driver } from 'mocha-webdriver';
+import * as gu from 'test/nbrowser/gristUtils';
+import { server, setupTestSuite } from 'test/nbrowser/testUtils';
+import { EnvironmentSnapshot } from 'test/server/testUtils';
+
+describe('Terms of service link', function () {
+  this.timeout(20000);
+  setupTestSuite({samples: true});
+
+  let session: gu.Session;
+  let oldEnv: EnvironmentSnapshot;
+
+  before(async function () {
+    oldEnv = new EnvironmentSnapshot();
+    session = await gu.session().teamSite.login();
+  });
+
+  after(async function () {
+    oldEnv.restore();
+    await server.restart();
+  });
+
+  it('is visible in home menu', async function () {
+    process.env.GRIST_TERMS_OF_SERVICE_URL = 'https://example.com/tos';
+    await server.restart();
+    await session.loadDocMenu('/');
+    assert.isTrue(await driver.find('.test-dm-tos').isDisplayed());
+    assert.equal(await driver.find('.test-dm-tos').getAttribute('href'), 'https://example.com/tos');
+  });
+
+  it('is not visible when environment variable is not set', async function () {
+    delete process.env.GRIST_TERMS_OF_SERVICE_URL;
+    await server.restart();
+    await session.loadDocMenu('/');
+    assert.isFalse(await driver.find('.test-dm-tos').isPresent());
+  });
+});
--- a/test/server/lib/DocApi.ts
+++ b/test/server/lib/DocApi.ts
@ -34,7 +34,7 @@ import {serveSomething, Serving} from 'test/server/customUtil';
 import {prepareDatabase} from 'test/server/lib/helpers/PrepareDatabase';
 import {prepareFilesystemDirectoryForTests} from 'test/server/lib/helpers/PrepareFilesystemDirectoryForTests';
 import {signal} from 'test/server/lib/helpers/Signal';
-import {TestServer} from 'test/server/lib/helpers/TestServer';
+import {TestServer, TestServerReverseProxy} from 'test/server/lib/helpers/TestServer';
 import * as testUtils from 'test/server/testUtils';
 import {waitForIt} from 'test/server/wait';
 import defaultsDeep = require('lodash/defaultsDeep');
@ -42,12 +42,6 @@ import pick = require('lodash/pick');
 import { getDatabase } from 'test/testUtils';
 import {testDailyApiLimitFeatures} from 'test/gen-server/seed';

-const chimpy = configForUser('Chimpy');
-const kiwi = configForUser('Kiwi');
-const charon = configForUser('Charon');
-const nobody = configForUser('Anonymous');
-const support = configForUser('support');
-
 // some doc ids
 const docIds: { [name: string]: string } = {
  ApiDataRecordsTest: 'sampledocid_7',
@ -68,6 +62,18 @@ let hasHomeApi: boolean;
 let home: TestServer;
 let docs: TestServer;
 let userApi: UserAPIImpl;
+let extraHeadersForConfig = {};
+
+function makeConfig(username: string): AxiosRequestConfig {
+  const originalConfig = configForUser(username);
+  return {
+    ...originalConfig,
+    headers: {
+      ...originalConfig.headers,
+      ...extraHeadersForConfig
+    }
+  };
+}

 describe('DocApi', function () {
  this.timeout(30000);
@ -77,12 +83,7 @@ describe('DocApi', function () {
  before(async function () {
    oldEnv = new testUtils.EnvironmentSnapshot();

-    // Clear redis test database if redis is in use.
-    if (process.env.TEST_REDIS_URL) {
-      const cli = createClient(process.env.TEST_REDIS_URL);
-      await cli.flushdbAsync();
-      await cli.quitAsync();
-    }
+    await flushAllRedis();

    // Create the tmp dir removing any previous one
    await prepareFilesystemDirectoryForTests(tmpDir);
@ -136,6 +137,7 @@ describe('DocApi', function () {
    });

    it('should not allow anonymous users to create new docs', async () => {
+      const nobody = makeConfig('Anonymous');
      const resp = await axios.post(`${serverUrl}/api/docs`, null, nobody);
      assert.equal(resp.status, 403);
    });
@ -158,6 +160,95 @@ describe('DocApi', function () {
      testDocApi();
    });

+    describe('behind a reverse-proxy', function () {
+      async function setupServersWithProxy(suitename: string, overrideEnvConf?: NodeJS.ProcessEnv) {
+        const proxy = new TestServerReverseProxy();
+        const additionalEnvConfiguration = {
+          ALLOWED_WEBHOOK_DOMAINS: `example.com,localhost:${webhooksTestPort}`,
+          GRIST_DATA_DIR: dataDir,
+          APP_HOME_URL: await proxy.getServerUrl(),
+          GRIST_ORG_IN_PATH: 'true',
+          GRIST_SINGLE_PORT: '0',
+          ...overrideEnvConf
+        };
+        const home = await TestServer.startServer('home', tmpDir, suitename, additionalEnvConfiguration);
+        const docs = await TestServer.startServer(
+          'docs', tmpDir, suitename, additionalEnvConfiguration, home.serverUrl
+        );
+        proxy.requireFromOutsideHeader();
+
+        await proxy.start(home, docs);
+
+        homeUrl = serverUrl = await proxy.getServerUrl();
+        hasHomeApi = true;
+        extraHeadersForConfig = {
+          Origin: serverUrl,
+          ...TestServerReverseProxy.FROM_OUTSIDE_HEADER,
+        };
+
+        return {proxy, home, docs};
+      }
+
+      async function tearDown(proxy: TestServerReverseProxy, servers: TestServer[]) {
+        proxy.stop();
+        for (const server of servers) {
+          await server.stop();
+        }
+        await flushAllRedis();
+      }
+
+      let proxy: TestServerReverseProxy;
+
+      describe('should run usual DocApi test', function () {
+        setup('behind-proxy-testdocs', async () => {
+          ({proxy, home, docs} = await setupServersWithProxy(suitename));
+        });
+
+        after(() => tearDown(proxy, [home, docs]));
+
+        testDocApi();
+      });
+
+      async function testCompareDocs(proxy: TestServerReverseProxy, home: TestServer) {
+        const chimpy = makeConfig('chimpy');
+        const userApiServerUrl = await proxy.getServerUrl();
+        const chimpyApi = home.makeUserApi(
+          ORG_NAME, 'chimpy', { serverUrl: userApiServerUrl, headers: chimpy.headers as Record<string, string> }
+        );
+        const ws1 = (await chimpyApi.getOrgWorkspaces('current'))[0].id;
+        const docId1 = await chimpyApi.newDoc({name: 'testdoc1'}, ws1);
+        const docId2 = await chimpyApi.newDoc({name: 'testdoc2'}, ws1);
+        const doc1 = chimpyApi.getDocAPI(docId1);
+
+        return doc1.compareDoc(docId2);
+      }
+
+      describe('with APP_HOME_INTERNAL_URL', function () {
+        setup('behind-proxy-with-apphomeinternalurl', async () => {
+          // APP_HOME_INTERNAL_URL will be set by TestServer.
+          ({proxy, home, docs} = await setupServersWithProxy(suitename));
+        });
+        after(() => tearDown(proxy, [home, docs]));
+
+        it('should succeed to compare docs', async function () {
+          const res = await testCompareDocs(proxy, home);
+          assert.exists(res);
+        });
+      });
+
+      describe('without APP_HOME_INTERNAL_URL', function () {
+        setup('behind-proxy-without-apphomeinternalurl', async () => {
+          ({proxy, home, docs} = await setupServersWithProxy(suitename, {APP_HOME_INTERNAL_URL: ''}));
+        });
+        after(() => tearDown(proxy, [home, docs]));
+
+        it('should succeed to compare docs', async function () {
+          const promise = testCompareDocs(proxy, home);
+          await assert.isRejected(promise, /TestServerReverseProxy: called public URL/);
+        });
+      });
+    });
+
    describe("should work directly with a docworker", async () => {
      setup('docs', async () => {
        const additionalEnvConfiguration = {
@ -233,6 +324,17 @@ describe('DocApi', function () {

 // Contains the tests. This is where you want to add more test.
 function testDocApi() {
+  let chimpy: AxiosRequestConfig, kiwi: AxiosRequestConfig,
+    charon: AxiosRequestConfig, nobody: AxiosRequestConfig, support: AxiosRequestConfig;
+
+  before(function () {
+    chimpy = makeConfig('Chimpy');
+    kiwi = makeConfig('Kiwi');
+    charon = makeConfig('Charon');
+    nobody = makeConfig('Anonymous');
+    support = makeConfig('support');
+  });
+
  async function generateDocAndUrl(docName: string = "Dummy") {
    const wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
    const docId = await userApi.newDoc({name: docName}, wid);
@ -1341,7 +1443,7 @@ function testDocApi() {
    it(`GET /docs/{did}/tables/{tid}/data supports sorts and limits in ${mode}`, async function () {
      function makeQuery(sort: string[] | null, limit: number | null) {
        const url = new URL(`${serverUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`);
-        const config = configForUser('chimpy');
+        const config = makeConfig('chimpy');
        if (mode === 'url') {
          if (sort) {
            url.searchParams.append('sort', sort.join(','));
@ -2615,6 +2717,18 @@ function testDocApi() {
    await worker1.copyDoc(docId, undefined, 'copy');
  });

+  it("POST /docs/{did} with sourceDocId copies a document", async function () {
+    const chimpyWs = await userApi.newWorkspace({name: "Chimpy's Workspace"}, ORG_NAME);
+    const resp = await axios.post(`${serverUrl}/api/docs`, {
+      sourceDocumentId: docIds.TestDoc,
+      documentName: 'copy of TestDoc',
+      asTemplate: false,
+      workspaceId: chimpyWs
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    assert.isString(resp.data);
+  });
+
  it("GET /docs/{did}/download/csv serves CSV-encoded document", async function () {
    const resp = await axios.get(`${serverUrl}/api/docs/${docIds.Timesheets}/download/csv?tableId=Table1`, chimpy);
    assert.equal(resp.status, 200);
@ -2801,7 +2915,7 @@ function testDocApi() {
  });

  it('POST /workspaces/{wid}/import handles empty filenames', async function () {
-    if (!process.env.TEST_REDIS_URL) {
+    if (!process.env.TEST_REDIS_URL || docs.proxiedServer) {
      this.skip();
    }
    const worker1 = await userApi.getWorkerAPI('import');
@ -2809,7 +2923,7 @@ function testDocApi() {
    const fakeData1 = await testUtils.readFixtureDoc('Hello.grist');
    const uploadId1 = await worker1.upload(fakeData1, '.grist');
    const resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
-      configForUser('Chimpy'));
+      makeConfig('Chimpy'));
    assert.equal(resp.status, 200);
    assert.equal(resp.data.title, 'Untitled upload');
    assert.equal(typeof resp.data.id, 'string');
@ -2855,11 +2969,11 @@ function testDocApi() {
  });

  it("document is protected during upload-and-import sequence", async function () {
-    if (!process.env.TEST_REDIS_URL) {
+    if (!process.env.TEST_REDIS_URL || home.proxiedServer) {
      this.skip();
    }
    // Prepare an API for a different user.
-    const kiwiApi = new UserAPIImpl(`${home.serverUrl}/o/Fish`, {
+    const kiwiApi = new UserAPIImpl(`${homeUrl}/o/Fish`, {
      headers: {Authorization: 'Bearer api_key_for_kiwi'},
      fetch: fetch as any,
      newFormData: () => new FormData() as any,
@ -2875,18 +2989,18 @@ function testDocApi() {
    // Check that kiwi only has access to their own upload.
    let wid = (await kiwiApi.getOrgWorkspaces('current')).find((w) => w.name === 'Big')!.id;
    let resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
-      configForUser('Kiwi'));
+      makeConfig('Kiwi'));
    assert.equal(resp.status, 403);
    assert.deepEqual(resp.data, {error: "access denied"});

    resp = await axios.post(`${worker2.url}/api/workspaces/${wid}/import`, {uploadId: uploadId2},
-      configForUser('Kiwi'));
+      makeConfig('Kiwi'));
    assert.equal(resp.status, 200);

    // Check that chimpy has access to their own upload.
    wid = (await userApi.getOrgWorkspaces('current')).find((w) => w.name === 'Private')!.id;
    resp = await axios.post(`${worker1.url}/api/workspaces/${wid}/import`, {uploadId: uploadId1},
-      configForUser('Chimpy'));
+      makeConfig('Chimpy'));
    assert.equal(resp.status, 200);
  });

@ -2963,10 +3077,11 @@ function testDocApi() {
  });

  it('filters urlIds by org', async function () {
+    if (home.proxiedServer) { this.skip(); }
    // Make two documents with same urlId
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1', urlId: 'urlid'}, ws1);
-    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
+    const nasaApi = new UserAPIImpl(`${homeUrl}/o/nasa`, {
      headers: {Authorization: 'Bearer api_key_for_chimpy'},
      fetch: fetch as any,
      newFormData: () => new FormData() as any,
@ -2995,9 +3110,10 @@ function testDocApi() {

  it('allows docId access to any document from merged org', async function () {
    // Make two documents
+    if (home.proxiedServer) { this.skip(); }
    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
    const doc1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
-    const nasaApi = new UserAPIImpl(`${home.serverUrl}/o/nasa`, {
+    const nasaApi = new UserAPIImpl(`${homeUrl}/o/nasa`, {
      headers: {Authorization: 'Bearer api_key_for_chimpy'},
      fetch: fetch as any,
      newFormData: () => new FormData() as any,
@ -3125,11 +3241,17 @@ function testDocApi() {
  });

  it("GET /docs/{did1}/compare/{did2} tracks changes between docs", async function () {
-    const ws1 = (await userApi.getOrgWorkspaces('current'))[0].id;
-    const docId1 = await userApi.newDoc({name: 'testdoc1'}, ws1);
-    const docId2 = await userApi.newDoc({name: 'testdoc2'}, ws1);
-    const doc1 = userApi.getDocAPI(docId1);
-    const doc2 = userApi.getDocAPI(docId2);
+    // Pass kiwi's headers as it contains both Authorization and Origin headers
+    // if run behind a proxy, so we can ensure that the Origin header check is not made.
+    const userApiServerUrl = docs.proxiedServer ? serverUrl : undefined;
+    const chimpyApi = home.makeUserApi(
+      ORG_NAME, 'chimpy', { serverUrl: userApiServerUrl, headers: chimpy.headers as Record<string, string> }
+    );
+    const ws1 = (await chimpyApi.getOrgWorkspaces('current'))[0].id;
+    const docId1 = await chimpyApi.newDoc({name: 'testdoc1'}, ws1);
+    const docId2 = await chimpyApi.newDoc({name: 'testdoc2'}, ws1);
+    const doc1 = chimpyApi.getDocAPI(docId1);
+    const doc2 = chimpyApi.getDocAPI(docId2);

    // Stick some content in column A so it has a defined type
    // so diffs are smaller and simpler.
@ -3327,6 +3449,9 @@ function testDocApi() {
  });

  it('doc worker endpoints ignore any /dw/.../ prefix', async function () {
+    if (docs.proxiedServer) {
+      this.skip();
+    }
    const docWorkerUrl = docs.serverUrl;
    let resp = await axios.get(`${docWorkerUrl}/api/docs/${docIds.Timesheets}/tables/Table1/data`, chimpy);
    assert.equal(resp.status, 200);
@ -4970,18 +5095,26 @@ function testDocApi() {
        }
      });

-      const chimpyConfig = configForUser("Chimpy");
-      const anonConfig = configForUser("Anonymous");
+      const chimpyConfig = makeConfig("Chimpy");
+      const anonConfig = makeConfig("Anonymous");
      delete chimpyConfig.headers!["X-Requested-With"];
      delete anonConfig.headers!["X-Requested-With"];

+      let allowedOrigin;
+
      // Target a more realistic Host than "localhost:port"
-      anonConfig.headers!.Host = chimpyConfig.headers!.Host = 'api.example.com';
+      // (if behind a proxy, we already benefit from a custom and realistic host).
+      if (!home.proxiedServer) {
+        anonConfig.headers!.Host = chimpyConfig.headers!.Host =
+          'api.example.com';
+        allowedOrigin = 'http://front.example.com';
+      } else {
+        allowedOrigin = serverUrl;
+      }

      const url = `${serverUrl}/api/docs/${docId}/tables/Table1/records`;
      const data = { records: [{ fields: {} }] };

-      const allowedOrigin = 'http://front.example.com';
      const forbiddenOrigin = 'http://evil.com';

      // Normal same origin requests
@ -5213,6 +5346,7 @@ function setup(name: string, cb: () => Promise<void>) {
  before(async function () {
    suitename = name;
    dataDir = path.join(tmpDir, `${suitename}-data`);
+    await flushAllRedis();
    await fse.mkdirs(dataDir);
    await setupDataDir(dataDir);
    await cb();
@ -5231,6 +5365,7 @@ function setup(name: string, cb: () => Promise<void>) {
    // stop all servers
    await home.stop();
    await docs.stop();
+    extraHeadersForConfig = {};
  });
 }

@ -5259,3 +5394,12 @@ async function flushAuth() {
  await home.testingHooks.flushAuthorizerCache();
  await docs.testingHooks.flushAuthorizerCache();
 }
+
+async function flushAllRedis() {
+  // Clear redis test database if redis is in use.
+  if (process.env.TEST_REDIS_URL) {
+    const cli = createClient(process.env.TEST_REDIS_URL);
+    await cli.flushdbAsync();
+    await cli.quitAsync();
+  }
+}
--- a/test/server/lib/helpers/TestProxyServer.ts
+++ b/test/server/lib/helpers/TestProxyServer.ts
@ -7,7 +7,6 @@ export class TestProxyServer {
    const server = new TestProxyServer();
    await server._prepare(portNumber);
    return server;
-
  }

  private _proxyCallsCounter: number = 0;
@ -38,7 +37,6 @@ export class TestProxyServer {
        }
        res.sendStatus(responseCode);
        res.end();
-        //next();
      });
    }, portNumber);
  }
--- a/test/server/lib/helpers/TestServer.ts
+++ b/test/server/lib/helpers/TestServer.ts
@ -1,15 +1,20 @@
 import {connectTestingHooks, TestingHooksClient} from "app/server/lib/TestingHooks";
 import {ChildProcess, execFileSync, spawn} from "child_process";
+import * as http from "http";
 import FormData from 'form-data';
 import path from "path";
 import * as fse from "fs-extra";
 import * as testUtils from "test/server/testUtils";
 import {UserAPIImpl} from "app/common/UserAPI";
-import {exitPromise} from "app/server/lib/serverUtils";
+import {exitPromise, getAvailablePort} from "app/server/lib/serverUtils";
 import log from "app/server/lib/log";
 import {delay} from "bluebird";
 import fetch from "node-fetch";
 import {Writable} from "stream";
+import express from "express";
+import { AddressInfo } from "net";
+import { isAffirmative } from "app/common/gutil";
+import httpProxy from 'http-proxy';

 /**
 * This starts a server in a separate process.
@ -24,18 +29,26 @@ export class TestServer {
    options: {output?: Writable} = {},      // Pipe server output to the given stream
  ): Promise<TestServer> {

-    const server = new TestServer(serverTypes, tempDirectory, suitename);
+    const server = new this(serverTypes, tempDirectory, suitename);
    await server.start(_homeUrl, customEnv, options);
    return server;
  }

  public testingSocket: string;
  public testingHooks: TestingHooksClient;
-  public serverUrl: string;
  public stopped = false;
+  public get serverUrl() {
+    if (this._proxiedServer) {
+      throw new Error('Direct access to this test server is disallowed');
+    }
+    return this._serverUrl;
+  }
+  public get proxiedServer() { return this._proxiedServer; }

+  private _serverUrl: string;
  private _server: ChildProcess;
  private _exitPromise: Promise<number | string>;
+  private _proxiedServer: boolean = false;

  private readonly _defaultEnv;

@ -44,9 +57,6 @@ export class TestServer {
      GRIST_INST_DIR: this.rootDir,
      GRIST_DATA_DIR: path.join(this.rootDir, "data"),
      GRIST_SERVERS: this._serverTypes,
-      // with port '0' no need to hard code a port number (we can use testing hooks to find out what
-      // port server is listening on).
-      GRIST_PORT: '0',
      GRIST_DISABLE_S3: 'true',
      REDIS_URL: process.env.TEST_REDIS_URL,
      GRIST_TRIGGER_WAIT_DELAY: '100',
@ -68,9 +78,16 @@ export class TestServer {
      // Unix socket paths typically can't be longer than this. Who knew. Make the error obvious.
      throw new Error(`Path of testingSocket too long: ${this.testingSocket.length} (${this.testingSocket})`);
    }
-    const env = {
-      APP_HOME_URL: _homeUrl,
+
+    const port = await getAvailablePort();
+    this._serverUrl = `http://localhost:${port}`;
+    const homeUrl = _homeUrl ?? (this._serverTypes.includes('home') ? this._serverUrl : undefined);
+
+    const env: NodeJS.ProcessEnv = {
+      APP_HOME_URL: homeUrl,
+      APP_HOME_INTERNAL_URL: homeUrl,
      GRIST_TESTING_SOCKET: this.testingSocket,
+      GRIST_PORT: String(port),
      ...this._defaultEnv,
      ...customEnv
    };
@ -98,7 +115,7 @@ export class TestServer {
      .catch(() => undefined);

    await this._waitServerReady();
-    log.info(`server ${this._serverTypes} up and listening on ${this.serverUrl}`);
+    log.info(`server ${this._serverTypes} up and listening on ${this._serverUrl}`);
  }

  public async stop() {
@ -125,11 +142,9 @@ export class TestServer {

      // create testing hooks and get own port
      this.testingHooks = await connectTestingHooks(this.testingSocket);
-      const port: number = await this.testingHooks.getOwnPort();
-      this.serverUrl = `http://localhost:${port}`;

      // wait for check
-      return (await fetch(`${this.serverUrl}/status/hooks`, {timeout: 1000})).ok;
+      return (await fetch(`${this._serverUrl}/status/hooks`, {timeout: 1000})).ok;
    } catch (err) {
      log.warn("Failed to initialize server", err);
      return false;
@ -142,14 +157,32 @@ export class TestServer {
  // Returns the promise for the ChildProcess's signal or exit code.
  public getExitPromise(): Promise<string|number> { return this._exitPromise; }

-  public makeUserApi(org: string, user: string = 'chimpy'): UserAPIImpl {
-    return new UserAPIImpl(`${this.serverUrl}/o/${org}`, {
-      headers: {Authorization: `Bearer api_key_for_${user}`},
+  public makeUserApi(
+    org: string,
+    user: string = 'chimpy',
+    {
+      headers = {Authorization: `Bearer api_key_for_${user}`},
+      serverUrl = this._serverUrl,
+    }: {
+      headers?: Record<string, string>
+      serverUrl?: string,
+    } = { headers: undefined, serverUrl: undefined },
+  ): UserAPIImpl {
+    return new UserAPIImpl(`${serverUrl}/o/${org}`, {
+      headers,
      fetch: fetch as unknown as typeof globalThis.fetch,
      newFormData: () => new FormData() as any,
    });
  }

+  /**
+   * Assuming that the server is behind a reverse-proxy (like TestServerReverseProxy),
+   * disallow access to the serverUrl to prevent the tests to join the server directly.
+   */
+  public disallowDirectAccess() {
+    this._proxiedServer = true;
+  }
+
  private async _waitServerReady() {
    // It's important to clear the timeout, because it can prevent node from exiting otherwise,
    // which is annoying when running only this test for debugging.
@ -170,3 +203,103 @@ export class TestServer {
    }
  }
 }
+
+/**
+ * Creates a reverse-proxy for a home and a doc worker.
+ *
+ * The workers are then disallowed to be joined directly, the tests are assumed to
+ * pass through this reverse-proxy.
+ *
+ * You may use it like follow:
+ * ```ts
+ * const proxy = new TestServerReverseProxy();
+ * // Create here a doc and a home workers with their env variables
+ * proxy.requireFromOutsideHeader(); // Optional
+ * await proxy.start(home, docs);
+ * ```
+ */
+export class TestServerReverseProxy {
+
+  // Use a different hostname for the proxy than the doc and home workers'
+  // so we can ensure that either we omit the Origin header (so the internal calls to home and doc workers
+  // are not considered as CORS requests), or otherwise we fail because the hostnames are different
+  // https://github.com/gristlabs/grist-core/blob/24b39c651b9590cc360cc91b587d3e1b301a9c63/app/server/lib/requestUtils.ts#L85-L98
+  public static readonly HOSTNAME: string = 'grist-test-proxy.127.0.0.1.nip.io';
+
+  public static FROM_OUTSIDE_HEADER = {"X-FROM-OUTSIDE": true};
+
+  private _app = express();
+  private _proxyServer: http.Server;
+  private _proxy: httpProxy = httpProxy.createProxy();
+  private _address: Promise<AddressInfo>;
+  private _requireFromOutsideHeader = false;
+
+  public get stopped() { return !this._proxyServer.listening; }
+
+  public constructor() {
+    this._proxyServer = this._app.listen(0);
+    this._address = new Promise((resolve) => {
+      this._proxyServer.once('listening', () => {
+        resolve(this._proxyServer.address() as AddressInfo);
+      });
+    });
+  }
+
+  /**
+  * Require the reverse-proxy to be called from the outside world.
+  * This assumes that every requests to the proxy includes the header
+  * provided in TestServerReverseProxy.FROM_OUTSIDE_HEADER
+  *
+  * If a call is done by a worker (assuming they don't include that header),
+  * the proxy rejects with a FORBIDEN http status.
+  */
+  public requireFromOutsideHeader() {
+    this._requireFromOutsideHeader = true;
+  }
+
+  public async start(homeServer: TestServer, docServer: TestServer) {
+    this._app.all(['/dw/dw1', '/dw/dw1/*'], (oreq, ores) => this._getRequestHandlerFor(docServer));
+    this._app.all('/*', this._getRequestHandlerFor(homeServer));
+
+    // Forbid now the use of serverUrl property, so we don't allow the tests to
+    // call the workers directly
+    homeServer.disallowDirectAccess();
+    docServer.disallowDirectAccess();
+
+    log.info('proxy server running on ', await this.getServerUrl());
+  }
+
+  public async getAddress() {
+    return this._address;
+  }
+
+  public async getServerUrl() {
+    const address = await this.getAddress();
+    return `http://${TestServerReverseProxy.HOSTNAME}:${address.port}`;
+  }
+
+  public stop() {
+    if (this.stopped) {
+      return;
+    }
+    log.info("Stopping node TestServerReverseProxy");
+    this._proxyServer.close();
+    this._proxy.close();
+  }
+
+  private _getRequestHandlerFor(server: TestServer) {
+    const serverUrl = new URL(server.serverUrl);
+
+    return (oreq: express.Request, ores: express.Response) => {
+      log.debug(`[proxy] Requesting (method=${oreq.method}): ${new URL(oreq.url, serverUrl).href}`);
+
+      // See the requireFromOutsideHeader() method for the explanation
+      if (this._requireFromOutsideHeader && !isAffirmative(oreq.get("X-FROM-OUTSIDE"))) {
+        log.error('TestServerReverseProxy: called public URL from internal');
+        return ores.status(403).json({error: "TestServerReverseProxy: called public URL from internal "});
+      }
+
+      this._proxy.web(oreq, ores, { target: serverUrl });
+    };
+  }
+}