(core) make Grist easier to run with a single server

Summary:
This makes many small changes so that Grist is less fussy to run as a single instance behind a reverse proxy. Some users had difficulty with the self-connections Grist would make, due to internal network setup, and since these are unnecessary in any case in this scenario, they are now optimized away. Likewise some users had difficulties related to doc worker urls, which are now also optimized away. With these changes, users should be able to get a lot further on first try, at least far enough to open and edit documents.

The `GRIST_SINGLE_ORG` setting was proving a bit confusing, since it appeared to only work when set to `docs`. This diff
adds a check for whether the specified org exists, and if not, it creates it. This still depends on having a user email to make as the owner of the team, so there could be remaining difficulties there.

Test Plan: tested manually with nginx

Reviewers: jarek

Reviewed By: jarek

Differential Revision: https://phab.getgrist.com/D3299

app/server/lib/AppEndpoint.ts

@@ -17,6 +17,7 @@ import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap} from 'app/server/lib/expressWrap';
+import {DocTemplate, GristServer} from 'app/server/lib/GristServer';
 import {getAssignmentId} from 'app/server/lib/idUtils';
 import * as log from 'app/server/lib/log';
 import {adaptServerUrl, addOrgToPathIfNeeded, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
@@ -31,6 +32,7 @@ export interface AttachOptions {
   sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
   dbManager: HomeDBManager;
   plugins: LocalPlugin[];
+  gristServer: GristServer;
 }
 
 /**
@@ -61,7 +63,13 @@ export interface AttachOptions {
  * TODO: doc worker registration could be redesigned to remove the assumption
  * of a fixed base domain.
  */
-function customizeDocWorkerUrl(docWorkerUrlSeed: string, req: express.Request) {
+function customizeDocWorkerUrl(docWorkerUrlSeed: string|undefined, req: express.Request): string|null {
+  if (!docWorkerUrlSeed) {
+    // When no doc worker seed, we're in single server mode.
+    // Return null, to signify that the URL prefix serving the
+    // current endpoint is the only one available.
+    return null;
+  }
   const docWorkerUrl = new URL(docWorkerUrlSeed);
   const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org;
   adaptServerUrl(docWorkerUrl, req);
@@ -105,6 +113,14 @@ function customizeDocWorkerUrl(docWorkerUrlSeed: string, req: express.Request) {
  */
 async function getWorker(docWorkerMap: IDocWorkerMap, assignmentId: string,
                          urlPath: string, config: RequestInit = {}) {
+  if (!useWorkerPool()) {
+    // This should never happen. We are careful to not use getWorker
+    // when everything is on a single server, since it is burdensome
+    // for self-hosted users to figure out the correct settings for
+    // the server to be able to contact itself, and there are cases
+    // of the defaults not working.
+    throw new Error("AppEndpoint.getWorker was called unnecessarily");
+  }
   let docStatus: DocStatus|undefined;
   const workersAreManaged = Boolean(process.env.GRIST_MANAGED_WORKERS);
   for (;;) {
@@ -151,13 +167,27 @@ async function getWorker(docWorkerMap: IDocWorkerMap, assignmentId: string,
 }
 
 export function attachAppEndpoint(options: AttachOptions): void {
-  const {app, middleware, docMiddleware, docWorkerMap, forceLogin, sendAppPage, dbManager, plugins} = options;
+  const {app, middleware, docMiddleware, docWorkerMap, forceLogin,
+         sendAppPage, dbManager, plugins, gristServer} = options;
   // Per-workspace URLs open the same old Home page, and it's up to the client to notice and
   // render the right workspace.
   app.get(['/', '/ws/:wsId', '/p/:page'], ...middleware, expressWrap(async (req, res) =>
     sendAppPage(req, res, {path: 'app.html', status: 200, config: {plugins}, googleTagManager: 'anon'})));
 
   app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => {
+    if (!useWorkerPool()) {
+      // Let the client know there is not a separate pool of workers,
+      // so they should continue to use the same base URL for accessing
+      // documents. For consistency, return a prefix to add into that
+      // URL, as there would be for a pool of workers. It would be nice
+      // to go ahead and provide the full URL, but that requires making
+      // more assumptions about how Grist is configured.
+      // Alternatives could be: have the client to send their base URL
+      // in the request; or use headers commonly added by reverse proxies.
+      const selfPrefix =  "/dw/self/v/" + gristServer.getTag();
+      res.json({docWorkerUrl: null, selfPrefix});
+      return;
+    }
     if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); }
     res.header("Access-Control-Allow-Credentials", "true");
 
@@ -237,25 +267,31 @@ export function attachAppEndpoint(options: AttachOptions): void {
       throw err;
     }
 
-    // The reason to pass through app.html fetched from docWorker is in case it is a different
-    // version of Grist (could be newer or older).
-    // TODO: More must be done for correct version tagging of URLs: <base href> assumes all
-    // links and static resources come from the same host, but we'll have Home API, DocWorker,
-    // and static resources all at hostnames different from where this page is served.
-    // TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
+    let body: DocTemplate;
+    let docStatus: DocStatus|undefined;
     const docId = doc.id;
-    const headers = {
-      Accept: 'application/json',
-      ...getTransitiveHeaders(req),
-    };
-    const {docStatus, resp} = await getWorker(docWorkerMap, docId,
-                                              `/${docId}/app.html`, {headers});
-    const body = await resp.json();
+    if (!useWorkerPool()) {
+      body = await gristServer.getDocTemplate();
+    } else {
+      // The reason to pass through app.html fetched from docWorker is in case it is a different
+      // version of Grist (could be newer or older).
+      // TODO: More must be done for correct version tagging of URLs: <base href> assumes all
+      // links and static resources come from the same host, but we'll have Home API, DocWorker,
+      // and static resources all at hostnames different from where this page is served.
+      // TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
+      const headers = {
+        Accept: 'application/json',
+        ...getTransitiveHeaders(req),
+      };
+      const workerInfo = await getWorker(docWorkerMap, docId, `/${docId}/app.html`, {headers});
+      docStatus = workerInfo.docStatus;
+      body = await workerInfo.resp.json();
+    }
 
     await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
                                  googleTagManager: 'anon', config: {
       assignmentId: docId,
-      getWorker: {[docId]: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)},
+      getWorker: {[docId]: customizeDocWorkerUrl(docStatus?.docWorker?.publicUrl, req)},
       getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)},
       plugins
     }});
@@ -266,3 +302,8 @@ export function attachAppEndpoint(options: AttachOptions): void {
   app.get('/:urlId([^/]{12,})/:slug([^/]+):remainder(*)',
           ...docMiddleware, docHandler);
 }
+
+// Return true if document related endpoints are served by separate workers.
+function useWorkerPool() {
+  return process.env.GRIST_SINGLE_PORT !== 'true';
+}

app/server/lib/FlexServer.ts

@@ -34,7 +34,7 @@ import {DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {expressWrap, jsonErrorHandler, secureJsonErrorHandler} from 'app/server/lib/expressWrap';
 import {Hosts, RequestWithOrg} from 'app/server/lib/extractOrg';
 import {addGoogleAuthEndpoint} from "app/server/lib/GoogleAuth";
-import {GristLoginMiddleware, GristServer, RequestWithGrist} from 'app/server/lib/GristServer';
+import {DocTemplate, GristLoginMiddleware, GristServer, RequestWithGrist} from 'app/server/lib/GristServer';
 import {initGristSessions, SessionStore} from 'app/server/lib/gristSessions';
 import {HostedStorageManager} from 'app/server/lib/HostedStorageManager';
 import {IBilling} from 'app/server/lib/IBilling';
@@ -766,7 +766,8 @@ export class FlexServer implements GristServer {
       docWorkerMap: isSingleUserMode() ? null : this._docWorkerMap,
       sendAppPage: this._sendAppPage,
       dbManager: this._dbManager,
-      plugins : (await this._addPluginManager()).getPlugins()
+      plugins : (await this._addPluginManager()).getPlugins(),
+      gristServer: this,
     });
   }
 
@@ -1301,6 +1302,20 @@ export class FlexServer implements GristServer {
     addGoogleAuthEndpoint(this.app, messagePage);
   }
 
+  // Get the HTML template sent for document pages.
+  public async getDocTemplate(): Promise<DocTemplate> {
+    const page = await fse.readFile(path.join(getAppPathTo(this.appRoot, 'static'),
+                                              'app.html'), 'utf8');
+    return {
+      page,
+      tag: this.tag
+    };
+  }
+
+  public getTag(): string {
+    return this.tag;
+  }
+
   // Adds endpoints that support imports and exports.
   private _addSupportPaths(docAccessMiddleware: express.RequestHandler[]) {
     if (!this._docWorker) { throw new Error("need DocWorker"); }
@@ -1552,12 +1567,7 @@ export class FlexServer implements GristServer {
     // TODO: We should be the ones to fill in the base href here to ensure that the browser fetches
     // the correct version of static files for this app.html.
     this.app.get('/:docId/app.html', this._userIdMiddleware, expressWrap(async (req, res) => {
-      const page = await fse.readFile(path.join(getAppPathTo(this.appRoot, 'static'),
-                                                'app.html'), 'utf8');
-      res.json({
-        page,
-        tag: this.tag
-      });
+      res.json(await this.getDocTemplate());
     }));
   }
 

app/server/lib/GristServer.ts

@@ -23,6 +23,7 @@ export interface GristServer {
   getHost(): string;
   getHomeUrl(req: express.Request, relPath?: string): string;
   getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
+  getOwnUrl(): string;
   getDocUrl(docId: string): Promise<string>;
   getOrgUrl(orgKey: string|number): Promise<string>;
   getMergedOrgUrl(req: RequestWithLogin, pathname?: string): string;
@@ -36,6 +37,8 @@ export interface GristServer {
   getHomeDBManager(): HomeDBManager;
   getStorageManager(): IDocStorageManager;
   getNotifier(): INotifier;
+  getDocTemplate(): Promise<DocTemplate>;
+  getTag(): string;
 }
 
 export interface GristLoginSystem {
@@ -55,3 +58,8 @@ export interface GristLoginMiddleware {
 export interface RequestWithGrist extends express.Request {
   gristServer?: GristServer;
 }
+
+export interface DocTemplate {
+  page: string,
+  tag: string,
+}

app/server/lib/MinimalLogin.ts

@@ -1,5 +1,6 @@
 import { UserProfile } from 'app/common/UserAPI';
 import { GristLoginSystem, GristServer } from 'app/server/lib/GristServer';
+import { fromCallback } from 'app/server/lib/serverUtils';
 import { Request } from 'express';
 
 /**
@@ -47,6 +48,12 @@ export async function getMinimalLoginSystem(): Promise<GristLoginSystem> {
  */
 async function setSingleUser(req: Request, gristServer: GristServer) {
   const scopedSession = gristServer.getSessions().getOrCreateSessionFromRequest(req);
+  // Make sure session is up to date before operating on it.
+  // Behavior on a completely fresh session is a little awkward currently.
+  const reqSession = (req as any).session;
+  if (reqSession?.save) {
+    await fromCallback(cb => reqSession.save(cb));
+  }
   await scopedSession.operateOnScopedSession(req, async (user) => Object.assign(user, {
     profile: getDefaultProfile()
   }));

app/server/lib/Sessions.ts

@@ -85,9 +85,9 @@ export class Sessions {
     if (req.headers.cookie) {
       const cookies = cookie.parse(req.headers.cookie);
       const sessionId = this.getSessionIdFromCookie(cookies[cookieName]);
-      return sessionId;
+      if (sessionId) { return sessionId; }
     }
-    return null;
+    return (req as any).sessionID || null;  // sessionID set by express-session
   }
 
   /**

app/server/lib/uploads.ts

@@ -1,6 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {FetchUrlOptions, FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
+import {getDocWorkerUrl} from 'app/common/UserAPI';
 import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {expressWrap} from 'app/server/lib/expressWrap';
@@ -67,7 +68,7 @@ export function addUploadRoute(server: GristServer, expressApp: Application, ...
     if (!docId) { throw new Error('doc must be specified'); }
     const accessId = makeAccessId(req, getAuthorizedUserId(req));
     try {
-      const uploadResult: UploadResult = await fetchDoc(server.getHomeUrl(req), docId, req, accessId,
+      const uploadResult: UploadResult = await fetchDoc(server, docId, req, accessId,
                                                         req.query.template === '1');
       if (name) {
         globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, name);
@@ -398,21 +399,22 @@ async function _fetchURL(url: string, accessId: string|null, options?: FetchUrlO
  * Fetches a Grist doc potentially managed by a different doc worker.  Passes on credentials
  * supplied in the current request.
  */
-async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId: string|null,
+async function fetchDoc(server: GristServer, docId: string, req: Request, accessId: string|null,
                         template: boolean): Promise<UploadResult> {
-
   // Prepare headers that preserve credentials of current user.
   const headers = getTransitiveHeaders(req);
 
   // Find the doc worker responsible for the document we wish to copy.
+  // The backend needs to be well configured for this to work.
+  const homeUrl = server.getHomeUrl(req);
   const fetchUrl = new URL(`/api/worker/${docId}`, homeUrl);
   const response: FetchResponse = await Deps.fetch(fetchUrl.href, {headers});
   await _checkForError(response);
-  const {docWorkerUrl} = await response.json();
-
+  const docWorkerUrl = getDocWorkerUrl(server.getOwnUrl(), await response.json());
   // Download the document, in full or as a template.
-  const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`;
-  return _fetchURL(url, accessId, {headers});
+  const url = new URL(`download?doc=${docId}&template=${Number(template)}`,
+                      docWorkerUrl.replace(/\/*$/, '/'));
+  return _fetchURL(url.href, accessId, {headers});
 }
 
 // Re-issue failures as exceptions.