(core) support python3 in grist-core, and running engine via docker and/or gvisor

Summary:
 * Moves essential plugins to grist-core, so that basic imports (e.g. csv) work.
 * Adds support for a `GRIST_SANDBOX_FLAVOR` flag that can systematically override how the data engine is run.
   - `GRIST_SANDBOX_FLAVOR=pynbox` is "classic" nacl-based sandbox.
   - `GRIST_SANDBOX_FLAVOR=docker` runs engines in individual docker containers. It requires an image specified in `sandbox/docker` (alternative images can be named with `GRIST_SANDBOX` flag - need to contain python and engine requirements). It is a simple reference implementation for sandboxing.
   - `GRIST_SANDBOX_FLAVOR=unsandboxed` runs whatever local version of python is specified by a `GRIST_SANDBOX` flag directly, with no sandboxing. Engine requirements must be installed, so an absolute path to a python executable in a virtualenv is easiest to manage.
   - `GRIST_SANDBOX_FLAVOR=gvisor` runs the data engine via gvisor's runsc. Experimental, with implementation not included in grist-core. Since gvisor runs on Linux only, this flavor supports wrapping the sandboxes in a single shared docker container.
 * Tweaks some recent express query parameter code to work in grist-core, which has a slightly different version of express (smoke test doesn't catch this since in Jenkins core is built within a workspace that has node_modules, and wires get crossed - in a dev environment the problem on master can be seen by doing `buildtools/build_core.sh /tmp/any_path_outside_grist`).

The new sandbox options do not have tests yet, nor does this they change the behavior of grist servers today. They are there to clean up and consolidate a collection of patches I've been using that were getting cumbersome, and make it easier to run experiments.

I haven't looked closely at imports beyond core.

Test Plan: tested manually against regular grist and grist-core, including imports

Reviewers: alexmojaki, dsagal

Reviewed By: alexmojaki

Differential Revision: https://phab.getgrist.com/D2942

app/server/lib/Export.ts

@@ -10,6 +10,7 @@ import {TableData} from 'app/common/TableData';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {docSessionFromRequest} from 'app/server/lib/DocSession';
+import { integerParam, optJsonParam, stringParam} from 'app/server/lib/requestUtils';
 import {ServerColumnGetters} from 'app/server/lib/ServerColumnGetters';
 import * as express from 'express';
 import * as _ from 'underscore';
@@ -68,17 +69,17 @@ export interface ExportParameters {
   tableId: string;
   viewSectionId: number;
   sortOrder: number[];
-  filters: Filter[]
+  filters: Filter[];
 }
 
 /**
  * Gets export parameters from a request.
  */
 export function parseExportParameters(req: express.Request): ExportParameters {
-  const tableId = req.query.tableId;
-  const viewSectionId = parseInt(req.query.viewSection, 10);
-  const sortOrder = gutil.safeJsonParse(req.query.activeSortSpec, null) as number[];
-  const filters: Filter[] = gutil.safeJsonParse(req.query.filters, []) || [];
+  const tableId = stringParam(req.query.tableId);
+  const viewSectionId = integerParam(req.query.viewSection);
+  const sortOrder = optJsonParam(req.query.activeSortSpec, []) as number[];
+  const filters: Filter[] = optJsonParam(req.query.filters, []);
 
   return {
     tableId,

app/server/lib/GoogleAuth.ts

@@ -3,7 +3,7 @@ import {ApiError} from 'app/common/ApiError';
 import {parseSubdomain} from 'app/common/gristUrls';
 import {expressWrap} from 'app/server/lib/expressWrap';
 import * as log from 'app/server/lib/log';
-import {getOriginUrl} from 'app/server/lib/requestUtils';
+import {getOriginUrl, optStringParam, stringParam} from 'app/server/lib/requestUtils';
 import * as express from 'express';
 import {URL} from 'url';
 
@@ -93,13 +93,13 @@ export async function googleAuthTokenMiddleware(
   res: express.Response,
   next: express.NextFunction) {
   // If access token is in place, proceed
-  if (!req.query.code) {
+  if (!optStringParam(req.query.code)) {
     throw new ApiError("Google Auth endpoint requires a code parameter in the query string", 400);
   } else {
     try {
       const oAuth2Client = _googleAuthClient();
       // Decrypt code that was send back from Google Auth service. Uses GOOGLE_CLIENT_SECRET key.
-      const tokenResponse = await oAuth2Client.getToken(req.query.code);
+      const tokenResponse = await oAuth2Client.getToken(stringParam(req.query.code));
       // Get the access token (access token will be present in a default request configuration).
       const access_token = tokenResponse.tokens.access_token!;
       req.query.access_token = access_token;
@@ -122,7 +122,7 @@ export function addGoogleAuthEndpoint(
   messagePage: (req: express.Request, res: express.Response, message: any) => any
 ) {
   if (!process.env.GOOGLE_CLIENT_SECRET) {
-    log.error("Failed to create GoogleAuth endpoint: GOOGLE_CLIENT_SECRET is not defined");
+    log.warn("Failed to create GoogleAuth endpoint: GOOGLE_CLIENT_SECRET is not defined");
     expressApp.get(authHandlerPath, expressWrap(async (req: express.Request, res: express.Response) => {
       throw new Error("Send to Google Drive is not configured.");
     }));
@@ -136,20 +136,22 @@ export function addGoogleAuthEndpoint(
     // our request. It is encrypted (with CLIENT_SECRET) and signed with redirect url.
     // In state query parameter we will receive an url that was send as part of the request to Google.
 
-    if (req.query.code) {
+    if (optStringParam(req.query.code)) {
       log.debug("GoogleAuth - response from Google with valid code");
-      messagePage(req, res, { code: req.query.code, origin: req.query.state });
-    } else if (req.query.error) {
-      log.debug("GoogleAuth - response from Google with error code", req.query.error);
-      if (req.query.error === "access_denied") {
-        messagePage(req, res, { error: req.query.error, origin: req.query.state });
+      messagePage(req, res, { code: stringParam(req.query.code),
+                              origin: stringParam(req.query.state) });
+    } else if (optStringParam(req.query.error)) {
+      log.debug("GoogleAuth - response from Google with error code", stringParam(req.query.error));
+      if (stringParam(req.query.error) === "access_denied") {
+        messagePage(req, res, { error: stringParam(req.query.error),
+                                origin: stringParam(req.query.state) });
       } else {
         // This should not happen, either code or error is a mandatory query parameter.
         throw new ApiError("Error authenticating with Google", 500);
       }
     } else {
       const oAuth2Client = _googleAuthClient();
-      const scope = req.query.scope || DRIVE_SCOPE;
+      const scope = optStringParam(req.query.scope) || DRIVE_SCOPE;
       // Create url for origin parameter for a popup window.
       const origin = getOriginUrl(req);
       const authUrl = oAuth2Client.generateAuthUrl({

app/server/lib/GoogleExport.ts

@@ -3,6 +3,7 @@ import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {makeXLSX} from 'app/server/lib/ExportXLSX';
 import * as log from 'app/server/lib/log';
+import {optStringParam} from 'app/server/lib/requestUtils';
 import {Request, Response} from 'express';
 import {PassThrough} from 'stream';
 
@@ -16,7 +17,7 @@ export async function exportToDrive(
   res: Response
 ) {
   // Token should come from auth middleware
-  const access_token = req.query.access_token;
+  const access_token = optStringParam(req.query.access_token);
   if (!access_token) {
     throw new Error("No access token - Can't send file to Google Drive");
   }
@@ -78,6 +79,6 @@ async function sendFileToDrive(fileNameNoExt: string, data: ArrayBuffer, oauth_t
 // Makes excel file the same way as export to excel works.
 async function prepareFile(doc: ActiveDoc, req: Request) {
   const data = await makeXLSX(doc, req);
-  const name = (req.query.title || doc.docName);
+  const name = (optStringParam(req.query.title) || doc.docName);
   return { name, data };
 }

app/server/lib/NSandbox.ts

@@ -12,21 +12,59 @@ import {ChildProcess, spawn} from 'child_process';
 import * as path from 'path';
 import {Stream, Writable} from 'stream';
 import * as _ from 'lodash';
-import * as fs from "fs";
+import * as fs from 'fs';
+import * as which from 'which';
 
 type SandboxMethod = (...args: any[]) => any;
 
-export interface ISandboxOptions {
+/**
+ *
+ * A collection of options for weird and wonderful ways to run Grist.
+ * The sandbox at heart is just python, but run in different ways
+ * (sandbox 'flavors': pynbox, docker, gvisor, and unsandboxed).
+ *
+ * The "command" is an external program/container to call to run the
+ * sandbox, and it depends on sandbox flavor. Pynbox is built into
+ * Grist and has a hard-wired command, so the command option should be
+ * empty.  For gvisor and unsandboxed, command is the path to an
+ * external program to run.  For docker, it is the name of an image.
+ *
+ * Once python is running, ordinarily some Grist code should be
+ * started by setting `useGristEntrypoint` (the only exception is
+ * in tests).
+ *
+ * The Grist code that runs is by default grist/main.py.  For plugins,
+ * this is overridden, to run whatever is specified by plugin.script.
+ *
+ */
+interface ISandboxOptions {
+  command?: string;       // External program or container to call to run the sandbox.
   args: string[];         // The arguments to pass to the python process.
+
+  // When doing imports, the sandbox is started somewhat differently.
+  // Directories are shared with the sandbox that are not otherwise.
+  // Options for that that are collected in `plugin`.  TODO: update
+  // ISandboxCreationOptions to talk about directories instead of
+  // mounts, since it may not be possible to remap directories as
+  // mounts (e.g. for unsandboxed operation).
+  plugin?: {
+    importDir: string;  // a directory containing data file(s) to import.
+    pluginDir: string;  // a directory containing code for running the import.
+    script: string;     // an entrypoint, relative to pluginDir.
+  }
+
+  docUrl?: string;               // URL to the document, for SELF_HYPERLINK
+  minimalPipeMode?: boolean;     // Whether to use newer 3-pipe operation
+  deterministicMode?: boolean;   // Whether to override time + randomness
+
   exports?: {[name: string]: SandboxMethod}; // Functions made available to the sandboxed process.
   logCalls?: boolean;     // (Not implemented) Whether to log all system calls from the python sandbox.
   logTimes?: boolean;     // Whether to log time taken by calls to python sandbox.
-  unsilenceLog?: boolean; // Don't silence the sel_ldr logging.
-  selLdrArgs?: string[];  // Arguments passed to selLdr, for instance the following sets an
-                          // environment variable `{ ... selLdrArgs: ['-E', 'PYTHONPATH=grist'] ... }`.
+  unsilenceLog?: boolean; // Don't silence the sel_ldr logging (pynbox only).
   logMeta?: log.ILogMeta; // Log metadata (e.g. including docId) to report in all log messages.
-  command?: string;
-  env?: NodeJS.ProcessEnv;
+
+  useGristEntrypoint?: boolean;  // Should be set for everything except tests, which
+                                 // may want to pass arguments to python directly.
 }
 
 type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
@@ -39,62 +77,6 @@ type MsgCode = null | true | false;
 const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR;
 
 export class NSandbox implements ISandbox {
-  /**
-   * Helper function to run the nacl sandbox. It takes care of most arguments, similarly to
-   * nacl/bin/run script, but without the reliance on bash. We can't use bash when -r/-w options
-   * because on Windows it doesn't pass along the open file descriptors. Bash is also unavailable
-   * when installing a standalone version on Windows.
-   */
-  public static spawn(options: ISandboxOptions): ChildProcess {
-    const {command, args: pythonArgs, unsilenceLog, env} = options;
-    const spawnOptions = {
-      stdio: ['pipe', 'pipe', 'pipe'] as 'pipe'[],
-      env
-    };
-    const selLdrArgs = [];
-    if (!NSandbox._useMinimalPipes(env)) {
-      // add two more pipes
-      spawnOptions.stdio.push('pipe', 'pipe');
-      // We use these options to set up communication with the sandbox:
-      // -r 3:3  to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the
-      //         inside, for reading from the inside. This becomes `this._streamToSandbox`.
-      // -w 4:4  to associate FD 4 on the outside with FD 4 on the inside for writing from the inside.
-      //         This becomes `this._streamFromSandbox`
-      selLdrArgs.push('-r', '3:3', '-w', '4:4');
-    }
-    if (options.selLdrArgs) { selLdrArgs.push(...options.selLdrArgs); }
-
-    if (command) {
-      return spawn(command, pythonArgs,
-                   {cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
-    }
-
-    const noLog = unsilenceLog ? [] :
-      (process.env.OS === 'Windows_NT' ? ['-l', 'NUL'] : ['-l', '/dev/null']);
-    for (const [key, value] of _.toPairs(env)) {
-      selLdrArgs.push("-E");
-      selLdrArgs.push(`${key}=${value}`);
-    }
-    return spawn('sandbox/nacl/bin/sel_ldr', [
-        '-B', './sandbox/nacl/lib/irt_core.nexe', '-m', './sandbox/nacl/root:/:ro',
-        ...noLog,
-        ...selLdrArgs,
-        './sandbox/nacl/lib/runnable-ld.so',
-        '--library-path', '/slib', '/python/bin/python2.7.nexe',
-        ...pythonArgs
-      ],
-      spawnOptions,
-    );
-  }
-
-  // Check if environment is configured for minimal pipes.
-  private static _useMinimalPipes(env: NodeJS.ProcessEnv | undefined) {
-    if (!env?.PIPE_MODE) { return false; }
-    if (env.PIPE_MODE !== 'minimal') {
-      throw new Error(`unrecognized pipe mode: ${env.PIPE_MODE}`);
-    }
-    return true;
-  }
 
   public readonly childProc: ChildProcess;
   private _logTimes: boolean;
@@ -119,16 +101,25 @@ export class NSandbox implements ISandbox {
   /*
    * Callers may listen to events from sandbox.childProc (a ChildProcess), e.g. 'close' and 'error'.
    * The sandbox listens for 'aboutToExit' event on the process, to properly shut down.
+   *
+   * Grist interacts with the sandbox via message passing through pipes to an isolated
+   * process.  Some read-only shared code is made available to the sandbox.
+   * For plugins, read-only data files are made available.
+   *
+   * At the time of writing, Grist has been using an NaCl sandbox with python2.7 compiled
+   * for it for several years (pynbox), and we are now experimenting with other sandboxing
+   * options.  Variants can be activated by passing in a non-default "spawner" function.
+   *
    */
-  constructor(options: ISandboxOptions) {
+  constructor(options: ISandboxOptions, spawner: SpawnFn = pynbox) {
     this._logTimes = Boolean(options.logTimes || options.logCalls);
     this._exportedFunctions = options.exports || {};
 
-    this.childProc = NSandbox.spawn(options);
+    this.childProc = spawner(options);
 
     this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
 
-    if (NSandbox._useMinimalPipes(options.env)) {
+    if (options.minimalPipeMode) {
       log.rawDebug("3-pipe Sandbox started", this._logMeta);
       this._streamToSandbox = this.childProc.stdin;
       this._streamFromSandbox = this.childProc.stdout;
@@ -343,67 +334,365 @@ export class NSandbox implements ISandbox {
   }
 }
 
+/**
+ * Functions for spawning all of the currently supported sandboxes.
+ */
+const spawners = {
+  pynbox,             // Grist's "classic" sandbox - python2 within NaCl.
+  unsandboxed,        // No sandboxing, straight to host python.
+                      // This offers no protection to the host.
+  docker,             // Run sandboxes in distinct docker containers.
+  gvisor,             // Gvisor's runsc sandbox.
+};
+
+/**
+ * A sandbox factory.  This doesn't do very much beyond remembering a default
+ * flavor of sandbox (which at the time of writing differs between hosted grist and
+ * grist-core), and trying to regularize creation options a bit.
+ *
+ * The flavor of sandbox to use can be overridden by two environment variables:
+ *   - GRIST_SANDBOX_FLAVOR: should be one of the spawners (pynbox, unsandboxed, docker,
+ *     gvisor)
+ *   - GRIST_SANDBOX: a program or image name to run as the sandbox.  Not needed for
+ *     pynbox (it is either built in or not avaiable).  For unsandboxed, should be an
+ *     absolute path to python within a virtualenv with all requirements installed.
+ *     For docker, it should be `grist-docker-sandbox` (an image built via makefile
+ *     in `sandbox/docker`) or a derived image.  For gvisor, it should be the full path
+ *     to `sandbox/gvisor/run.py` (if runsc available locally) or to
+ *     `sandbox/gvisor/wrap_in_docker.sh` (if runsc should be run using the docker
+ *     image built in that directory).  Gvisor is not yet available in grist-core.
+ */
 export class NSandboxCreator implements ISandboxCreator {
-  public constructor(private _flavor: 'pynbox' | 'unsandboxed') {
+  private _flavor: keyof typeof spawners;
+  private _command?: string;
+
+  public constructor(options: {defaultFlavor: keyof typeof spawners}) {
+    const flavor = process.env.GRIST_SANDBOX_FLAVOR || options.defaultFlavor;
+    if (!Object.keys(spawners).includes(flavor)) {
+      throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
+    }
+    this._flavor = flavor as keyof typeof spawners;
+    this._command = process.env.GRIST_SANDBOX;
   }
 
   public create(options: ISandboxCreationOptions): ISandbox {
-    const pynbox = this._flavor === 'pynbox';
-    // Main script to run.
-    const defaultEntryPoint = pynbox ? 'grist/main.pyc' : 'grist/main.py';
-    const args = [options.entryPoint || defaultEntryPoint];
+    const args: string[] = [];
     if (!options.entryPoint && options.comment) {
       // When using default entry point, we can add on a comment as an argument - it isn't
       // used, but will show up in `ps` output for the sandbox process.  Comment is intended
       // to be a document name/id.
       args.push(options.comment);
     }
-    const selLdrArgs: string[] = [];
-    if (options.sandboxMount) {
-      selLdrArgs.push(
-        // TODO: Only modules that we share with plugins should be mounted. They could be gathered in
-        // a "$APPROOT/sandbox/plugin" folder, only which get mounted.
-        '-m', `${options.sandboxMount}:/sandbox:ro`);
-    }
-    if (options.importMount) {
-      selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`);
-    }
-    const pythonVersion = 'python2.7';
-    const env: NodeJS.ProcessEnv = {
-      // Python library path is only configurable when flavor is unsandboxed.
-      // In this case, expect to find library files in a virtualenv built by core
-      // buildtools/prepare_python.sh
-      PYTHONPATH: pynbox ? 'grist:thirdparty' :
-        path.join(process.cwd(), 'sandbox', 'grist') + ':' +
-        path.join(process.cwd(), 'venv', 'lib', pythonVersion, 'site-packages'),
-
-      DOC_URL: (options.docUrl || '').replace(/[^-a-zA-Z0-9_:/?&.]/, ''),
-
-      // use stdin/stdout/stderr only.
-      PIPE_MODE: 'minimal',
-
-      // Making time and randomness act deterministically for testing purposes.
-      // See test/utils/recordPyCalls.ts
-      ...(process.env.LIBFAKETIME_PATH ? {  // path to compiled binary
-        DETERMINISTIC_MODE: '1',  // tells python to seed the random module
-        FAKETIME: "2020-01-01 00:00:00",  // setting for libfaketime
-
-        // For Linux
-        LD_PRELOAD: process.env.LIBFAKETIME_PATH,
-
-        // For Mac (https://github.com/wolfcw/libfaketime/blob/master/README.OSX)
-        DYLD_INSERT_LIBRARIES: process.env.LIBFAKETIME_PATH,
-        DYLD_FORCE_FLAT_NAMESPACE: '1',
-      } : {}),
-    };
-    return new NSandbox({
+    const translatedOptions: ISandboxOptions = {
+      minimalPipeMode: true,
+      deterministicMode: Boolean(process.env.LIBFAKETIME_PATH),
+      docUrl: options.docUrl,
       args,
       logCalls: options.logCalls,
-      logMeta: options.logMeta,
+      logMeta: {flavor: this._flavor, command: this._command,
+                entryPoint: options.entryPoint || '(default)',
+                ...options.logMeta},
       logTimes: options.logTimes,
-      selLdrArgs,
-      env,
-      ...(pynbox ? {} : {command: pythonVersion}),
-    });
+      command: this._command,
+      useGristEntrypoint: true,
+    };
+    if (options.entryPoint) {
+      translatedOptions.plugin = {
+        script: options.entryPoint,
+        pluginDir: options.sandboxMount || '',
+        importDir: options.importMount || '',
+      };
+    }
+    return new NSandbox(translatedOptions, spawners[this._flavor]);
   }
 }
+
+// A function that takes sandbox options and starts a sandbox process.
+type SpawnFn = (options: ISandboxOptions) => ChildProcess;
+
+/**
+ * Helper function to run a nacl sandbox. It takes care of most arguments, similarly to
+ * nacl/bin/run script, but without the reliance on bash. We can't use bash when -r/-w options
+ * because on Windows it doesn't pass along the open file descriptors. Bash is also unavailable
+ * when installing a standalone version on Windows.
+ *
+ * This is quite old code, with attention to Windows support that is no longer tested.
+ * I've done my best to avoid changing behavior by not touching it too much.
+ */
+function pynbox(options: ISandboxOptions): ChildProcess {
+  const {command, args: pythonArgs, unsilenceLog, plugin} = options;
+  if (command) {
+    throw new Error("NaCl can only run the specific python2.7 package built for it");
+  }
+  if (options.useGristEntrypoint) {
+    pythonArgs.unshift(plugin?.script || 'grist/main.pyc');
+  }
+  const spawnOptions = {
+    stdio: ['pipe', 'pipe', 'pipe'] as 'pipe'[],
+    env: getWrappingEnv(options)
+  };
+  const wrapperArgs = new FlagBag({env: '-E', mount: '-m'});
+  if (plugin) {
+
+    // TODO: Only modules that we share with plugins should be mounted. They could be gathered in
+    // a "$APPROOT/sandbox/plugin" folder, only which get mounted.
+    wrapperArgs.addMount(`${plugin.pluginDir}:/sandbox:ro`);
+    wrapperArgs.addMount(`${plugin.importDir}:/importdir:ro`);
+  }
+
+  if (!options.minimalPipeMode) {
+    // add two more pipes
+    spawnOptions.stdio.push('pipe', 'pipe');
+    // We use these options to set up communication with the sandbox:
+    // -r 3:3  to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the
+    //         inside, for reading from the inside. This becomes `this._streamToSandbox`.
+    // -w 4:4  to associate FD 4 on the outside with FD 4 on the inside for writing from the inside.
+    //         This becomes `this._streamFromSandbox`
+    wrapperArgs.push('-r', '3:3', '-w', '4:4');
+  }
+  wrapperArgs.addAllEnv(getInsertedEnv(options));
+  wrapperArgs.addEnv('PYTHONPATH', 'grist:thirdparty');
+
+  const noLog = unsilenceLog ? [] :
+    (process.env.OS === 'Windows_NT' ? ['-l', 'NUL'] : ['-l', '/dev/null']);
+  return spawn('sandbox/nacl/bin/sel_ldr', [
+    '-B', './sandbox/nacl/lib/irt_core.nexe', '-m', './sandbox/nacl/root:/:ro',
+    ...noLog,
+    ...wrapperArgs.get(),
+    './sandbox/nacl/lib/runnable-ld.so',
+    '--library-path', '/slib', '/python/bin/python2.7.nexe',
+    ...pythonArgs
+  ], spawnOptions);
+}
+
+/**
+ * Helper function to run python without sandboxing.  GRIST_SANDBOX should have
+ * been set with an absolute path to a version of python within a virtualenv that
+ * has all the dependencies installed (e.g. the sandbox_venv3 virtualenv created
+ * by `./build python3`.  Using system python works too, if all dependencies have
+ * been installed globally.
+ */
+function unsandboxed(options: ISandboxOptions): ChildProcess {
+  const {args: pythonArgs, plugin} = options;
+  const paths = getAbsolutePaths(options);
+  if (options.useGristEntrypoint) {
+    pythonArgs.unshift(paths.plugin?.script || paths.main);
+  }
+  const spawnOptions = {
+    stdio: ['pipe', 'pipe', 'pipe'] as 'pipe'[],
+    env: {
+      PYTHONPATH: paths.engine,
+      IMPORTDIR: plugin?.importDir,
+      ...getInsertedEnv(options),
+      ...getWrappingEnv(options),
+    }
+  };
+  if (!options.minimalPipeMode) {
+    spawnOptions.stdio.push('pipe', 'pipe');
+  }
+  let command = options.command;
+  if (!command) {
+    // No command specified.  In this case, grist-core looks for a "venv"
+    // virtualenv; a python3 virtualenv would be in "sandbox_venv3".
+    // TODO: rationalize this, it is a product of haphazard growth.
+    for (const venv of ['sandbox_venv3', 'venv']) {
+      const pythonPath = path.join(process.cwd(), venv, 'bin', 'python');
+      if (fs.existsSync(pythonPath)) {
+        command = pythonPath;
+        break;
+      }
+    }
+    // Fall back on system python.
+    if (!command) {
+      command = which.sync('python');
+    }
+  }
+  return spawn(command, pythonArgs,
+               {cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
+}
+
+/**
+ * Helper function to run python in gvisor's runsc, with multiple
+ * sandboxes run within the same container.  GRIST_SANDBOX should
+ * point to `sandbox/gvisor/run.py` (to map call onto gvisor's runsc
+ * directly) or `wrap_in_docker.sh` (to use runsc within a container).
+ * Be sure to read setup instructions in that directory.
+ */
+function gvisor(options: ISandboxOptions): ChildProcess {
+  const {command, args: pythonArgs} = options;
+  if (!command) { throw new Error("gvisor operation requires GRIST_SANDBOX"); }
+  if (!options.minimalPipeMode) {
+    throw new Error("gvisor only supports 3-pipe operation");
+  }
+  const paths = getAbsolutePaths(options);
+  const wrapperArgs = new FlagBag({env: '-E', mount: '-m'});
+  wrapperArgs.addEnv('PYTHONPATH', paths.engine);
+  wrapperArgs.addAllEnv(getInsertedEnv(options));
+  wrapperArgs.addMount(paths.sandboxDir);
+  if (paths.plugin) {
+    wrapperArgs.addMount(paths.plugin.pluginDir);
+    wrapperArgs.addMount(paths.plugin.importDir);
+    wrapperArgs.addEnv('IMPORTDIR', paths.plugin.importDir);
+    pythonArgs.unshift(paths.plugin.script);
+  } else if (options.useGristEntrypoint) {
+    pythonArgs.unshift(paths.main);
+  }
+  if (options.deterministicMode) {
+    wrapperArgs.push('--faketime', FAKETIME);
+  }
+  return spawn(command, [...wrapperArgs.get(), 'python', '--', ...pythonArgs]);
+}
+
+/**
+ * Helper function to run python in a container. Each sandbox run in a
+ * distinct container.  GRIST_SANDBOX should be the name of an image where
+ * `python` can be run and all Grist dependencies are installed.  See
+ * `sandbox/docker` for more.
+ */
+function docker(options: ISandboxOptions): ChildProcess {
+  const {args: pythonArgs, command} = options;
+  if (options.useGristEntrypoint) {
+    pythonArgs.unshift(options.plugin?.script || 'grist/main.py');
+  }
+  if (!options.minimalPipeMode) {
+    throw new Error("docker only supports 3-pipe operation (although runc has --preserve-file-descriptors)");
+  }
+  const paths = getAbsolutePaths(options);
+  const plugin = paths.plugin;
+  const wrapperArgs = new FlagBag({env: '--env', mount: '-v'});
+  if (plugin) {
+    wrapperArgs.addMount(`${plugin.pluginDir}:/sandbox:ro`);
+    wrapperArgs.addMount(`${plugin.importDir}:/importdir:ro`);
+  }
+  wrapperArgs.addMount(`${paths.engine}:/grist:ro`);
+  wrapperArgs.addAllEnv(getInsertedEnv(options));
+  wrapperArgs.addEnv('PYTHONPATH', 'grist:thirdparty');
+  const commandParts: string[] = ['python'];
+  if (options.deterministicMode) {
+    // DETERMINISTIC_MODE is already set by getInsertedEnv().  We also take
+    // responsibility here for running faketime around python.
+    commandParts.unshift('faketime', '-f', FAKETIME);
+  }
+  const dockerPath = which.sync('docker');
+  return spawn(dockerPath, [
+    'run', '--rm', '-i', '--network', 'none',
+    ...wrapperArgs.get(),
+    command || 'grist-docker-sandbox',  // this is the docker image to use
+    ...commandParts,
+    ...pythonArgs,
+  ]);
+}
+
+/**
+ * Collect environment variables that should end up set within the sandbox.
+ */
+function getInsertedEnv(options: ISandboxOptions) {
+  const env: NodeJS.ProcessEnv = {
+    DOC_URL: (options.docUrl || '').replace(/[^-a-zA-Z0-9_:/?&.]/, ''),
+
+    // use stdin/stdout/stderr only.
+    PIPE_MODE: options.minimalPipeMode ? 'minimal' : 'classic',
+  };
+
+  if (options.deterministicMode) {
+    // Making time and randomness act deterministically for testing purposes.
+    // See test/utils/recordPyCalls.ts
+    // tells python to seed the random module
+    env.DETERMINISTIC_MODE = '1';
+  }
+  return env;
+}
+
+/**
+ * Collect environment variables to activate faketime if needed.  The paths
+ * here only make sense for unsandboxed operation, or for pynbox.  For gvisor,
+ * faketime doesn't work, and must be done inside the sandbox.  For docker,
+ * likewise wrapping doesn't make sense.  In those cases, LIBFAKETIME_PATH can
+ * just be set to ON to activate faketime in a sandbox dependent manner.
+ */
+function getWrappingEnv(options: ISandboxOptions) {
+  const env: NodeJS.ProcessEnv = options.deterministicMode ? {
+    // Making time and randomness act deterministically for testing purposes.
+    // See test/utils/recordPyCalls.ts
+    FAKETIME,  // setting for libfaketime
+    // For Linux
+    LD_PRELOAD: process.env.LIBFAKETIME_PATH,
+
+    // For Mac (https://github.com/wolfcw/libfaketime/blob/master/README.OSX)
+    DYLD_INSERT_LIBRARIES: process.env.LIBFAKETIME_PATH,
+    DYLD_FORCE_FLAT_NAMESPACE: '1',
+  } : {};
+  return env;
+}
+
+/**
+ * Extract absolute paths from options.  By sticking with the directory
+ * structure on the host rather than remapping, we can simplify nesting
+ * wrappers, or cases where remapping isn't possible.  It does leak the names
+ * of the host directories though, and there could be silly complications if the
+ * directories have spaces or other idiosyncracies.  When committing to a sandbox
+ * technology, for stand-alone Grist, it would be worth rethinking this.
+ */
+function getAbsolutePaths(options: ISandboxOptions) {
+  // Get path to sandbox directory - this is a little idiosyncratic to work well
+  // in grist-core.  It is important to use real paths since we may be viewing
+  // the file system through a narrow window in a container.
+  const sandboxDir = path.join(fs.realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
+                               '..');
+  // Copy plugin options, and then make them absolute.
+  const plugin = options.plugin && { ...options.plugin };
+  if (plugin) {
+    plugin.pluginDir = fs.realpathSync(plugin.pluginDir);
+    plugin.importDir = fs.realpathSync(plugin.importDir);
+    // Plugin dir is ..../sandbox, and entry point is sandbox/...
+    // This may not be a general rule, it may be just for the "core" plugin, but
+    // that suffices for now.
+    plugin.script = path.join(plugin.pluginDir, '..', plugin.script);
+  }
+  return {
+    sandboxDir,
+    plugin,
+    main: path.join(sandboxDir, 'grist/main.py'),
+    engine: path.join(sandboxDir, 'grist'),
+  };
+}
+
+/**
+ * A tiny abstraction to make code setting up command line arguments a bit
+ * easier to read.  The sandboxes are quite similar in spirit, but differ
+ * a bit in exact flags used.
+ */
+class FlagBag {
+  private _args: string[] = [];
+
+  constructor(private _options: {env: '--env'|'-E', mount: '-m'|'-v'}) {
+  }
+
+  // channel env variables for sandbox via -E / --env
+  public addEnv(key: string, value: string|undefined) {
+    this._args.push(this._options.env, key + '=' + (value || ''));
+  }
+
+  // Channel all of the supplied env variables
+  public addAllEnv(env: NodeJS.ProcessEnv) {
+    for (const [key, value] of _.toPairs(env)) {
+      this.addEnv(key, value);
+    }
+  }
+
+  // channel shared directory for sandbox via -m / -v
+  public addMount(share: string) {
+    this._args.push(this._options.mount, share);
+  }
+
+  // add some ad-hoc arguments
+  public push(...args: string[]) {
+    this._args.push(...args);
+  }
+
+  // get the final list of arguments
+  public get() { return this._args; }
+}
+
+// Standard time to default to if faking time.
+const FAKETIME = '2020-01-01 00:00:00';

app/server/lib/requestUtils.ts

@@ -233,6 +233,10 @@ export function optIntegerParam(p: any): number|undefined {
   return undefined;
 }
 
+export function optJsonParam(p: any, defaultValue: any): any {
+  if (typeof p !== 'string') { return defaultValue; }
+  return gutil.safeJsonParse(p, defaultValue);
+}
 
 export interface RequestWithGristInfo extends Request {
   gristInfo?: string;

app/server/serverMethods.ts

@@ -3,6 +3,7 @@ import {parseExportFileName, parseExportParameters} from 'app/server/lib/Export'
 import {makeCSV} from 'app/server/lib/ExportCSV';
 import {makeXLSX} from 'app/server/lib/ExportXLSX';
 import * as log from 'app/server/lib/log';
+import {integerParam, stringParam} from 'app/server/lib/requestUtils';
 import * as contentDisposition from 'content-disposition';
 import * as express from 'express';
 
@@ -14,8 +15,8 @@ export async function generateCSV(req: express.Request, res: express.Response, c
     sortOrder
   } = parseExportParameters(req);
 
-  const clientId = req.query.clientId;
-  const docFD = parseInt(req.query.docFD, 10);
+  const clientId = stringParam(req.query.clientId);
+  const docFD = integerParam(req.query.docFD);
   const client = comm.getClient(clientId);
   const docSession = client.getDocSession(docFD);
   const activeDoc = docSession.activeDoc;
@@ -41,8 +42,8 @@ export async function generateCSV(req: express.Request, res: express.Response, c
 
 export async function generateXLSX(req: express.Request, res: express.Response, comm: Comm) {
   log.debug(`Generating .xlsx file`);
-  const clientId = req.query.clientId;
-  const docFD = parseInt(req.query.docFD, 10);
+  const clientId = stringParam(req.query.clientId);
+  const docFD = integerParam(req.query.docFD);
   const client = comm.getClient(clientId);
   const docSession = client.getDocSession(docFD);
   const activeDoc = docSession.activeDoc;