From e492dfdb22ce9f386e2c0a0c3271b731d4fd155c Mon Sep 17 00:00:00 2001
From: Paul Fitzpatrick <paulfitz@alum.mit.edu>
Date: Wed, 25 Aug 2021 23:12:34 -0400
Subject: [PATCH] (core) add experimental support for python3 in staging

Summary:
This adds `runsc` and `python3` to the grist-server images. For deployments with GRIST_EXPERIMENTAL_PLUGINS=1 (dev + staging but not prod) a hack is added to use `python3` under `runsc` for documents with a special title (`activate-python3-magic` or similar).

This will simplify experiments on behavior of this configuration under realistic conditions.

Hopefully, before landing this, I'll be able to switch to storing a python flag in a document options cell being added by @georgegevoian in a parallel diff, since using the doc title is super hacky :-).

Test Plan: tested manually on worker built locally

Reviewers: dsagal, alexmojaki

Reviewed By: dsagal, alexmojaki

Subscribers: georgegevoian

Differential Revision: https://phab.getgrist.com/D2998
---
 app/server/lib/ActiveDoc.ts | 10 ++++++++++
 app/server/lib/ISandbox.ts  |  2 ++
 app/server/lib/NSandbox.ts  | 21 +++++++++++++++++----
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts
index 8bf74fde..25d91511 100644
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -163,12 +163,22 @@ export class ActiveDoc extends EventEmitter {
     // Our DataEngine is a separate sandboxed process (one per open document). The data engine runs
     // user-defined python code including formula calculations. It maintains all document data and
     // metadata, and applies translates higher-level UserActions into lower-level DocActions.
+
+    // HACK: If doc title as a slug contains "activate-python3-magic", and we are
+    // in an environment with GRIST_EXPERIMENTAL_PLUGINS=1 (dev or staging but not
+    // prod), use Python3.  This is just for experimentation at this point.
+    // TODO: use a less hacky way to say we want to use py3 in a document.
+    const preferredPythonVersion =
+      (options?.docUrl?.match(/activate-python3-magic/) && process.env.GRIST_EXPERIMENTAL_PLUGINS === '1')
+      ? '3' : undefined;
+
     this._dataEngine = this._docManager.gristServer.create.NSandbox({
       comment: docName,
       logCalls: false,
       logTimes: true,
       logMeta: {docId: docName},
       docUrl: options?.docUrl,
+      preferredPythonVersion,
     });
 
     this._activeDocImport = new ActiveDocImport(this);
diff --git a/app/server/lib/ISandbox.ts b/app/server/lib/ISandbox.ts
index ea7420bc..19d75c56 100644
--- a/app/server/lib/ISandbox.ts
+++ b/app/server/lib/ISandbox.ts
@@ -17,6 +17,8 @@ export interface ISandboxCreationOptions {
   importMount?: string;  // if defined, make this path available read-only as "/importdir"
 
   docUrl?: string;      // to support SELF_HYPERLINK.
+
+  preferredPythonVersion?: '2' | '3';
 }
 
 export interface ISandbox {
diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts
index 926ed9f3..6e4d1acc 100644
--- a/app/server/lib/NSandbox.ts
+++ b/app/server/lib/NSandbox.ts
@@ -37,6 +37,8 @@ interface ISandboxOptions {
   command?: string;       // External program or container to call to run the sandbox.
   args: string[];         // The arguments to pass to the python process.
 
+  preferredPythonVersion?: string;  // Mandatory for gvisor; ignored by other methods.
+
   // TODO: update
   // ISandboxCreationOptions to talk about directories instead of
   // mounts, since it may not be possible to remap directories as
@@ -357,14 +359,24 @@ const spawners = {
 export class NSandboxCreator implements ISandboxCreator {
   private _flavor: keyof typeof spawners;
   private _command?: string;
+  private _preferredPythonVersion?: string;
 
-  public constructor(options: {defaultFlavor: keyof typeof spawners}) {
-    const flavor = process.env.GRIST_SANDBOX_FLAVOR || options.defaultFlavor;
+  public constructor(options: {
+    defaultFlavor: keyof typeof spawners,
+    ignoreEnvironment?: boolean,
+    command?: string,
+    preferredPythonVersion?: string,
+  }) {
+    const flavor = (!options.ignoreEnvironment && process.env.GRIST_SANDBOX_FLAVOR) ||
+      options.defaultFlavor;
     if (!Object.keys(spawners).includes(flavor)) {
       throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
     }
     this._flavor = flavor as keyof typeof spawners;
-    this._command = process.env.GRIST_SANDBOX;
+    this._command = (!options.ignoreEnvironment && process.env.GRIST_SANDBOX) ||
+      options.command;
+    this._preferredPythonVersion = (!options.ignoreEnvironment && process.env.PYTHON_VERSION) ||
+      options.preferredPythonVersion;
   }
 
   public create(options: ISandboxCreationOptions): ISandbox {
@@ -386,6 +398,7 @@ export class NSandboxCreator implements ISandboxCreator {
                 ...options.logMeta},
       logTimes: options.logTimes,
       command: this._command,
+      preferredPythonVersion: this._preferredPythonVersion,
       useGristEntrypoint: true,
       importDir: options.importMount,
     };
@@ -521,7 +534,7 @@ function gvisor(options: ISandboxOptions): ChildProcess {
   if (options.deterministicMode) {
     wrapperArgs.push('--faketime', FAKETIME);
   }
-  const pythonVersion = process.env.PYTHON_VERSION;
+  const pythonVersion = options.preferredPythonVersion;
   if (pythonVersion !== '2' && pythonVersion !== '3') {
     throw new Error("PYTHON_VERSION must be set to 2 or 3");
   }