(core) add machinery for self-managed flavor of Grist

Summary:
Currently, we have two ways that we deliver Grist. One is grist-core,
which has simple defaults and is relatively easy for third parties to
deploy. The second is our internal build for our SaaS, which is the
opposite. For self-managed Grist, a planned paid on-premise version
of Grist, I adopt the following approach:

 * Use the `grist-core` build mechanism, extending it to accept an
   overlay of extra code if present.
 * Extra code is supplied in a self-contained `ext` directory, with
   an `ext/app` directory that is of same structure as core `app`
   and `stubs/app`.
 * The `ext` directory also contains information about extra
   node dependencies needed beyond that of `grist-core`.
 * The `ext` directory is contained within our monorepo rather than
   `grist-core` since it may contain material not under the Apache
   license.

Docker builds are achieved in our monorepo by using the `--build-context`
functionality to add in `ext` during the regular `grist-core` build:

```
docker buildx build --load -t gristlabs/grist-ee --build-context=ext=../ext .
```

Incremental builds in our monorepo are achieved with the `build_core.sh` helper,
like:

```
buildtools/build_core.sh /tmp/self-managed
cd /tmp/self-managed
yarn start
```

The initial `ext` directory contains material for snapshotting to S3.
If you build the docker image as above, and have S3 access, you can
do something like:

```
docker run -p 8484:8484 --env GRIST_SESSION_SECRET=a-secret \
  --env GRIST_DOCS_S3_BUCKET=grist-docs-test \
  --env GRIST_DOCS_S3_PREFIX=self-managed \
  -v $HOME/.aws:/root/.aws -it gristlabs/grist-ee
```

This will start a version of Grist that is like `grist-core` but with
S3 snapshots enabled. To release this code to `grist-core`, it would
just need to move from `ext/app` to `app` within core.

I tried a lot of ways of organizing self-managed Grist, and this was
what made me happiest. There are a lot of trade-offs, but here is what
I was looking for:

 * Only OSS-code in grist-core. Adding mixed-license material there
   feels unfair to people already working with the repo. That said,
   a possible future is to move away from our private monorepo to
   a public mixed-licence repo, which could have the same relationship
   with grist-core as the monorepo has.
 * Minimal differences between self-managed builds and one of our
   existing builds, ideally hewing as close to grist-core as possible
   for ease of documentation, debugging, and maintenance.
 * Ideally, docker builds without copying files around (the new
   `--build-context` functionality made that possible).
 * Compatibility with monorepo build.

Expressing dependencies of the extra code in `ext` proved tricky to
do in a clean way. Yarn/npm fought me every step of the way - everything
related to optional dependencies was unsatisfactory in some respect.
Yarn2 is flexible but smells like it might be overreach. In the end,
organizing to install non-core dependencies one directory up from the
main build was a good simple trick that saved my bacon.

This diff gets us to the point of building `grist-ee` images conveniently,
but there isn't a public repo people can go look at to see its source. This
could be generated by taking `grist-core`, adding the `ext` directory
to it, and pushing to a distinct repository. I'm not in a hurry to do that,
since a PR to that repo would be hard to sync with our monorepo and
`grist-core`. Also, we don't have any licensing text ready for the `ext`
directory. So leaving that for future work.

Test Plan: manual

Reviewers: georgegevoian, alexmojaki

Reviewed By: georgegevoian, alexmojaki

Differential Revision: https://phab.getgrist.com/D3415

.dockerignore

@@ -3,6 +3,7 @@
 !package.json
 !yarn.lock
 !tsconfig.json
+!tsconfig-ext.json
 !stubs
 !app
 !buildtools
@@ -12,3 +13,4 @@
 !sandbox
 !plugins
 !test
+!ext

Dockerfile

@@ -1,3 +1,11 @@
+################################################################################
+## The Grist source can be extended. This is a stub that can be overridden
+## from command line, as:
+##   docker buildx build -t ... --build-context=ext=<path> .
+## The code in <path> will then be built along with the rest of Grist.
+################################################################################
+FROM scratch as ext
+
 ################################################################################
 ## Javascript build stage
 ################################################################################
@@ -5,17 +13,25 @@
 FROM node:14-buster as builder
 
 # Install all node dependencies.
-ADD package.json package.json
-ADD yarn.lock yarn.lock
-RUN yarn install --frozen-lockfile
+WORKDIR /grist
+COPY package.json yarn.lock /grist/
+RUN yarn install --frozen-lockfile --verbose
+
+# Install any extra node dependencies (at root level, to avoid having to wrestle
+# with merging them).
+COPY --from=ext / /grist/ext
+RUN \
+ mkdir /node_modules && \
+ cd /grist/ext && \
+ { if [ -e package.json ] ; then yarn install --frozen-lockfile --modules-folder=/node_modules --verbose ; fi }
 
 # Build node code.
-ADD tsconfig.json tsconfig.json
-ADD app app
-ADD stubs stubs
-ADD buildtools buildtools
-ADD static static
-ADD test/tsconfig.json test/tsconfig.json
+COPY tsconfig.json /grist
+COPY tsconfig-ext.json /grist
+COPY test/tsconfig.json /grist/test/tsconfig.json
+COPY app /grist/app
+COPY stubs /grist/stubs
+COPY buildtools /grist/buildtools
 RUN yarn run build:prod
 
 ################################################################################
@@ -63,9 +79,10 @@ RUN \
 RUN mkdir -p /persist/docs
 
 # Copy node files.
-COPY --from=builder /node_modules node_modules
-COPY --from=builder /_build _build
-COPY --from=builder /static static
+COPY --from=builder /node_modules /node_modules
+COPY --from=builder /grist/node_modules /grist/node_modules
+COPY --from=builder /grist/_build /grist/_build
+COPY --from=builder /grist/static /grist/static-built
 
 # Copy python files.
 COPY --from=collector /usr/bin/python2.7 /usr/bin/python2.7
@@ -84,11 +101,19 @@ RUN \
 COPY --from=sandbox /runsc /usr/bin/runsc
 
 # Add files needed for running server.
-ADD package.json package.json
-ADD ormconfig.js ormconfig.js
-ADD bower_components bower_components
-ADD sandbox sandbox
-ADD plugins plugins
+ADD package.json /grist/package.json
+ADD ormconfig.js /grist/ormconfig.js
+ADD bower_components /grist/bower_components
+ADD sandbox /grist/sandbox
+ADD plugins /grist/plugins
+ADD static /grist/static
+
+# Finalize static directory
+RUN \
+  mv /grist/static-built/* /grist/static && \
+  rmdir /grist/static-built
+
+WORKDIR /grist
 
 # Set some default environment variables to give a setup that works out of the box when
 # started as:

README.md

@@ -268,5 +268,7 @@ GRIST_TEST_ROUTER | if set, then the home server will serve a mock version of ro
 
 This repository, `grist-core`, is released under the [Apache License, Version
 2.0](http://www.apache.org/licenses/LICENSE-2.0), which is an
-[OSI](https://opensource.org/)-approved free software license. See LICENSE.txt and NOTICE.txt for
-more information.
+[OSI](https://opensource.org/)-approved free software license.
+See LICENSE.txt and NOTICE.txt for more information.
+If you have received a version of Grist with an `ext` directory,
+the material within it is separately licensed.

app/server/lib/ExternalStorage.ts

@@ -366,3 +366,38 @@ export interface PropStorage {
 }
 
 export const Unchanged = Symbol('Unchanged');
+
+export interface ExternalStorageSettings {
+  purpose: 'doc' | 'meta';
+  basePrefix?: string;
+  extraPrefix?: string;
+}
+
+/**
+ * The storage mapping we use for our SaaS. A reasonable default, but relies
+ * on appropriate lifecycle rules being set up in the bucket.
+ */
+export function getExternalStorageKeyMap(settings: ExternalStorageSettings): (docId: string) => string {
+  const {basePrefix, extraPrefix, purpose} = settings;
+  let fullPrefix = basePrefix + (basePrefix?.endsWith('/') ? '' : '/');
+  if (extraPrefix) {
+    fullPrefix += extraPrefix + (extraPrefix.endsWith('/') ? '' : '/');
+  }
+
+  // Set up how we name files/objects externally.
+  let fileNaming: (docId: string) => string;
+  if (purpose === 'doc') {
+    fileNaming = docId => `${docId}.grist`;
+  } else if (purpose === 'meta') {
+    // Put this in separate prefix so a lifecycle rule can prune old versions of the file.
+    // Alternatively, could go in separate bucket.
+    fileNaming = docId => `assets/unversioned/${docId}/meta.json`;
+  } else {
+    throw new Error('create.ExternalStorage: unrecognized purpose');
+  }
+  return docId => (fullPrefix + fileNaming(docId));
+}
+
+export function wrapWithKeyMappedStorage(rawStorage: ExternalStorage, settings: ExternalStorageSettings) {
+  return new KeyMappedExternalStorage(rawStorage, getExternalStorageKeyMap(settings));
+}

app/server/lib/FlexServer.ts

@@ -1005,7 +1005,7 @@ export class FlexServer implements GristServer {
       const workers = this._docWorkerMap;
       const docWorkerId = await this._addSelfAsWorker(workers);
 
-      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableS3, '', workers,
+      const storageManager = new HostedStorageManager(this.docsRoot, docWorkerId, this._disableS3, workers,
                                                       this._dbManager, this.create);
       this._storageManager = storageManager;
     } else {

app/server/lib/HostedStorageManager.ts

@@ -53,7 +53,7 @@ export interface HostedStorageOptions {
   // which may then be wrapped in additional layer(s) of ExternalStorage.
   // See ICreate.ExternalStorage.
   // Uses S3 by default in hosted Grist.
-  innerExternalStorageCreate?: (bucket: string) => ExternalStorage;
+  externalStorageCreator?: (purpose: 'doc'|'meta') => ExternalStorage;
 }
 
 const defaultOptions: HostedStorageOptions = {
@@ -127,16 +127,15 @@ export class HostedStorageManager implements IDocStorageManager {
     private _docsRoot: string,
     private _docWorkerId: string,
     private _disableS3: boolean,
-    extraS3Prefix: string,
     private _docWorkerMap: IDocWorkerMap,
     dbManager: HomeDBManager,
     create: ICreate,
     options: HostedStorageOptions = defaultOptions
   ) {
+    const creator = options.externalStorageCreator || ((purpose) => create.ExternalStorage(purpose, ''));
     // We store documents either in a test store, or in an s3 store
     // at s3://<s3Bucket>/<s3Prefix><docId>.grist
-    const externalStoreDoc = this._disableS3 ? undefined :
-      create.ExternalStorage('doc', extraS3Prefix, options.innerExternalStorageCreate);
+    const externalStoreDoc = this._disableS3 ? undefined : creator('doc');
     if (!externalStoreDoc) { this._disableS3 = true; }
     const secondsBeforePush = options.secondsBeforePush;
     if (options.pushDocUpdateTimes) {
@@ -157,7 +156,7 @@ export class HostedStorageManager implements IDocStorageManager {
       this._ext = this._getChecksummedExternalStorage('doc', this._baseStore,
                                                       this._latestVersions, options);
 
-      const baseStoreMeta = create.ExternalStorage('meta', extraS3Prefix, options.innerExternalStorageCreate);
+      const baseStoreMeta = creator('meta');
       if (!baseStoreMeta) {
         throw new Error('bug: external storage should be created for "meta" if it is created for "doc"');
       }

app/server/lib/ICreate.ts

@@ -8,6 +8,7 @@ import {IBilling} from 'app/server/lib/IBilling';
 import {INotifier} from 'app/server/lib/INotifier';
 import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox';
 import {IShell} from 'app/server/lib/IShell';
+import {createSandbox} from 'app/server/lib/NSandbox';
 
 export interface ICreate {
 
@@ -20,14 +21,7 @@ export interface ICreate {
   //  - meta. This store need not be versioned, and can be eventually consistent.
   // For test purposes an extra prefix may be supplied.  Stores with different prefixes
   // should not interfere with each other.
-  // innerCreate should be a function returning the core ExternalStorage implementation,
-  // which this method may wrap in additional layer(s) of ExternalStorage.
-  // Uses S3 by default in hosted Grist.
-  ExternalStorage(
-    purpose: 'doc' | 'meta',
-    testExtraPrefix: string,
-    innerCreate?: (bucket: string) => ExternalStorage
-  ): ExternalStorage | undefined;
+  ExternalStorage(purpose: 'doc' | 'meta', testExtraPrefix: string): ExternalStorage | undefined;
 
   ActiveDoc(docManager: DocManager, docName: string, options: ICreateActiveDocOptions): ActiveDoc;
   NSandbox(options: ISandboxCreationOptions): ISandbox;
@@ -42,3 +36,60 @@ export interface ICreateActiveDocOptions {
   docUrl?: string;
   doc?: Document;
 }
+
+export interface ICreateStorageOptions {
+  check(): Record<string, string>|undefined;
+  create(purpose: 'doc'|'meta', extraPrefix: string): ExternalStorage|undefined;
+}
+
+export function makeSimpleCreator(opts: {
+  sessionSecret?: string,
+  storage?: ICreateStorageOptions[],
+}): ICreate {
+  return {
+    Billing() {
+      return {
+        addEndpoints() { /* do nothing */ },
+        addEventHandlers() { /* do nothing */ },
+        addWebhooks() { /* do nothing */ }
+      };
+    },
+    Notifier() {
+      return {
+        get testPending() { return false; },
+        deleteUser()      { throw new Error('deleteUser unavailable'); },
+      };
+    },
+    Shell() {
+      return {
+        moveItemToTrash()  { throw new Error('moveToTrash unavailable'); },
+        showItemInFolder() { throw new Error('showItemInFolder unavailable'); }
+      };
+    },
+    ExternalStorage(purpose, extraPrefix) {
+      for (const storage of opts.storage || []) {
+        const config = storage.check();
+        if (config) { return storage.create(purpose, extraPrefix); }
+      }
+      return undefined;
+    },
+    ActiveDoc(docManager, docName, options) { return new ActiveDoc(docManager, docName, options); },
+    NSandbox(options) {
+      return createSandbox('unsandboxed', options);
+    },
+    sessionSecret() {
+      const secret = process.env.GRIST_SESSION_SECRET || opts.sessionSecret;
+      if (!secret) {
+        throw new Error('need GRIST_SESSION_SECRET');
+      }
+      return secret;
+    },
+    configurationOptions() {
+      for (const storage of opts.storage || []) {
+        const config = storage.check();
+        if (config) { return config; }
+      }
+      return {};
+    }
+  };
+}

buildtools/build.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -e
+
+PROJECT=""
+export GRIST_EXT=stubs
+if [[ -e ext/app ]]; then
+  PROJECT="tsconfig-ext.json"
+fi
+
+set -x
+tsc --build $PROJECT
+webpack --config buildtools/webpack.config.js --mode production
+webpack --config buildtools/webpack.check.js --mode production
+cat app/client/*.css app/client/*/*.css > static/bundle.css

buildtools/tsconfig-base-ext.json

@@ -0,0 +1,12 @@
+{
+  "extends": "./tsconfig-base.json",
+  "compilerOptions": {
+    "paths": {
+      "*": [
+        "*",
+        "ext/*",
+        "stubs/*"
+      ],
+    }
+  }
+}

buildtools/tsconfig-base.json

@@ -16,7 +16,8 @@
       "*": [
         "*",
         "grist-core/*",
-        "stubs/*"
+        "stubs/*",
+        "ext/*"
       ],
     },
     "composite": true,

buildtools/webpack.config.js

@@ -34,6 +34,7 @@ module.exports = {
   resolve: {
     modules: [
       path.resolve('./_build'),
+      path.resolve('./_build/ext'),
       path.resolve('./_build/stubs'),
       path.resolve('./node_modules')
     ],

package.json

@@ -6,15 +6,15 @@
   "homepage": "https://github.com/gristlabs/grist-core",
   "repository": "git://github.com/gristlabs/grist-core.git",
   "scripts": {
-    "start": "tsc --build -w --preserveWatchOutput & catw app/client/*.css app/client/*/*.css -o static/bundle.css -v & webpack --config buildtools/webpack.config.js --mode development --watch --hide-modules & NODE_PATH=_build:_build/stubs nodemon --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js & wait",
+    "start": "sandbox/watch.sh",
     "install:python": "buildtools/prepare_python.sh",
     "install:python2": "buildtools/prepare_python2.sh",
     "install:python3": "buildtools/prepare_python3.sh",
-    "build:prod": "tsc --build && webpack --config buildtools/webpack.config.js --mode production && webpack --config buildtools/webpack.check.js --mode production && cat app/client/*.css app/client/*/*.css > static/bundle.css",
-    "start:prod": "NODE_PATH=_build:_build/stubs node _build/stubs/app/server/server.js",
-    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
-    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js",
-    "test:smoke": "NODE_PATH=_build:_build/stubs mocha _build/test/nbrowser/Smoke.js",
+    "build:prod": "buildtools/build.sh",
+    "start:prod": "sandbox/run.sh",
+    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh"
   },
   "keywords": [

sandbox/gvisor/get_checkpoint_path.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # This defines a GRIST_CHECKPOINT environment variable, where we will store
 # a sandbox checkpoint. The path is in principle arbitrary. In practice,

sandbox/gvisor/update_engine_checkpoint.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Create a checkpoint of a gvisor sandbox. It is best to make the
 # checkpoint in as close to the same circumstances as it will be used,
@@ -18,7 +18,7 @@ set -ex
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-export NODE_PATH=_build:_build/core:_build/stubs
+export NODE_PATH=_build:_build/core:_build/stubs:_build/ext
 source $SCRIPT_DIR/get_checkpoint_path.sh
 
 if [[ -z "GRIST_CHECKPOINT" ]]; then

sandbox/run.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 
@@ -7,4 +7,4 @@ if [[ "$GRIST_SANDBOX_FLAVOR" = "gvisor" ]]; then
   source ./sandbox/gvisor/get_checkpoint_path.sh
 fi
 
-exec yarn run start:prod
+NODE_PATH=_build:_build/stubs:_build/ext node _build/stubs/app/server/server.js

sandbox/watch.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -x
+
+PROJECT=""
+export GRIST_EXT=stubs
+if [[ -e ext/app ]]; then
+  PROJECT="tsconfig-ext.json"
+fi
+
+if [ ! -e _build ]; then
+  buildtools/build.sh
+fi
+
+tsc --build -w --preserveWatchOutput $PROJECT &
+catw app/client/*.css app/client/*/*.css -o static/bundle.css -v & webpack --config buildtools/webpack.config.js --mode development --watch --hide-modules &
+NODE_PATH=_build:_build/stubs:_build/ext nodemon --delay 1 -w _build/app/server -w _build/app/common _build/stubs/app/server/server.js &
+
+wait

stubs/app/server/lib/create.ts

@@ -1,37 +1,5 @@
-import {ActiveDoc} from 'app/server/lib/ActiveDoc';
-import {ICreate} from 'app/server/lib/ICreate';
-import {createSandbox} from 'app/server/lib/NSandbox';
+import { makeSimpleCreator } from 'app/server/lib/ICreate';
 
-export const create: ICreate = {
-  Billing() {
-    return {
-      addEndpoints() { /* do nothing */ },
-      addEventHandlers() { /* do nothing */ },
-      addWebhooks() { /* do nothing */ }
-    };
-  },
-  Notifier() {
-    return {
-      get testPending() { return false; },
-      deleteUser()      { throw new Error('deleteUser unavailable'); },
-    };
-  },
-  Shell() {
-    return {
-      moveItemToTrash()  { throw new Error('moveToTrash unavailable'); },
-      showItemInFolder() { throw new Error('showItemInFolder unavailable'); }
-    };
-  },
-  ExternalStorage() { return undefined; },
-  ActiveDoc(docManager, docName, options) { return new ActiveDoc(docManager, docName, options); },
-  NSandbox(options) {
-    return createSandbox('unsandboxed', options);
-  },
-  sessionSecret() {
-    return process.env.GRIST_SESSION_SECRET ||
-      'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh';
-  },
-  configurationOptions() {
-    return {};
-  }
-};
+export const create = makeSimpleCreator({
+  sessionSecret: 'Phoo2ag1jaiz6Moo2Iese2xoaphahbai3oNg7diemohlah0ohtae9iengafieS2Hae7quungoCi9iaPh'
+});

tsconfig-ext.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./buildtools/tsconfig-base-ext.json",
+  "files": [],
+  "include": [],
+  "references": [
+    { "path": "./ext/app" }
+  ],
+}