(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552

app/gen-server/lib/DocApiForwarder.ts

@@ -0,0 +1,103 @@
+import * as express from "express";
+import fetch, { RequestInit } from 'node-fetch';
+
+import { ApiError } from 'app/common/ApiError';
+import { removeTrailingSlash } from 'app/common/gutil';
+import { HomeDBManager } from "app/gen-server/lib/HomeDBManager";
+import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer';
+import { IDocWorkerMap } from "app/server/lib/DocWorkerMap";
+import { expressWrap } from "app/server/lib/expressWrap";
+import { getAssignmentId } from "app/server/lib/idUtils";
+
+/**
+ * Forwards all /api/docs/:docId/tables requests to the doc worker handling the :docId document. Makes
+ * sure the user has at least view access to the document otherwise rejects the request. For
+ * performance reason we stream the body directly from the request, which requires that no-one reads
+ * the req before, in particular you should register DocApiForwarder before bodyParser.
+ *
+ * Use:
+ *   const home = new ApiServer(false);
+ *   const docApiForwarder = new DocApiForwarder(getDocWorkerMap(), home);
+ *   app.use(docApiForwarder.getMiddleware());
+ *
+ * Note that it expects userId, and jsonErrorHandler middleware to be set up outside
+ * to apply to these routes.
+ */
+export class DocApiForwarder {
+
+  constructor(private _docWorkerMap: IDocWorkerMap, private _dbManager: HomeDBManager) {
+  }
+
+  public addEndpoints(app: express.Application) {
+    // Middleware to forward a request about an existing document that user has access to.
+    // We do not check whether the document has been soft-deleted; that will be checked by
+    // the worker if needed.
+    const withDoc = expressWrap(this._forwardToDocWorker.bind(this, true));
+    // Middleware to forward a request without a pre-existing document (for imports/uploads).
+    const withoutDoc = expressWrap(this._forwardToDocWorker.bind(this, false));
+    app.use('/api/docs/:docId/tables', withDoc);
+    app.use('/api/docs/:docId/force-reload', withDoc);
+    app.use('/api/docs/:docId/remove', withDoc);
+    app.delete('/api/docs/:docId', withDoc);
+    app.use('/api/docs/:docId/download', withDoc);
+    app.use('/api/docs/:docId/apply', withDoc);
+    app.use('/api/docs/:docId/attachments', withDoc);
+    app.use('/api/docs/:docId/snapshots', withDoc);
+    app.use('/api/docs/:docId/replace', withDoc);
+    app.use('/api/docs/:docId/flush', withDoc);
+    app.use('/api/docs/:docId/states', withDoc);
+    app.use('/api/docs/:docId/compare', withDoc);
+    app.use('^/api/docs$', withoutDoc);
+  }
+
+  private async _forwardToDocWorker(withDocId: boolean, req: express.Request, res: express.Response): Promise<void> {
+    let docId: string|null = null;
+    if (withDocId) {
+      const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, req.params.docId);
+      assertAccess('viewers', docAuth, {allowRemoved: true});
+      docId = docAuth.docId;
+    }
+    // Use the docId for worker assignment, rather than req.params.docId, which could be a urlId.
+    const assignmentId = getAssignmentId(this._docWorkerMap, docId === null ? 'import' : docId);
+
+    if (!this._docWorkerMap) {
+      throw new ApiError('no worker map', 404);
+    }
+    const docStatus = await this._docWorkerMap.assignDocWorker(assignmentId);
+
+    // Construct new url by keeping only origin and path prefixes of `docWorker.internalUrl`,
+    // and otherwise reflecting fully the original url (remaining path, and query params).
+    const docWorkerUrl = new URL(docStatus.docWorker.internalUrl);
+    const url = new URL(req.originalUrl, docWorkerUrl.origin);
+    url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
+
+    const headers: {[key: string]: string} = {
+      ...getTransitiveHeaders(req),
+      'Content-Type': req.get('Content-Type') || 'application/json',
+    };
+    for (const key of ['X-Sort', 'X-Limit']) {
+      const hdr = req.get(key);
+      if (hdr) { headers[key] = hdr; }
+    }
+    const options: RequestInit = {
+      method: req.method,
+      headers,
+    };
+    if (['POST', 'PATCH'].includes(req.method)) {
+      // uses `req` as a stream
+      options.body = req;
+    }
+    const docWorkerRes = await fetch(url.href, options);
+    res.status(docWorkerRes.status);
+    for (const key of ['content-type', 'content-disposition', 'cache-control']) {
+      const value = docWorkerRes.headers.get(key);
+      if (value) { res.set(key, value); }
+    }
+    return new Promise<void>((resolve, reject) => {
+      docWorkerRes.body.on('error', reject);
+      res.on('error', reject);
+      res.on('finish', resolve);
+      docWorkerRes.body.pipe(res);
+    });
+  }
+}

app/gen-server/lib/DocWorkerMap.ts

@@ -0,0 +1,440 @@
+import {MapWithTTL} from 'app/common/AsyncCreate';
+import * as version from 'app/common/version';
+import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
+import * as log from 'app/server/lib/log';
+import {checkPermitKey, formatPermitKey, Permit} from 'app/server/lib/Permit';
+import {promisifyAll} from 'bluebird';
+import mapValues = require('lodash/mapValues');
+import {createClient, Multi, RedisClient} from 'redis';
+import * as Redlock from 'redlock';
+import * as uuidv4 from 'uuid/v4';
+
+promisifyAll(RedisClient.prototype);
+promisifyAll(Multi.prototype);
+
+// Max time for which we will hold a lock, by default.  In milliseconds.
+const LOCK_TIMEOUT = 3000;
+
+// How long do checksums stored in redis last.  In milliseconds.
+// Should be long enough to allow S3 to reach consistency with very high probability.
+// Consistency failures shorter than this interval will be detectable, failures longer
+// than this interval will not be detectable.
+const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000;  // 24 hours
+
+// How long do permits stored in redis last, in milliseconds.
+const PERMIT_TTL_MSEC = 1 * 60 * 1000;  // 1 minute
+
+class DummyDocWorkerMap implements IDocWorkerMap {
+  private _worker?: DocWorkerInfo;
+  private _available: boolean = false;
+  private _permits = new MapWithTTL<string, string>(PERMIT_TTL_MSEC);
+  private _elections = new MapWithTTL<string, string>(1);  // default ttl never used
+
+  public async getDocWorker(docId: string) {
+    if (!this._worker) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async assignDocWorker(docId: string) {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    if (this._worker.id !== workerId) { throw new Error('worker not known'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async updateDocStatus(docId: string, checksum: string) {
+    // nothing to do
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    this._worker = info;
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    this._worker = undefined;
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    this._available = available;
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    // nothing to do
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return [];
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    this._permits.set(key, JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit> {
+    const result = this._permits.get(key);
+    return result ? JSON.parse(result) : null;
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    this._permits.delete(key);
+  }
+
+  public async close(): Promise<void> {
+    this._permits.clear();
+    this._elections.clear();
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    if (this._elections.get(name)) { return null; }
+    const key = uuidv4();
+    this._elections.setWithCustomTTL(name, key, durationInMs);
+    return key;
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    if (this._elections.get(name) === electionKey) {
+      this._elections.delete(name);
+    }
+  }
+}
+
+/**
+ * Manage the relationship between document and workers.  Backed by Redis.
+ * Can also assign workers to "groups" for serving particular documents.
+ * Keys used:
+ *   workers - the set of known active workers, identified by workerId
+ *   workers-available - the set of workers available for assignment (a subset of the workers set)
+ *   workers-available-{group} - the set of workers available for a given group
+ *   worker-{workerId} - a hash of contact information for a worker
+ *   worker-{workerId}-docs - a set of docs assigned to a worker, identified by docId
+ *   worker-{workerId}-group - if set, marks the worker as serving a particular group
+ *   doc-${docId} - a hash containing (JSON serialized) DocStatus fields, other than docMD5.
+ *   doc-${docId}-checksum - the docs docMD5, or 'null' if docMD5 is null
+ *   doc-${docId}-group - if set, marks the doc as to be served by workers in a given group
+ *   workers-lock - a lock used when working with the list of workers
+ *   groups - a hash from groupIds (arbitrary strings) to desired number of workers in group
+ *   elections-${deployment} - a hash, from groupId to a (serialized json) list of worker ids
+ *
+ * Assignments of documents to workers can end abruptly at any time.  Clients
+ * should be prepared to retry if a worker is not responding or denies that a document
+ * is assigned to it.
+ *
+ * If the groups key is set, workers assign themselves to groupIds to
+ * fill the counts specified in groups (in order of groupIds), and
+ * once those are exhausted, get assigned to the special group
+ * "default".
+ */
+export class DocWorkerMap implements IDocWorkerMap {
+  private _client: RedisClient;
+  private _redlock: Redlock;
+
+  // Optional deploymentKey argument supplies a key unique to the deployment (this is important
+  // for maintaining groups across redeployments only)
+  constructor(_clients?: RedisClient[], private _deploymentKey?: string, private _options?: {
+    permitMsec?: number
+  }) {
+    this._deploymentKey = this._deploymentKey || version.version;
+    _clients = _clients || [createClient(process.env.REDIS_URL)];
+    this._redlock = new Redlock(_clients);
+    this._client = _clients[0]!;
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    log.info(`DocWorkerMap.addWorker ${info.id}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Make a worker-{workerId} key with contact info, then add this worker to available set.
+      await this._client.hmsetAsync(`worker-${info.id}`, info);
+      await this._client.saddAsync('workers', info.id);
+      // Figure out if worker should belong to a group
+      const groups = await this._client.hgetallAsync('groups');
+      if (groups) {
+        const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`) || {};
+        for (const group of Object.keys(groups).sort()) {
+          const count = parseInt(groups[group], 10) || 0;
+          if (count < 1) { continue; }
+          const elected: string[] = JSON.parse(elections[group] || '[]');
+          if (elected.length >= count) { continue; }
+          elected.push(info.id);
+          await this._client.setAsync(`worker-${info.id}-group`, group);
+          await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, JSON.stringify(elected));
+          break;
+        }
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    log.info(`DocWorkerMap.removeWorker ${workerId}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Drop out of available set first.
+      await this._client.sremAsync('workers-available', workerId);
+      const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+      // At this point, this worker should no longer be receiving new doc assignments, though
+      // clients may still be directed to the worker.
+
+      // If we were elected for anything, back out.
+      const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`);
+      if (elections) {
+        if (group in elections) {
+          const elected: string[] = JSON.parse(elections[group]);
+          const newElected = elected.filter(worker => worker !== workerId);
+          if (elected.length !== newElected.length) {
+            if (newElected.length > 0) {
+              await this._client.hsetAsync(`elections-${this._deploymentKey}`, group,
+                                           JSON.stringify(newElected));
+            } else {
+              await this._client.hdelAsync(`elections-${this._deploymentKey}`, group);
+              delete elections[group];
+            }
+          }
+          // We're the last one involved in elections - remove the key entirely.
+          if (Object.keys(elected).length === 0) {
+            await this._client.delAsync(`elections-${this._deploymentKey}`);
+          }
+        }
+      }
+
+      // Now, we start removing the assignments.
+      const assignments = await this._client.smembersAsync(`worker-${workerId}-docs`);
+      if (assignments) {
+        const op = this._client.multi();
+        for (const doc of assignments) { op.del(`doc-${doc}`); }
+        await op.execAsync();
+      }
+
+      // Now remove worker-{workerId}* keys.
+      await this._client.delAsync(`worker-${workerId}-docs`);
+      await this._client.delAsync(`worker-${workerId}-group`);
+      await this._client.delAsync(`worker-${workerId}`);
+
+      // Forget about this worker completely.
+      await this._client.sremAsync('workers', workerId);
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
+    const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+    if (available) {
+      await this._client.saddAsync(`workers-available-${group}`, workerId);
+      await this._client.saddAsync('workers-available', workerId);
+    } else {
+      await this._client.sremAsync('workers-available', workerId);
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+    }
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    const op = this._client.multi();
+    op.del(`doc-${docId}`);
+    op.srem(`worker-${workerId}-docs`, docId);
+    await op.execAsync();
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return this._client.smembersAsync(`worker-${workerId}-docs`);
+  }
+
+  /**
+   * Defined by IDocWorkerMap.
+   *
+   * Looks up which DocWorker is responsible for this docId.
+   * Responsibility could change at any time after this call, so it
+   * should be treated as a hint, and clients should be prepared to be
+   * refused and need to retry.
+   */
+  public async getDocWorker(docId: string): Promise<DocStatus|null> {
+    // Fetch the various elements that go into making a DocStatus
+    const props = await this._client.multi()
+      .hgetall(`doc-${docId}`)
+      .get(`doc-${docId}-checksum`)
+      .execAsync() as [{[key: string]: any}|null, string|null]|null;
+    if (!props) { return null; }
+
+    // If there is no worker, return null.  An alternative would be to modify
+    // DocStatus so that it is possible for it to not have a worker assignment.
+    if (!props[0]) { return null; }
+
+    // Fields are JSON encoded since redis cannot store them directly.
+    const doc = mapValues(props[0], (val) => JSON.parse(val));
+
+    // Redis cannot store a null value, so we encode it as 'null', which does
+    // not match any possible MD5.
+    doc.docMD5 = props[1] === 'null' ? null : props[1];
+
+    // Ok, we have a valid DocStatus at this point.
+    return doc as DocStatus;
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a DocWorker to this docId if one is not yet assigned.
+   * Note that the assignment could be unmade at any time after this
+   * call if the worker dies, is brought down, or for other potential
+   * reasons in the future such as migration of individual documents
+   * between workers.
+   *
+   * A preferred doc worker can be specified, which will be assigned
+   * if no assignment is already made.
+   *
+   */
+  public async assignDocWorker(docId: string, workerId?: string): Promise<DocStatus> {
+    // Check if a DocWorker is already assigned; if so return result immediately
+    // without locking.
+    let docStatus = await this.getDocWorker(docId);
+    if (docStatus) { return docStatus; }
+
+    // No assignment yet, so let's lock and set an assignment up.
+    const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
+
+    try {
+      // Now that we've locked, recheck that the worker hasn't been reassigned
+      // in the meantime.  Return immediately if it has.
+      docStatus = await this.getDocWorker(docId);
+      if (docStatus) { return docStatus; }
+
+      if (!workerId) {
+        // Check if document has a preferred worker group set.
+        const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
+
+        // Let's start off by assigning documents to available workers randomly.
+        // TODO: use a smarter algorithm.
+        workerId = await this._client.srandmemberAsync(`workers-available-${group}`) || undefined;
+        if (!workerId) {
+          // No workers available in the desired worker group.  Rather than refusing to
+          // open the document, we fall back on assigning a worker from any of the workers
+          // available, regardless of grouping.
+          // This limits the impact of operational misconfiguration (bad redis setup,
+          // or not starting enough workers).  It has the downside of potentially disguising
+          // problems, so we log a warning.
+          log.warn(`DocWorkerMap.assignDocWorker ${docId} found no workers for group ${group}`);
+          workerId = await this._client.srandmemberAsync('workers-available') || undefined;
+        }
+        if (!workerId) { throw new Error('no doc workers available'); }
+      }  else {
+        if (!await this._client.sismemberAsync('workers-available', workerId)) {
+          throw new Error(`worker ${workerId} not known or not available`);
+        }
+      }
+
+      // Look up how to contact the worker.
+      const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
+      if (!docWorker) { throw new Error('no doc worker contact info available'); }
+
+      // We can now construct a DocStatus.
+      const newDocStatus = {docMD5: null, docWorker, isActive: true};
+
+      // We add the assignment to worker-{workerId}-docs and save doc-{docId}.
+      const result = await this._client.multi()
+        .sadd(`worker-${workerId}-docs`, docId)
+        .hmset(`doc-${docId}`, {
+          docWorker: JSON.stringify(docWorker),  // redis can't store nested objects, strings only
+          isActive: JSON.stringify(true)         // redis can't store booleans, strings only
+        })
+        .setex(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, 'null')
+        .execAsync();
+      if (!result) { throw new Error('failed to store new assignment'); }
+      return newDocStatus;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a specific DocWorker to this docId if one is not yet assigned.
+   *
+   */
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    return this.assignDocWorker(docId, workerId);
+  }
+
+  public async updateDocStatus(docId: string, checksum: string): Promise<void> {
+    await this._client.setexAsync(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, checksum);
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    const duration = (this._options && this._options.permitMsec) || PERMIT_TTL_MSEC;
+      // seems like only integer seconds are supported?
+    await this._client.setexAsync(key, Math.ceil(duration / 1000.0),
+                                  JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit|null> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    const result = await this._client.getAsync(key);
+    return result && JSON.parse(result);
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    await this._client.delAsync(key);
+  }
+
+  public async close(): Promise<void> {
+    // nothing to do
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    // Could use "set nx" for election, but redis docs don't encourage that any more,
+    // favoring redlock:
+    //   https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      if (await this._client.getAsync(redisKey) !== null) { return null; }
+      const electionKey = uuidv4();
+      // seems like only integer seconds are supported?
+      await this._client.setexAsync(redisKey, Math.ceil(durationInMs / 1000.0), electionKey);
+      return electionKey;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      const current = await this._client.getAsync(redisKey);
+      if (current === electionKey) {
+        await this._client.delAsync(redisKey);
+      } else if (current !== null) {
+        throw new Error('could not remove election');
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+}
+
+// If we don't have redis available and use a DummyDocWorker, it should be a singleton.
+let dummyDocWorkerMap: DummyDocWorkerMap|null = null;
+
+export function getDocWorkerMap(): IDocWorkerMap {
+  if (process.env.REDIS_URL) {
+    return new DocWorkerMap();
+  } else {
+    dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap();
+    return dummyDocWorkerMap;
+  }
+}

app/gen-server/lib/Permissions.ts

@@ -0,0 +1,21 @@
+export enum Permissions {
+  NONE          = 0x0,
+  // Note that the view permission bit provides view access ONLY to the resource to which
+  // the aclRule belongs - it does not allow listing that resource's children. A resource's
+  // children may only be listed if those children also have the view permission set.
+  VIEW          = 0x1,
+  UPDATE        = 0x2,
+  ADD           = 0x4,
+  // Note that the remove permission bit provides remove access to a resource AND all of
+  // its child resources/ACLs
+  REMOVE        = 0x8,
+  SCHEMA_EDIT   = 0x10,
+  ACL_EDIT      = 0x20,
+  EDITOR        = VIEW | UPDATE | ADD | REMOVE,   // tslint:disable-line:no-bitwise
+  ADMIN         = EDITOR | SCHEMA_EDIT,           // tslint:disable-line:no-bitwise
+  OWNER         = ADMIN | ACL_EDIT,               // tslint:disable-line:no-bitwise
+
+  // A virtual permission bit signifying that the general public has some access to
+  // the resource via ACLs involving the everyone@ user.
+  PUBLIC        = 0x80
+}

app/gen-server/lib/TypeORMPatches.ts

@@ -0,0 +1,196 @@
+// This contains two TypeORM patches.
+
+// Patch 1:
+// TypeORM Sqlite driver does not support using transactions in async code, if it is possible
+// for two transactions to get called (one of the whole point of transactions).  This
+// patch adds support for that, based on a monkey patch published in:
+//   https://gist.github.com/keenondrums/556f8c61d752eff730841170cd2bc3f1
+// Explanation at https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213
+
+// Patch 2:
+// TypeORM parameters are global, and collisions in setting them are not detected.
+// We add a patch to throw an exception if a parameter value is ever set and then
+// changed during construction of a query.
+
+import * as sqlite3 from '@gristlabs/sqlite3';
+import isEqual = require('lodash/isEqual');
+import {EntityManager, QueryRunner} from 'typeorm';
+import {SqliteDriver} from 'typeorm/driver/sqlite/SqliteDriver';
+import {SqliteQueryRunner} from 'typeorm/driver/sqlite/SqliteQueryRunner';
+import {
+  QueryRunnerProviderAlreadyReleasedError
+} from 'typeorm/error/QueryRunnerProviderAlreadyReleasedError';
+import {QueryBuilder} from 'typeorm/query-builder/QueryBuilder';
+
+
+/**********************
+ * Patch 1
+ **********************/
+
+type Releaser = () => void;
+type Worker<T> = () => Promise<T>|T;
+
+interface MutexInterface {
+  acquire(): Promise<Releaser>;
+  runExclusive<T>(callback: Worker<T>): Promise<T>;
+  isLocked(): boolean;
+}
+
+class Mutex implements MutexInterface {
+  private _queue: Array<(release: Releaser) => void> = [];
+  private _pending = false;
+
+  public isLocked(): boolean {
+    return this._pending;
+  }
+
+  public acquire(): Promise<Releaser> {
+    const ticket = new Promise<Releaser>(resolve => this._queue.push(resolve));
+    if (!this._pending) {
+      this._dispatchNext();
+    }
+    return ticket;
+  }
+
+  public runExclusive<T>(callback: Worker<T>): Promise<T> {
+    return this
+      .acquire()
+      .then(release => {
+        let result: T|Promise<T>;
+
+        try {
+          result = callback();
+        } catch (e) {
+          release();
+          throw(e);
+        }
+
+        return Promise
+          .resolve(result)
+          .then(
+            (x: T) => (release(), x),
+            e => {
+              release();
+              throw e;
+            }
+          );
+      }
+           );
+  }
+
+  private _dispatchNext(): void {
+    if (this._queue.length > 0) {
+      this._pending = true;
+      this._queue.shift()!(this._dispatchNext.bind(this));
+    } else {
+      this._pending = false;
+    }
+  }
+
+}
+
+// A singleton mutex for all sqlite transactions.
+const mutex = new Mutex();
+
+class SqliteQueryRunnerPatched extends SqliteQueryRunner {
+  private _releaseMutex: Releaser | null;
+
+  public async startTransaction(level?: any): Promise<void> {
+    this._releaseMutex = await mutex.acquire();
+    return super.startTransaction(level);
+  }
+
+  public async commitTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.commitTransaction -> mutex releaser unknown');
+    }
+    await super.commitTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async rollbackTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.rollbackTransaction -> mutex releaser unknown');
+    }
+    await super.rollbackTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async connect(): Promise<any> {
+    if (!this.isTransactionActive) {
+      const release = await mutex.acquire();
+      release();
+    }
+    return super.connect();
+  }
+}
+
+class SqliteDriverPatched extends SqliteDriver {
+  public createQueryRunner(): QueryRunner {
+    if (!this.queryRunner) {
+      this.queryRunner = new SqliteQueryRunnerPatched(this);
+    }
+    return this.queryRunner;
+  }
+  protected loadDependencies(): void {
+    // Use our own sqlite3 module, which is a fork of the original.
+    this.sqlite = sqlite3;
+  }
+}
+
+// Patch the underlying SqliteDriver, since it's impossible to convince typeorm to use only our
+// patched classes. (Previously we patched DriverFactory and Connection, but those would still
+// create an unpatched SqliteDriver and then overwrite it.)
+SqliteDriver.prototype.createQueryRunner = SqliteDriverPatched.prototype.createQueryRunner;
+(SqliteDriver.prototype as any).loadDependencies = (SqliteDriverPatched.prototype as any).loadDependencies;
+
+export function applyPatch() {
+  // tslint: disable-next-line
+  EntityManager.prototype.transaction = async function <T>(arg1: any,  arg2?: any): Promise<T> {
+    if (this.queryRunner && this.queryRunner.isReleased) {
+      throw new QueryRunnerProviderAlreadyReleasedError();
+    }
+    if (this.queryRunner && this.queryRunner.isTransactionActive) {
+      throw new Error(`Cannot start transaction because its already started`);
+    }
+    const queryRunner = this.connection.createQueryRunner();
+    const runInTransaction = typeof arg1 === "function" ? arg1 : arg2;
+    try {
+      await queryRunner.startTransaction();
+      const result = await runInTransaction(queryRunner.manager);
+      await queryRunner.commitTransaction();
+      return result;
+    } catch (err) {
+      try {
+        // we throw original error even if rollback thrown an error
+        await queryRunner.rollbackTransaction();
+        // tslint: disable-next-line
+      } catch (rollbackError) {
+        // tslint: disable-next-line
+      }
+      throw err;
+    } finally {
+      await queryRunner.release();
+    }
+  };
+}
+
+
+/**********************
+ * Patch 2
+ **********************/
+
+abstract class QueryBuilderPatched<T> extends QueryBuilder<T> {
+  public setParameter(key: string, value: any): this {
+    const prev = this.expressionMap.parameters[key];
+    if (prev !== undefined && !isEqual(prev, value)) {
+      throw new Error(`TypeORM parameter collision for key '${key}' ('${prev}' vs '${value}')`);
+    }
+    this.expressionMap.parameters[key] = value;
+    return this;
+  }
+}
+
+(QueryBuilder.prototype as any).setParameter = (QueryBuilderPatched.prototype as any).setParameter;

app/gen-server/lib/Usage.ts

@@ -0,0 +1,62 @@
+import {Document} from 'app/gen-server/entity/Document';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {User} from 'app/gen-server/entity/User';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import * as log from 'app/server/lib/log';
+
+// Frequency of logging usage information.  Not something we need
+// to track with much granularity.
+const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000;   // log every 1 hour
+
+/**
+ * Occasionally log usage information - number of users, orgs,
+ * docs, etc.
+ */
+export class Usage {
+  private _interval: NodeJS.Timeout;
+
+  public constructor(private _dbManager: HomeDBManager) {
+    this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS);
+    // Log once at beginning, in case we roll over servers faster than
+    // the logging period for an extended length of time,
+    // and to raise the visibility of this logging step so if it gets
+    // slow devs notice.
+    this.apply().catch(log.warn.bind(log));
+  }
+
+  public close() {
+    clearInterval(this._interval);
+  }
+
+  public async apply() {
+    const manager = this._dbManager.connection.manager;
+    // raw count of users
+    const userCount = await manager.count(User);
+    // users who have logged in at least once
+    const userWithLoginCount = await manager.createQueryBuilder()
+      .from(User, 'users')
+      .where('first_login_at is not null')
+      .getCount();
+    // raw count of organizations (excluding personal orgs)
+    const orgCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .where('owner_id is null')
+      .getCount();
+    // organizations with subscriptions that are in a non-terminated state
+    const orgInGoodStandingCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .where('owner_id is null')
+      .andWhere('billing_accounts.in_good_standing = true')
+      .getCount();
+    // raw count of documents
+    const docCount = await manager.count(Document);
+    log.rawInfo('activity', {
+      docCount,
+      orgCount,
+      orgInGoodStandingCount,
+      userCount,
+      userWithLoginCount,
+    });
+  }
+}

app/gen-server/lib/scrubUserFromOrg.ts

@@ -0,0 +1,209 @@
+import {EntityManager} from "typeorm";
+import * as roles from 'app/common/roles';
+import {Document} from "app/gen-server/entity/Document";
+import {Group} from "app/gen-server/entity/Group";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Workspace} from "app/gen-server/entity/Workspace";
+import pick = require('lodash/pick');
+
+/**
+ *
+ * Remove the given user from the given org and every resource inside the org.
+ * If the user being removed is an owner of any resources in the org, the caller replaces
+ * them as the owner. This is to prevent complete loss of access to any resource.
+ *
+ * This method transforms ownership without regard to permissions.  We all talked this
+ * over and decided this is what we wanted, but there's no denying it is funky and could
+ * be surprising.
+ * TODO: revisit user scrubbing when we can.
+ *
+ */
+export async function scrubUserFromOrg(
+  orgId: number,
+  removeUserId: number,
+  callerUserId: number,
+  manager: EntityManager
+): Promise<void> {
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+
+  // This will be a list of all mentions of removeUser and callerUser in any resource
+  // within the org.
+  const mentions: Mention[] = [];
+
+  // Base query for all group_users related to these two users and this org.
+  const q = manager.createQueryBuilder()
+    .select('group_users.group_id, group_users.user_id')
+    .from('group_users', 'group_users')
+    .leftJoin(Group, 'groups', 'group_users.group_id = groups.id')
+    .addSelect('groups.name as name')
+    .leftJoin('groups.aclRule', 'acl_rules')
+    .where('(group_users.user_id = :removeUserId or group_users.user_id = :callerUserId)',
+           {removeUserId, callerUserId})
+    .andWhere('orgs.id = :orgId', {orgId});
+
+  // Pick out group_users related specifically to the org resource, in 'mentions' format
+  // (including resource id, a tag for the kind of resource, the group name, the user
+  // id, and the group id).
+  const orgs = q.clone()
+    .addSelect(`'org' as kind, orgs.id`)
+    .innerJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id');
+  mentions.push(...await orgs.getRawMany());
+
+  // Pick out mentions related to any workspace within the org.
+  const wss = q.clone()
+    .innerJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id')
+    .addSelect(`'ws' as kind, workspaces.id`)
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await wss.getRawMany());
+
+  // Pick out mentions related to any doc within the org.
+  const docs = q.clone()
+    .innerJoin(Document, 'docs', 'docs.id = acl_rules.doc_id')
+    .addSelect(`'doc' as kind, docs.id`)
+    .innerJoin('docs.workspace', 'workspaces')
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await docs.getRawMany());
+
+  // Prepare to add and delete group_users.
+  const toDelete: Mention[] = [];
+  const toAdd: Mention[] = [];
+
+  // Now index the mentions by whether they are for the removeUser or the callerUser,
+  // and the resource they apply to.
+  const removeUserMentions = new Map<MentionKey, Mention>();
+  const callerUserMentions = new Map<MentionKey, Mention>();
+  for (const mention of mentions) {
+    const isGuest = mention.name === roles.GUEST;
+    if (mention.user_id === removeUserId) {
+      // We can safely remove any guest roles for the removeUser without any
+      // further inspection.
+      if (isGuest) { toDelete.push(mention); continue; }
+      removeUserMentions.set(getMentionKey(mention), mention);
+    } else {
+      if (isGuest) { continue; }
+      callerUserMentions.set(getMentionKey(mention), mention);
+    }
+  }
+  // Now iterate across the mentions of removeUser, and see what we need to do
+  // for each of them.
+  for (const [key, removeUserMention] of removeUserMentions) {
+    toDelete.push(removeUserMention);
+    if (removeUserMention.name !== roles.OWNER) {
+      // Nothing fancy needed for cases where the removeUser is not the owner.
+      // Just discard those.
+      continue;
+    }
+    // The removeUser was a direct owner on this resource, but the callerUser was
+    // not.  We set the callerUser as a direct owner on this resource, to preserve
+    // access to it.
+    // TODO: the callerUser might inherit sufficient access, in which case this
+    // step is unnecessary and could be skipped.  I believe it does no harm though.
+    const callerUserMention = callerUserMentions.get(key);
+    if (callerUserMention && callerUserMention.name === roles.OWNER) { continue; }
+    if (callerUserMention) { toDelete.push(callerUserMention); }
+    toAdd.push({...removeUserMention, user_id: callerUserId});
+  }
+  if (toDelete.length > 0) {
+    await manager.createQueryBuilder()
+      .delete()
+      .from('group_users')
+      .whereInIds(toDelete.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+  if (toAdd.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(toAdd.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+
+  // TODO: At this point, we've removed removeUserId from every mention in group_users.
+  // The user may still be mentioned in billing_account_managers.  If the billing_account
+  // is linked to just this single organization, perhaps it would make sense to remove
+  // the user there, if the callerUser is themselves a billing account manager?
+
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+}
+
+/**
+ * Adds specified user to any guest groups for the resources of an org where the
+ * user needs to be and is not already.
+ */
+export async function addMissingGuestMemberships(userId: number, orgId: number,
+                                                 manager: EntityManager) {
+  // For workspaces:
+  // User should be in guest group if mentioned in a doc within that workspace.
+  let groupUsers = await manager.createQueryBuilder()
+    .select('workspace_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Workspace, 'workspaces')
+    .where('workspaces.org_id = :orgId', {orgId})
+    .innerJoin('workspaces.docs', 'docs')
+    .innerJoin('docs.aclRules', 'doc_acl_rules')
+    .innerJoin('doc_acl_rules.group', 'doc_groups')
+    .innerJoin('doc_groups.memberUsers', 'doc_group_users')
+    .andWhere('doc_group_users.id = :userId', {userId})
+    .leftJoin('workspaces.aclRules', 'workspace_acl_rules')
+    .leftJoin('workspace_acl_rules.group', 'workspace_groups')
+    .leftJoin('group_users', 'workspace_group_users',
+              'workspace_group_users.group_id = workspace_groups.id and ' +
+              'workspace_group_users.user_id = :userId')
+    .andWhere('workspace_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('workspaces.id, workspace_groups.id, workspace_group_users.user_id')
+    .having('workspace_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For org:
+  // User should be in guest group if mentioned in a workspace within that org.
+  groupUsers = await manager.createQueryBuilder()
+    .select('org_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Organization, 'orgs')
+    .where('orgs.id = :orgId', {orgId})
+    .innerJoin('orgs.workspaces', 'workspaces')
+    .innerJoin('workspaces.aclRules', 'workspaces_acl_rules')
+    .innerJoin('workspaces_acl_rules.group', 'workspace_groups')
+    .innerJoin('workspace_groups.memberUsers', 'workspace_group_users')
+    .andWhere('workspace_group_users.id = :userId', {userId})
+    .leftJoin('orgs.aclRules', 'org_acl_rules')
+    .leftJoin('org_acl_rules.group', 'org_groups')
+    .leftJoin('group_users', 'org_group_users',
+              'org_group_users.group_id = org_groups.id and ' +
+              'org_group_users.user_id = :userId')
+    .andWhere('org_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('org_groups.id, org_group_users.user_id')
+    .having('org_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For doc:
+  // Guest groups are not used.
+}
+
+interface Mention {
+  id: string|number;       // id of resource
+  kind: 'org'|'ws'|'doc';  // type of resource
+  user_id: number;         // id of user in group
+  group_id: number;        // id of group
+  name: string;            // name of group
+}
+
+type MentionKey = string;
+
+function getMentionKey(mention: Mention): MentionKey {
+  return `${mention.kind} ${mention.id}`;
+}

app/gen-server/lib/values.ts

@@ -0,0 +1,36 @@
+/**
+ * This smoothes over some awkward differences between TypeORM treatment of
+ * booleans and json in sqlite and postgres.  Booleans and json work fine
+ * with each db, but have different levels of driver-level support.
+ */
+
+export interface NativeValues {
+  // Json columns are handled natively by the postgres driver, but for
+  // sqlite requires a typeorm wrapper (simple-json).
+  jsonEntityType: 'json' | 'simple-json';
+  jsonType: 'json' | 'varchar';
+  booleanType: 'boolean' | 'integer';
+  dateTimeType: 'timestamp with time zone' | 'datetime';
+  trueValue: boolean | number;
+  falseValue: boolean | number;
+}
+
+const sqliteNativeValues: NativeValues = {
+  jsonEntityType: 'simple-json',
+  jsonType: 'varchar',
+  booleanType: 'integer',
+  dateTimeType: 'datetime',
+  trueValue: 1,
+  falseValue: 0
+};
+
+const postgresNativeValues: NativeValues = {
+  jsonEntityType: 'json',
+  jsonType: 'json',
+  booleanType: 'boolean',
+  dateTimeType: 'timestamp with time zone',
+  trueValue: true,
+  falseValue: false
+};
+
+export const nativeValues = (process.env.TYPEORM_TYPE === 'postgres') ? postgresNativeValues : sqliteNativeValues;