(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552

app/gen-server/ApiServer.ts

@@ -0,0 +1,475 @@
+import * as crypto from 'crypto';
+import * as express from 'express';
+import {EntityManager} from 'typeorm';
+
+import {ApiError} from 'app/common/ApiError';
+import {FullUser} from 'app/common/LoginSessionAPI';
+import {OrganizationProperties} from 'app/common/UserAPI';
+import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession';
+import {expressWrap} from 'app/server/lib/expressWrap';
+import {RequestWithOrg} from 'app/server/lib/extractOrg';
+import * as log from 'app/server/lib/log';
+import {getDocScope, getScope, integerParam, isParameterOn, sendOkReply,
+        sendReply, stringParam} from 'app/server/lib/requestUtils';
+import {Request} from 'express';
+
+import {User} from './entity/User';
+import {HomeDBManager} from './lib/HomeDBManager';
+
+// exposed for testing purposes
+export const Deps = {
+  apiKeyGenerator: () => crypto.randomBytes(20).toString('hex')
+};
+
+// Fetch the org this request was made for, or null if it isn't tied to a particular org.
+// Early middleware should have put the org in the request object for us.
+export function getOrgFromRequest(req: Request): string|null {
+  return (req as RequestWithOrg).org || null;
+}
+
+/**
+ * Compute the signature of the user's email address using HelpScout's secret key, to prove to
+ * HelpScout the user identity for identifying customer information and conversation history.
+ */
+function helpScoutSign(email: string): string|undefined {
+  const secretKey = process.env.HELP_SCOUT_SECRET_KEY;
+  if (!secretKey) { return undefined; }
+  return crypto.createHmac('sha256', secretKey).update(email).digest('hex');
+}
+
+/**
+ * Fetch an identifier for an organization from the "oid" parameter of the request.
+ *   - Integers are accepted, and will be compared with values in orgs.id column
+ *   - Strings are accepted, and will be compared with values in orgs.domain column
+ *     (or, if they match the pattern docs-NNNN, will check orgs.owner_id)
+ *   - The special string "current" is replaced with the current org domain embedded
+ *     in the url
+ *   - If there is no identifier available, a 400 error is thrown.
+ */
+export function getOrgKey(req: Request): string|number {
+  let orgKey: string|null = stringParam(req.params.oid);
+  if (orgKey === 'current') {
+    orgKey = getOrgFromRequest(req);
+  }
+  if (!orgKey) {
+    throw new ApiError("No organization chosen", 400);
+  } else if (/^\d+$/.test(orgKey)) {
+    return parseInt(orgKey, 10);
+  }
+  return orgKey;
+}
+
+// Adds an non-personal org with a new billingAccout, with the given name and domain.
+// Returns a QueryResult with the orgId on success.
+export function addOrg(
+  dbManager: HomeDBManager,
+  userId: number,
+  props: Partial<OrganizationProperties>,
+): Promise<number> {
+  return dbManager.connection.transaction(async manager => {
+    const user = await manager.findOne(User, userId);
+    if (!user) { return handleDeletedUser(); }
+    const query = await dbManager.addOrg(user, props, false, true, manager);
+    if (query.status !== 200) { throw new ApiError(query.errMessage!, query.status); }
+    return query.data!;
+  });
+}
+
+/**
+ * Provides a REST API for the landing page, which returns user's workspaces, organizations and documents.
+ * Temporarily sqlite database is used. Later it will be changed to RDS Aurora or PostgreSQL.
+ */
+export class ApiServer {
+  /**
+   * Add API endpoints to the specified connection. An error handler is added to /api to make sure
+   * all error responses have a body in json format.
+   *
+   * Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside
+   * to apply to these routes, and trustOrigin too for cross-domain requests.
+   */
+  constructor(
+    private _app: express.Application,
+    private _dbManager: HomeDBManager,
+  ) {
+    this._addEndpoints();
+  }
+
+  private _addEndpoints(): void {
+    // GET /api/orgs
+    // Get all organizations user may have some access to.
+    this._app.get('/api/orgs', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      const domain = getOrgFromRequest(req);
+      const merged = Boolean(req.query.merged);
+      const query = merged ?
+        await this._dbManager.getMergedOrgs(userId, userId, domain) :
+        await this._dbManager.getOrgs(userId, domain);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/workspace/:wid
+    // Get workspace by id, returning nested documents that user has access to.
+    this._app.get('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.getWorkspace(getScope(req), wsId);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid
+    // Get organization by id
+    this._app.get('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrg(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid/workspaces
+    // Get all workspaces and nested documents of organization that user has access to.
+    this._app.get('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrgWorkspaces(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/orgs
+    // Body params: name (required), domain
+    // Create a new org.
+    this._app.post('/api/orgs', expressWrap(async (req, res) => {
+      // Don't let anonymous users end up owning organizations, it will be confusing.
+      // Maybe if the user has presented credentials this would be ok - but addOrg
+      // doesn't have access to that information yet, so punting on this.
+      // TODO: figure out who should be allowed to create organizations
+      const userId = getAuthorizedUserId(req);
+      const orgId = await addOrg(this._dbManager, userId, req.body);
+      return sendOkReply(req, res, orgId);
+    }));
+
+    // PATCH /api/orgs/:oid
+    // Body params: name, domain
+    // Update the specified org.
+    this._app.patch('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.updateOrg(getScope(req), org, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // // DELETE /api/orgs/:oid
+    // Delete the specified org and all included workspaces and docs.
+    this._app.delete('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.deleteOrg(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/orgs/:oid/workspaces
+    // Body params: name
+    // Create a new workspace owned by the specific organization.
+    this._app.post('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.addWorkspace(getScope(req), org, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/workspaces/:wid
+    // Body params: name
+    // Update the specified workspace.
+    this._app.patch('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.updateWorkspace(getScope(req), wsId, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // // DELETE /api/workspaces/:wid
+    // Delete the specified workspace and all included docs.
+    this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/workspaces/:wid/remove
+    // Soft-delete the specified workspace.  If query parameter "permanent" is set,
+    // delete permanently.
+    this._app.post('/api/workspaces/:wid/remove', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      if (isParameterOn(req.query.permanent)) {
+        const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
+        return sendReply(req, res, query);
+      } else {
+        await this._dbManager.softDeleteWorkspace(getScope(req), wsId);
+        return sendOkReply(req, res);
+      }
+    }));
+
+    // POST /api/workspaces/:wid/unremove
+    // Recover the specified workspace if it was previously soft-deleted and is
+    // still available.
+    this._app.post('/api/workspaces/:wid/unremove', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      await this._dbManager.undeleteWorkspace(getScope(req), wsId);
+      return sendOkReply(req, res);
+    }));
+
+    // POST /api/workspaces/:wid/docs
+    // Create a new doc owned by the specific workspace.
+    this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.addDocument(getScope(req), wsId, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did
+    // Update the specified doc.
+    this._app.patch('/api/docs/:did', expressWrap(async (req, res) => {
+      const query = await this._dbManager.updateDocument(getDocScope(req), req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/docs/:did/unremove
+    // Recover the specified doc if it was previously soft-deleted and is
+    // still available.
+    this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => {
+      await this._dbManager.undeleteDocument(getDocScope(req));
+      return sendOkReply(req, res);
+    }));
+
+    // PATCH /api/orgs/:oid/access
+    // Update the specified org acl rules.
+    this._app.patch('/api/orgs/:oid/access', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateOrgPermissions(getScope(req), org, delta);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/workspaces/:wid/access
+    // Update the specified workspace acl rules.
+    this._app.patch('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
+      const workspaceId = integerParam(req.params.wid);
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateWorkspacePermissions(getScope(req), workspaceId, delta);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/docs/:did
+    // Get information about a document.
+    this._app.get('/api/docs/:did', expressWrap(async (req, res) => {
+      const query = await this._dbManager.getDoc(getDocScope(req));
+      return sendOkReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did/access
+    // Update the specified doc acl rules.
+    this._app.patch('/api/docs/:did/access', expressWrap(async (req, res) => {
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateDocPermissions(getDocScope(req), delta);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did/move
+    // Move the doc to the workspace specified in the body.
+    this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => {
+      const workspaceId = req.body.workspace;
+      const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId);
+      return sendReply(req, res, query);
+    }));
+
+    this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => {
+      const query = await this._dbManager.pinDoc(getDocScope(req), true);
+      return sendReply(req, res, query);
+    }));
+
+    this._app.patch('/api/docs/:did/unpin', expressWrap(async (req, res) => {
+      const query = await this._dbManager.pinDoc(getDocScope(req), false);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid/access
+    // Get user access information regarding an org
+    this._app.get('/api/orgs/:oid/access', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrgAccess(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/workspaces/:wid/access
+    // Get user access information regarding a workspace
+    this._app.get('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
+      const workspaceId = integerParam(req.params.wid);
+      const query = await this._dbManager.getWorkspaceAccess(getScope(req), workspaceId);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/docs/:did/access
+    // Get user access information regarding a doc
+    this._app.get('/api/docs/:did/access', expressWrap(async (req, res) => {
+      const query = await this._dbManager.getDocAccess(getDocScope(req));
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/profile/user
+    // Get user's profile
+    this._app.get('/api/profile/user', expressWrap(async (req, res) => {
+      const fullUser = await this._getFullUser(req);
+      return sendOkReply(req, res, fullUser);
+    }));
+
+    // POST /api/profile/user/name
+    // Body params: string
+    // Update users profile.
+    this._app.post('/api/profile/user/name', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      if (!(req.body && req.body.name)) {
+        throw new ApiError('Name expected in the body', 400);
+      }
+      const name = req.body.name;
+      await this._dbManager.updateUserName(userId, name);
+      res.sendStatus(200);
+    }));
+
+    // GET /api/profile/apikey
+    // Get user's apiKey
+    this._app.get('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      const user = await User.findOne(userId);
+      if (user) {
+        // The null value is of no interest to the user, let's show empty string instead.
+        res.send(user.apiKey || '');
+        return;
+      }
+      handleDeletedUser();
+    }));
+
+    // POST /api/profile/apikey
+    // Update user's apiKey
+    this._app.post('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getAuthorizedUserId(req);
+      const force = req.body ? req.body.force : false;
+      const manager = this._dbManager.connection.manager;
+      let user = await manager.findOne(User, userId);
+      if (!user) { return handleDeletedUser(); }
+      if (!user.apiKey || force) {
+        user = await updateApiKeyWithRetry(manager, user);
+        res.status(200).send(user.apiKey);
+      } else {
+        res.status(400).send({error: "An apikey is already set, use `{force: true}` to override it."});
+      }
+    }));
+
+    // DELETE /api/profile/apiKey
+    // Delete apiKey
+    this._app.delete('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getAuthorizedUserId(req);
+      await this._dbManager.connection.transaction(async manager => {
+        const user = await manager.findOne(User, userId);
+        if (!user) {return handleDeletedUser(); }
+        user.apiKey = null;
+        await manager.save(User, user);
+      });
+      res.sendStatus(200);
+    }));
+
+    // GET /api/session/access/active
+    // Returns active user and active org (if any)
+    this._app.get('/api/session/access/active', expressWrap(async (req, res) => {
+      const fullUser = await this._getFullUser(req);
+      const domain = getOrgFromRequest(req);
+      const org = domain ? (await this._dbManager.getOrg(getScope(req), domain || null)) : null;
+      const orgError = (org && org.errMessage) ? {error: org.errMessage, status: org.status} : undefined;
+      return sendOkReply(req, res, {
+        user: {...fullUser, helpScoutSignature: helpScoutSign(fullUser.email)},
+        org: (org && org.data) || null,
+        orgError
+      });
+    }));
+
+    // POST /api/session/access/active
+    // Body params: email (required)
+    // Sets active user for active org
+    this._app.post('/api/session/access/active', expressWrap(async (req, res) => {
+      const mreq = req as RequestWithLogin;
+      const domain = getOrgFromRequest(mreq);
+      const email = req.body.email;
+      if (!email) { throw new ApiError('email required', 400); }
+      try {
+        // Modify session copy in request. Will be saved to persistent storage before responding
+        // by express-session middleware.
+        linkOrgWithEmail(mreq.session, req.body.email, domain || '');
+        return sendOkReply(req, res, {email});
+      } catch (e) {
+        throw new ApiError('email not available', 403);
+      }
+    }));
+
+    // GET /api/session/access/all
+    // Returns all user profiles (with ids) and all orgs they can access.
+    // Flattens personal orgs into a single org.
+    this._app.get('/api/session/access/all', expressWrap(async (req, res) => {
+      const domain = getOrgFromRequest(req);
+      const users = getUserProfiles(req);
+      const userId = getUserId(req);
+      const orgs = await this._dbManager.getMergedOrgs(userId, users, domain);
+      if (orgs.errMessage) { throw new ApiError(orgs.errMessage, orgs.status); }
+      return sendOkReply(req, res, {
+        users: await this._dbManager.completeProfiles(users),
+        orgs: orgs.data
+      });
+    }));
+
+    // DELETE /users/:uid
+    // Delete the specified user, their personal organization, removing them from all groups.
+    // Not available to the anonymous user.
+    // TODO: should orphan orgs, inaccessible by anyone else, get deleted when last user
+    // leaves?
+    this._app.delete('/api/users/:uid', expressWrap(async (req, res) => {
+      const userIdToDelete = parseInt(req.params.uid, 10);
+      if (!(req.body && req.body.name !== undefined)) {
+        throw new ApiError('to confirm deletion of a user, provide their name', 400);
+      }
+      const query = await this._dbManager.deleteUser(getScope(req), userIdToDelete, req.body.name);
+      return sendReply(req, res, query);
+    }));
+  }
+
+  private async _getFullUser(req: Request): Promise<FullUser> {
+    const mreq = req as RequestWithLogin;
+    const userId = getUserId(mreq);
+    const fullUser = await this._dbManager.getFullUser(userId);
+    const domain = getOrgFromRequest(mreq);
+    const sessionUser = getSessionUser(mreq.session, domain || '');
+    const loginMethod = sessionUser && sessionUser.profile ? sessionUser.profile.loginMethod : undefined;
+    return {...fullUser, loginMethod};
+  }
+}
+
+/**
+ * Throw the error for when a user has been deleted since point of call (very unlikely to happen).
+ */
+function handleDeletedUser(): never {
+  throw new ApiError("user not known", 401);
+}
+
+/**
+ * Helper to update a user's apiKey. Update might fail because of the DB uniqueness constraint on
+ * the apiKey (although it is very unlikely according to `crypto`), we retry until success. Fails
+ * after 5 unsuccessful attempts.
+ */
+async function updateApiKeyWithRetry(manager: EntityManager, user: User): Promise<User> {
+  const currentKey = user.apiKey;
+  for (let i = 0; i < 5; ++i) {
+    user.apiKey = Deps.apiKeyGenerator();
+    try {
+      // if new key is the same as the current, the db update won't fail so we check it here (very
+      // unlikely to happen but but still better to handle)
+      if (user.apiKey === currentKey) {
+        throw new Error('the new key is the same as the current key');
+      }
+      return await manager.save(User, user);
+    } catch (e) {
+      // swallow and retry
+      log.warn(`updateApiKeyWithRetry: failed attempt ${i}/5, %s`, e);
+    }
+  }
+  throw new Error('Could not generate a valid api key.');
+}

app/gen-server/entity/AclRule.ts

@@ -0,0 +1,58 @@
+import {BaseEntity, ChildEntity, Column, Entity, JoinColumn, ManyToOne, OneToOne,
+        PrimaryGeneratedColumn, RelationId, TableInheritance} from "typeorm";
+
+import {Document} from "./Document";
+import {Group} from "./Group";
+import {Organization} from "./Organization";
+import {Workspace} from "./Workspace";
+
+@Entity('acl_rules')
+@TableInheritance({ column: { type: "int", name: "type" } })
+export class AclRule extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public permissions: number;
+
+  @OneToOne(type => Group, group => group.aclRule)
+  @JoinColumn({name: "group_id"})
+  public group: Group;
+}
+
+
+@ChildEntity()
+export class AclRuleWs extends AclRule {
+
+  @ManyToOne(type => Workspace, workspace => workspace.aclRules)
+  @JoinColumn({name: "workspace_id"})
+  public workspace: Workspace;
+
+  @RelationId((aclRule: AclRuleWs) => aclRule.workspace)
+  public workspaceId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleOrg extends AclRule {
+
+  @ManyToOne(type => Organization, organization => organization.aclRules)
+  @JoinColumn({name: "org_id"})
+  public organization: Organization;
+
+  @RelationId((aclRule: AclRuleOrg) => aclRule.organization)
+  public orgId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleDoc extends AclRule {
+
+  @ManyToOne(type => Document, document => document.aclRules)
+  @JoinColumn({name: "doc_id"})
+  public document: Document;
+
+  @RelationId((aclRule: AclRuleDoc) => aclRule.document)
+  public docId: number;
+}

app/gen-server/entity/Alias.ts

@@ -0,0 +1,27 @@
+import {BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne,
+        PrimaryColumn} from 'typeorm';
+import {Document} from './Document';
+import {Organization} from './Organization';
+
+@Entity({name: 'aliases'})
+export class Alias extends BaseEntity {
+  @PrimaryColumn({name: 'org_id'})
+  public orgId: number;
+
+  @PrimaryColumn({name: 'url_id'})
+  public urlId: string;
+
+  @Column({name: 'doc_id'})
+  public docId: string;
+
+  @ManyToOne(type => Document)
+  @JoinColumn({name: 'doc_id'})
+  public doc: Document;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @CreateDateColumn({name: 'created_at'})
+  public createdAt: Date;
+}

app/gen-server/entity/BillingAccount.ts

@@ -0,0 +1,65 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Product} from 'app/gen-server/entity/Product';
+import {nativeValues} from 'app/gen-server/lib/values';
+
+// This type is for billing account status information.  Intended for stuff
+// like "free trial running out in N days".
+interface BillingAccountStatus {
+  stripeStatus?: string;
+  currentPeriodEnd?: Date;
+  message?: string;
+}
+
+/**
+ * This relates organizations to products.  It holds any stripe information
+ * needed to be able to update and pay for the product that applies to the
+ * organization.  It has a list of managers detailing which users have the
+ * right to view and edit these settings.
+ */
+@Entity({name: 'billing_accounts'})
+export class BillingAccount extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Product)
+  @JoinColumn({name: 'product_id'})
+  public product: Product;
+
+  @Column()
+  public individual: boolean;
+
+  // A flag for when all is well with the user's subscription.
+  // Probably shouldn't use this to drive whether service is provided or not.
+  // Strip recommends updating an end-of-service datetime every time payment
+  // is received, adding on a grace period of some days.
+  @Column({name: 'in_good_standing', default: nativeValues.trueValue})
+  public inGoodStanding: boolean;
+
+  @Column({type: nativeValues.jsonEntityType, nullable: true})
+  public status: BillingAccountStatus;
+
+  @Column({name: 'stripe_customer_id', type: String, nullable: true})
+  public stripeCustomerId: string | null;
+
+  @Column({name: 'stripe_subscription_id', type: String, nullable: true})
+  public stripeSubscriptionId: string | null;
+
+  @Column({name: 'stripe_plan_id', type: String, nullable: true})
+  public stripePlanId: string | null;
+
+  @OneToMany(type => BillingAccountManager, manager => manager.billingAccount)
+  public managers: BillingAccountManager[];
+
+  @OneToMany(type => Organization, org => org.billingAccount)
+  public orgs: Organization[];
+
+  // A calculated column that is true if it looks like there is a paid plan.
+  @Column({name: 'paid', type: 'boolean', insert: false, select: false})
+  public paid?: boolean;
+
+  // A calculated column summarizing whether active user is a manager of the billing account.
+  // (No @Column needed since calculation is done in javascript not sql)
+  public isManager?: boolean;
+}

app/gen-server/entity/BillingAccountManager.ts

@@ -0,0 +1,26 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {User} from 'app/gen-server/entity/User';
+
+/**
+ * A list of users with the right to modify a giving billing account.
+ */
+@Entity({name: 'billing_account_managers'})
+export class BillingAccountManager extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Document.ts

@@ -0,0 +1,69 @@
+import {ApiError} from 'app/common/ApiError';
+import {Role} from 'app/common/roles';
+import {DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
+import {AclRuleDoc} from "./AclRule";
+import {Alias} from "./Alias";
+import {Resource} from "./Resource";
+import {Workspace} from "./Workspace";
+
+// Acceptable ids for use in document urls.
+const urlIdRegex = /^[-a-z0-9]+$/i;
+
+function isValidUrlId(urlId: string) {
+  if (urlId === NEW_DOCUMENT_CODE) { return false; }
+  return urlIdRegex.exec(urlId);
+}
+
+@Entity({name: 'docs'})
+export class Document extends Resource {
+
+  @PrimaryColumn()
+  public id: string;
+
+  @ManyToOne(type => Workspace)
+  @JoinColumn({name: 'workspace_id'})
+  public workspace: Workspace;
+
+  @OneToMany(type => AclRuleDoc, aclRule => aclRule.document)
+  public aclRules: AclRuleDoc[];
+
+  // Indicates whether the doc is pinned to the org it lives in.
+  @Column({name: 'is_pinned', default: false})
+  public isPinned: boolean;
+
+  // Property that may be returned when the doc is fetched to indicate the access the
+  // fetching user has on the doc, i.e. 'owners', 'editors', 'viewers'
+  public access: Role|null;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating documents.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false, update: false})
+  public permissions?: any;
+
+  @Column({name: 'url_id', type: 'text', nullable: true})
+  public urlId: string|null;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  @OneToMany(type => Alias, alias => alias.doc)
+  public aliases: Alias[];
+
+  public checkProperties(props: any): props is Partial<DocumentProperties> {
+    return super.checkProperties(props, documentPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<DocumentProperties>) {
+    super.updateFromProperties(props);
+    if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
+    if (props.urlId !== undefined) {
+      if (props.urlId !== null && !isValidUrlId(props.urlId)) {
+        throw new ApiError('invalid urlId', 400);
+      }
+      this.urlId = props.urlId;
+    }
+  }
+}

app/gen-server/entity/Group.ts

@@ -0,0 +1,33 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToOne, PrimaryGeneratedColumn} from "typeorm";
+
+import {AclRule} from "./AclRule";
+import {User} from "./User";
+
+@Entity({name: 'groups'})
+export class Group extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @ManyToMany(type => User)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'user_id'}
+  })
+  public memberUsers: User[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_groups',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'subgroup_id'}
+  })
+  public memberGroups: Group[];
+
+  @OneToOne(type => AclRule, aclRule => aclRule.group)
+  public aclRule: AclRule;
+}

app/gen-server/entity/Login.ts

@@ -0,0 +1,25 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn} from "typeorm";
+
+import {User} from "./User";
+
+@Entity({name: 'logins'})
+export class Login extends BaseEntity {
+
+  @PrimaryColumn()
+  public id: number;
+
+  // This is the normalized email address we use for equality and indexing.
+  @Column()
+  public email: string;
+
+  // This is how the user's email address should be displayed.
+  @Column({name: 'display_email'})
+  public displayEmail: string;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User)
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Organization.ts

@@ -0,0 +1,79 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, OneToOne,
+        PrimaryGeneratedColumn, RelationId} from "typeorm";
+import {Role} from "app/common/roles";
+import {OrganizationProperties, organizationPropertyKeys} from "app/common/UserAPI";
+import {AclRuleOrg} from "./AclRule";
+import {BillingAccount} from "./BillingAccount";
+import {Resource} from "./Resource";
+import {User} from "./User";
+import {Workspace} from "./Workspace";
+
+// Information about how an organization may be accessed.
+export interface AccessOption {
+  id: number;     // a user id
+  email: string;  // a user email
+  name: string;   // a user name
+  perms: number;  // permissions the user would have on organization
+}
+
+export interface AccessOptionWithRole extends AccessOption {
+  access: Role;   // summary of permissions
+}
+
+@Entity({name: 'orgs'})
+export class Organization extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({
+    nullable: true
+  })
+  public domain: string;
+
+  @OneToOne(type => User, user => user.personalOrg)
+  @JoinColumn({name: 'owner_id'})
+  public owner: User;
+
+  @RelationId((org: Organization) => org.owner)
+  public ownerId: number;
+
+  @OneToMany(type => Workspace, workspace => workspace.org)
+  public workspaces: Workspace[];
+
+  @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization)
+  public aclRules: AclRuleOrg[];
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount)
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  // Property that may be returned when the org is fetched to indicate the access the
+  // fetching user has on the org, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // Property that may be used internally to track multiple ways an org can be accessed
+  public accessOptions?: AccessOptionWithRole[];
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating organizations.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  // For custom domains, this is the preferred host associated with this org/team.
+  @Column({name: 'host', type: 'text', nullable: true})
+  public host: string|null;
+
+  public checkProperties(props: any): props is Partial<OrganizationProperties> {
+    return super.checkProperties(props, organizationPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<OrganizationProperties>) {
+    super.updateFromProperties(props);
+    if (props.domain) { this.domain = props.domain; }
+  }
+}

app/gen-server/entity/Product.ts

@@ -0,0 +1,176 @@
+import {Features} from 'app/common/Features';
+import {nativeValues} from 'app/gen-server/lib/values';
+import * as assert from 'assert';
+import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm';
+
+/**
+ * A summary of features used in 'starter' plans.
+ */
+export const starterFeatures: Features = {
+  workspaces: true,
+  // no vanity domain
+  maxDocsPerOrg: 10,
+  maxSharesPerDoc: 2,
+  maxWorkspacesPerOrg: 1
+};
+
+/**
+ * A summary of features used in 'team' plans.
+ */
+export const teamFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  maxSharesPerWorkspace: 0,   // all workspace shares need to be org members.
+  maxSharesPerDoc: 2
+};
+
+/**
+ * A summary of features used in unrestricted grandfathered accounts, and also
+ * in some test settings.
+ */
+export const grandfatherFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+};
+
+export const suspendedFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  readOnlyDocs: true,
+  // clamp down on new docs/workspaces/shares
+  maxDocsPerOrg: 0,
+  maxSharesPerDoc: 0,
+  maxWorkspacesPerOrg: 0,
+};
+
+/**
+ * Basic fields needed for products supported by Grist.
+ */
+export interface IProduct {
+  name: string;
+  features: Features;
+}
+
+/**
+ *
+ * Products are a bundle of enabled features.  Most products in
+ * Grist correspond to products in stripe.  The correspondence is
+ * established by a gristProduct metadata field on stripe plans.
+ *
+ * In addition, there are the following products in Grist that don't
+ * exist in stripe:
+ *   - The product named 'Free'.  This is a product used for organizations
+ *     created prior to the billing system being set up.
+ *   - The product named 'stub'.  This is product assigned to new
+ *     organizations that should not be usable until a paid plan
+ *     is set up for them.
+ *
+ * TODO: change capitalization of name of grandfather product.
+ *
+ */
+const PRODUCTS: IProduct[] = [
+  // This is a product for grandfathered accounts/orgs.
+  {
+    name: 'Free',
+    features: grandfatherFeatures,
+  },
+
+  // This is a product for newly created accounts/orgs.
+  {
+    name: 'stub',
+    features: {},
+  },
+
+  // These are products set up in stripe.
+  {
+    name: 'starter',
+    features: starterFeatures,
+  },
+  {
+    name: 'professional',  // deprecated, can be removed once no longer referred to in stripe.
+    features: teamFeatures,
+  },
+  {
+    name: 'team',
+    features: teamFeatures,
+  },
+
+  // This is a product for a team site that is no longer in good standing, but isn't yet
+  // to be removed / deactivated entirely.
+  {
+    name: 'suspended',
+    features: suspendedFeatures,
+  },
+];
+
+/**
+ * Get names of products for different situations.
+ */
+export function getDefaultProductNames() {
+  return {
+    personal: 'starter',  // Personal site start off on a functional plan.
+    teamInitial: 'stub',  // Team site starts off on a limited plan, requiring subscription.
+    team: 'team',         // Functional team site
+  };
+}
+
+/**
+ * A Grist product.  Corresponds to a set of enabled features and a choice of limits.
+ */
+@Entity({name: 'products'})
+export class Product extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({type: nativeValues.jsonEntityType})
+  public features: Features;
+}
+
+/**
+ * Make sure the products defined for the current stripe setup are
+ * in the database and up to date.  Other products in the database
+ * are untouched.
+ *
+ * If `apply` is set, the products are changed in the db, otherwise
+ * the are left unchanged.  A summary of affected products is returned.
+ */
+export async function synchronizeProducts(connection: Connection, apply: boolean): Promise<string[]> {
+  try {
+    await connection.query('select name, features, stripe_product_id from products limit 1');
+  } catch (e) {
+    // No usable products table, do not try to synchronize.
+    return [];
+  }
+  const changingProducts: string[] = [];
+  await connection.transaction(async transaction => {
+    const desiredProducts = new Map(PRODUCTS.map(p => [p.name, p]));
+    const existingProducts = new Map((await transaction.find(Product))
+                                     .map(p => [p.name, p]));
+    for (const product of desiredProducts.values()) {
+      if (existingProducts.has(product.name)) {
+        const p = existingProducts.get(product.name)!;
+        try {
+          assert.deepStrictEqual(p.features, product.features);
+        } catch (e) {
+          if (apply) {
+            p.features = product.features;
+            await transaction.save(p);
+          }
+          changingProducts.push(p.name);
+        }
+      } else {
+        if (apply) {
+          const p = new Product();
+          p.name = product.name;
+          p.features = product.features;
+          await transaction.save(p);
+        }
+        changingProducts.push(product.name);
+      }
+    }
+  });
+  return changingProducts;
+}

app/gen-server/entity/Resource.ts

@@ -0,0 +1,46 @@
+import {BaseEntity, Column} from "typeorm";
+import {ApiError} from 'app/common/ApiError';
+import {CommonProperties} from "app/common/UserAPI";
+
+export class Resource extends BaseEntity {
+  @Column()
+  public name: string;
+
+  @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"})
+  public createdAt: Date;
+
+  @Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"})
+  public updatedAt: Date;
+
+  // a computed column which, when present, means the entity should be filtered out
+  // of results.
+  @Column({name: 'filtered_out', type: 'boolean', select: false, insert: false})
+  public filteredOut?: boolean;
+
+  public updateFromProperties(props: Partial<CommonProperties>) {
+    if (props.createdAt) { this.createdAt = _propertyToDate(props.createdAt); }
+    if (props.updatedAt) {
+      this.updatedAt = _propertyToDate(props.updatedAt);
+    } else {
+      this.updatedAt = new Date();
+    }
+    if (props.name) { this.name = props.name; }
+  }
+
+  protected checkProperties(props: any, keys: string[]): props is Partial<CommonProperties> {
+    for (const key of Object.keys(props)) {
+      if (!keys.includes(key)) {
+        throw new ApiError(`unrecognized property ${key}`, 400);
+      }
+    }
+    return true;
+  }
+}
+
+// Ensure iso-string-or-date value is converted to a date.
+function _propertyToDate(d: string|Date): Date {
+  if (typeof(d) === 'string') {
+    return new Date(d);
+  }
+  return d;
+}

app/gen-server/entity/User.ts

@@ -0,0 +1,54 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToMany, OneToOne,
+        PrimaryGeneratedColumn} from "typeorm";
+
+import {Group} from "./Group";
+import {Login} from "./Login";
+import {Organization} from "./Organization";
+
+@Entity({name: 'users'})
+export class User extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({name: 'api_key', type: String, nullable: true})
+  // Found how to make a type nullable in this discussion: https://github.com/typeorm/typeorm/issues/2567
+  // todo: adds constraint for api_key not to equal ''
+  public apiKey: string | null;
+
+  @Column({name: 'picture', type: String, nullable: true})
+  public picture: string | null;
+
+  @Column({name: 'first_login_at', type: Date, nullable: true})
+  public firstLoginAt: Date | null;
+
+  @OneToOne(type => Organization, organization => organization.owner)
+  public personalOrg: Organization;
+
+  @OneToMany(type => Login, login => login.user)
+  public logins: Login[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'user_id'},
+    inverseJoinColumn: {name: 'group_id'}
+  })
+  public groups: Group[];
+
+  @Column({name: 'is_first_time_user', default: false})
+  public isFirstTimeUser: boolean;
+
+  /**
+   * Get user's email.  Returns undefined if logins has not been joined, or no login
+   * is available
+   */
+  public get loginEmail(): string|undefined {
+    const login = this.logins && this.logins[0];
+    if (!login) { return undefined; }
+    return login.email;
+  }
+}

app/gen-server/entity/Workspace.ts

@@ -0,0 +1,49 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from "typeorm";
+import {WorkspaceProperties, workspacePropertyKeys} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {AclRuleWs} from "./AclRule";
+import {Document} from "./Document";
+import {Organization} from "./Organization";
+import {Resource} from "./Resource";
+
+@Entity({name: 'workspaces'})
+export class Workspace extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @OneToMany(type => Document, document => document.workspace)
+  public docs: Document[];
+
+  @OneToMany(type => AclRuleWs, aclRule => aclRule.workspace)
+  public aclRules: AclRuleWs[];
+
+  // Property that may be returned when the workspace is fetched to indicate the access the
+  // fetching user has on the workspace, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // A computed column that is true if the workspace is a support workspace.
+  @Column({name: 'support', type: 'boolean', insert: false, select: false})
+  public isSupportWorkspace?: boolean;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating workspaces.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  public checkProperties(props: any): props is Partial<WorkspaceProperties> {
+    return super.checkProperties(props, workspacePropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<WorkspaceProperties>) {
+    super.updateFromProperties(props);
+  }
+}

app/gen-server/lib/DocApiForwarder.ts

@@ -0,0 +1,103 @@
+import * as express from "express";
+import fetch, { RequestInit } from 'node-fetch';
+
+import { ApiError } from 'app/common/ApiError';
+import { removeTrailingSlash } from 'app/common/gutil';
+import { HomeDBManager } from "app/gen-server/lib/HomeDBManager";
+import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer';
+import { IDocWorkerMap } from "app/server/lib/DocWorkerMap";
+import { expressWrap } from "app/server/lib/expressWrap";
+import { getAssignmentId } from "app/server/lib/idUtils";
+
+/**
+ * Forwards all /api/docs/:docId/tables requests to the doc worker handling the :docId document. Makes
+ * sure the user has at least view access to the document otherwise rejects the request. For
+ * performance reason we stream the body directly from the request, which requires that no-one reads
+ * the req before, in particular you should register DocApiForwarder before bodyParser.
+ *
+ * Use:
+ *   const home = new ApiServer(false);
+ *   const docApiForwarder = new DocApiForwarder(getDocWorkerMap(), home);
+ *   app.use(docApiForwarder.getMiddleware());
+ *
+ * Note that it expects userId, and jsonErrorHandler middleware to be set up outside
+ * to apply to these routes.
+ */
+export class DocApiForwarder {
+
+  constructor(private _docWorkerMap: IDocWorkerMap, private _dbManager: HomeDBManager) {
+  }
+
+  public addEndpoints(app: express.Application) {
+    // Middleware to forward a request about an existing document that user has access to.
+    // We do not check whether the document has been soft-deleted; that will be checked by
+    // the worker if needed.
+    const withDoc = expressWrap(this._forwardToDocWorker.bind(this, true));
+    // Middleware to forward a request without a pre-existing document (for imports/uploads).
+    const withoutDoc = expressWrap(this._forwardToDocWorker.bind(this, false));
+    app.use('/api/docs/:docId/tables', withDoc);
+    app.use('/api/docs/:docId/force-reload', withDoc);
+    app.use('/api/docs/:docId/remove', withDoc);
+    app.delete('/api/docs/:docId', withDoc);
+    app.use('/api/docs/:docId/download', withDoc);
+    app.use('/api/docs/:docId/apply', withDoc);
+    app.use('/api/docs/:docId/attachments', withDoc);
+    app.use('/api/docs/:docId/snapshots', withDoc);
+    app.use('/api/docs/:docId/replace', withDoc);
+    app.use('/api/docs/:docId/flush', withDoc);
+    app.use('/api/docs/:docId/states', withDoc);
+    app.use('/api/docs/:docId/compare', withDoc);
+    app.use('^/api/docs$', withoutDoc);
+  }
+
+  private async _forwardToDocWorker(withDocId: boolean, req: express.Request, res: express.Response): Promise<void> {
+    let docId: string|null = null;
+    if (withDocId) {
+      const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, req.params.docId);
+      assertAccess('viewers', docAuth, {allowRemoved: true});
+      docId = docAuth.docId;
+    }
+    // Use the docId for worker assignment, rather than req.params.docId, which could be a urlId.
+    const assignmentId = getAssignmentId(this._docWorkerMap, docId === null ? 'import' : docId);
+
+    if (!this._docWorkerMap) {
+      throw new ApiError('no worker map', 404);
+    }
+    const docStatus = await this._docWorkerMap.assignDocWorker(assignmentId);
+
+    // Construct new url by keeping only origin and path prefixes of `docWorker.internalUrl`,
+    // and otherwise reflecting fully the original url (remaining path, and query params).
+    const docWorkerUrl = new URL(docStatus.docWorker.internalUrl);
+    const url = new URL(req.originalUrl, docWorkerUrl.origin);
+    url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
+
+    const headers: {[key: string]: string} = {
+      ...getTransitiveHeaders(req),
+      'Content-Type': req.get('Content-Type') || 'application/json',
+    };
+    for (const key of ['X-Sort', 'X-Limit']) {
+      const hdr = req.get(key);
+      if (hdr) { headers[key] = hdr; }
+    }
+    const options: RequestInit = {
+      method: req.method,
+      headers,
+    };
+    if (['POST', 'PATCH'].includes(req.method)) {
+      // uses `req` as a stream
+      options.body = req;
+    }
+    const docWorkerRes = await fetch(url.href, options);
+    res.status(docWorkerRes.status);
+    for (const key of ['content-type', 'content-disposition', 'cache-control']) {
+      const value = docWorkerRes.headers.get(key);
+      if (value) { res.set(key, value); }
+    }
+    return new Promise<void>((resolve, reject) => {
+      docWorkerRes.body.on('error', reject);
+      res.on('error', reject);
+      res.on('finish', resolve);
+      docWorkerRes.body.pipe(res);
+    });
+  }
+}

app/gen-server/lib/DocWorkerMap.ts

@@ -0,0 +1,440 @@
+import {MapWithTTL} from 'app/common/AsyncCreate';
+import * as version from 'app/common/version';
+import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
+import * as log from 'app/server/lib/log';
+import {checkPermitKey, formatPermitKey, Permit} from 'app/server/lib/Permit';
+import {promisifyAll} from 'bluebird';
+import mapValues = require('lodash/mapValues');
+import {createClient, Multi, RedisClient} from 'redis';
+import * as Redlock from 'redlock';
+import * as uuidv4 from 'uuid/v4';
+
+promisifyAll(RedisClient.prototype);
+promisifyAll(Multi.prototype);
+
+// Max time for which we will hold a lock, by default.  In milliseconds.
+const LOCK_TIMEOUT = 3000;
+
+// How long do checksums stored in redis last.  In milliseconds.
+// Should be long enough to allow S3 to reach consistency with very high probability.
+// Consistency failures shorter than this interval will be detectable, failures longer
+// than this interval will not be detectable.
+const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000;  // 24 hours
+
+// How long do permits stored in redis last, in milliseconds.
+const PERMIT_TTL_MSEC = 1 * 60 * 1000;  // 1 minute
+
+class DummyDocWorkerMap implements IDocWorkerMap {
+  private _worker?: DocWorkerInfo;
+  private _available: boolean = false;
+  private _permits = new MapWithTTL<string, string>(PERMIT_TTL_MSEC);
+  private _elections = new MapWithTTL<string, string>(1);  // default ttl never used
+
+  public async getDocWorker(docId: string) {
+    if (!this._worker) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async assignDocWorker(docId: string) {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    if (this._worker.id !== workerId) { throw new Error('worker not known'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async updateDocStatus(docId: string, checksum: string) {
+    // nothing to do
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    this._worker = info;
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    this._worker = undefined;
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    this._available = available;
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    // nothing to do
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return [];
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    this._permits.set(key, JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit> {
+    const result = this._permits.get(key);
+    return result ? JSON.parse(result) : null;
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    this._permits.delete(key);
+  }
+
+  public async close(): Promise<void> {
+    this._permits.clear();
+    this._elections.clear();
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    if (this._elections.get(name)) { return null; }
+    const key = uuidv4();
+    this._elections.setWithCustomTTL(name, key, durationInMs);
+    return key;
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    if (this._elections.get(name) === electionKey) {
+      this._elections.delete(name);
+    }
+  }
+}
+
+/**
+ * Manage the relationship between document and workers.  Backed by Redis.
+ * Can also assign workers to "groups" for serving particular documents.
+ * Keys used:
+ *   workers - the set of known active workers, identified by workerId
+ *   workers-available - the set of workers available for assignment (a subset of the workers set)
+ *   workers-available-{group} - the set of workers available for a given group
+ *   worker-{workerId} - a hash of contact information for a worker
+ *   worker-{workerId}-docs - a set of docs assigned to a worker, identified by docId
+ *   worker-{workerId}-group - if set, marks the worker as serving a particular group
+ *   doc-${docId} - a hash containing (JSON serialized) DocStatus fields, other than docMD5.
+ *   doc-${docId}-checksum - the docs docMD5, or 'null' if docMD5 is null
+ *   doc-${docId}-group - if set, marks the doc as to be served by workers in a given group
+ *   workers-lock - a lock used when working with the list of workers
+ *   groups - a hash from groupIds (arbitrary strings) to desired number of workers in group
+ *   elections-${deployment} - a hash, from groupId to a (serialized json) list of worker ids
+ *
+ * Assignments of documents to workers can end abruptly at any time.  Clients
+ * should be prepared to retry if a worker is not responding or denies that a document
+ * is assigned to it.
+ *
+ * If the groups key is set, workers assign themselves to groupIds to
+ * fill the counts specified in groups (in order of groupIds), and
+ * once those are exhausted, get assigned to the special group
+ * "default".
+ */
+export class DocWorkerMap implements IDocWorkerMap {
+  private _client: RedisClient;
+  private _redlock: Redlock;
+
+  // Optional deploymentKey argument supplies a key unique to the deployment (this is important
+  // for maintaining groups across redeployments only)
+  constructor(_clients?: RedisClient[], private _deploymentKey?: string, private _options?: {
+    permitMsec?: number
+  }) {
+    this._deploymentKey = this._deploymentKey || version.version;
+    _clients = _clients || [createClient(process.env.REDIS_URL)];
+    this._redlock = new Redlock(_clients);
+    this._client = _clients[0]!;
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    log.info(`DocWorkerMap.addWorker ${info.id}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Make a worker-{workerId} key with contact info, then add this worker to available set.
+      await this._client.hmsetAsync(`worker-${info.id}`, info);
+      await this._client.saddAsync('workers', info.id);
+      // Figure out if worker should belong to a group
+      const groups = await this._client.hgetallAsync('groups');
+      if (groups) {
+        const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`) || {};
+        for (const group of Object.keys(groups).sort()) {
+          const count = parseInt(groups[group], 10) || 0;
+          if (count < 1) { continue; }
+          const elected: string[] = JSON.parse(elections[group] || '[]');
+          if (elected.length >= count) { continue; }
+          elected.push(info.id);
+          await this._client.setAsync(`worker-${info.id}-group`, group);
+          await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, JSON.stringify(elected));
+          break;
+        }
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    log.info(`DocWorkerMap.removeWorker ${workerId}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Drop out of available set first.
+      await this._client.sremAsync('workers-available', workerId);
+      const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+      // At this point, this worker should no longer be receiving new doc assignments, though
+      // clients may still be directed to the worker.
+
+      // If we were elected for anything, back out.
+      const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`);
+      if (elections) {
+        if (group in elections) {
+          const elected: string[] = JSON.parse(elections[group]);
+          const newElected = elected.filter(worker => worker !== workerId);
+          if (elected.length !== newElected.length) {
+            if (newElected.length > 0) {
+              await this._client.hsetAsync(`elections-${this._deploymentKey}`, group,
+                                           JSON.stringify(newElected));
+            } else {
+              await this._client.hdelAsync(`elections-${this._deploymentKey}`, group);
+              delete elections[group];
+            }
+          }
+          // We're the last one involved in elections - remove the key entirely.
+          if (Object.keys(elected).length === 0) {
+            await this._client.delAsync(`elections-${this._deploymentKey}`);
+          }
+        }
+      }
+
+      // Now, we start removing the assignments.
+      const assignments = await this._client.smembersAsync(`worker-${workerId}-docs`);
+      if (assignments) {
+        const op = this._client.multi();
+        for (const doc of assignments) { op.del(`doc-${doc}`); }
+        await op.execAsync();
+      }
+
+      // Now remove worker-{workerId}* keys.
+      await this._client.delAsync(`worker-${workerId}-docs`);
+      await this._client.delAsync(`worker-${workerId}-group`);
+      await this._client.delAsync(`worker-${workerId}`);
+
+      // Forget about this worker completely.
+      await this._client.sremAsync('workers', workerId);
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
+    const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+    if (available) {
+      await this._client.saddAsync(`workers-available-${group}`, workerId);
+      await this._client.saddAsync('workers-available', workerId);
+    } else {
+      await this._client.sremAsync('workers-available', workerId);
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+    }
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    const op = this._client.multi();
+    op.del(`doc-${docId}`);
+    op.srem(`worker-${workerId}-docs`, docId);
+    await op.execAsync();
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return this._client.smembersAsync(`worker-${workerId}-docs`);
+  }
+
+  /**
+   * Defined by IDocWorkerMap.
+   *
+   * Looks up which DocWorker is responsible for this docId.
+   * Responsibility could change at any time after this call, so it
+   * should be treated as a hint, and clients should be prepared to be
+   * refused and need to retry.
+   */
+  public async getDocWorker(docId: string): Promise<DocStatus|null> {
+    // Fetch the various elements that go into making a DocStatus
+    const props = await this._client.multi()
+      .hgetall(`doc-${docId}`)
+      .get(`doc-${docId}-checksum`)
+      .execAsync() as [{[key: string]: any}|null, string|null]|null;
+    if (!props) { return null; }
+
+    // If there is no worker, return null.  An alternative would be to modify
+    // DocStatus so that it is possible for it to not have a worker assignment.
+    if (!props[0]) { return null; }
+
+    // Fields are JSON encoded since redis cannot store them directly.
+    const doc = mapValues(props[0], (val) => JSON.parse(val));
+
+    // Redis cannot store a null value, so we encode it as 'null', which does
+    // not match any possible MD5.
+    doc.docMD5 = props[1] === 'null' ? null : props[1];
+
+    // Ok, we have a valid DocStatus at this point.
+    return doc as DocStatus;
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a DocWorker to this docId if one is not yet assigned.
+   * Note that the assignment could be unmade at any time after this
+   * call if the worker dies, is brought down, or for other potential
+   * reasons in the future such as migration of individual documents
+   * between workers.
+   *
+   * A preferred doc worker can be specified, which will be assigned
+   * if no assignment is already made.
+   *
+   */
+  public async assignDocWorker(docId: string, workerId?: string): Promise<DocStatus> {
+    // Check if a DocWorker is already assigned; if so return result immediately
+    // without locking.
+    let docStatus = await this.getDocWorker(docId);
+    if (docStatus) { return docStatus; }
+
+    // No assignment yet, so let's lock and set an assignment up.
+    const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
+
+    try {
+      // Now that we've locked, recheck that the worker hasn't been reassigned
+      // in the meantime.  Return immediately if it has.
+      docStatus = await this.getDocWorker(docId);
+      if (docStatus) { return docStatus; }
+
+      if (!workerId) {
+        // Check if document has a preferred worker group set.
+        const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
+
+        // Let's start off by assigning documents to available workers randomly.
+        // TODO: use a smarter algorithm.
+        workerId = await this._client.srandmemberAsync(`workers-available-${group}`) || undefined;
+        if (!workerId) {
+          // No workers available in the desired worker group.  Rather than refusing to
+          // open the document, we fall back on assigning a worker from any of the workers
+          // available, regardless of grouping.
+          // This limits the impact of operational misconfiguration (bad redis setup,
+          // or not starting enough workers).  It has the downside of potentially disguising
+          // problems, so we log a warning.
+          log.warn(`DocWorkerMap.assignDocWorker ${docId} found no workers for group ${group}`);
+          workerId = await this._client.srandmemberAsync('workers-available') || undefined;
+        }
+        if (!workerId) { throw new Error('no doc workers available'); }
+      }  else {
+        if (!await this._client.sismemberAsync('workers-available', workerId)) {
+          throw new Error(`worker ${workerId} not known or not available`);
+        }
+      }
+
+      // Look up how to contact the worker.
+      const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
+      if (!docWorker) { throw new Error('no doc worker contact info available'); }
+
+      // We can now construct a DocStatus.
+      const newDocStatus = {docMD5: null, docWorker, isActive: true};
+
+      // We add the assignment to worker-{workerId}-docs and save doc-{docId}.
+      const result = await this._client.multi()
+        .sadd(`worker-${workerId}-docs`, docId)
+        .hmset(`doc-${docId}`, {
+          docWorker: JSON.stringify(docWorker),  // redis can't store nested objects, strings only
+          isActive: JSON.stringify(true)         // redis can't store booleans, strings only
+        })
+        .setex(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, 'null')
+        .execAsync();
+      if (!result) { throw new Error('failed to store new assignment'); }
+      return newDocStatus;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a specific DocWorker to this docId if one is not yet assigned.
+   *
+   */
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    return this.assignDocWorker(docId, workerId);
+  }
+
+  public async updateDocStatus(docId: string, checksum: string): Promise<void> {
+    await this._client.setexAsync(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, checksum);
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    const duration = (this._options && this._options.permitMsec) || PERMIT_TTL_MSEC;
+      // seems like only integer seconds are supported?
+    await this._client.setexAsync(key, Math.ceil(duration / 1000.0),
+                                  JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit|null> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    const result = await this._client.getAsync(key);
+    return result && JSON.parse(result);
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    await this._client.delAsync(key);
+  }
+
+  public async close(): Promise<void> {
+    // nothing to do
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    // Could use "set nx" for election, but redis docs don't encourage that any more,
+    // favoring redlock:
+    //   https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      if (await this._client.getAsync(redisKey) !== null) { return null; }
+      const electionKey = uuidv4();
+      // seems like only integer seconds are supported?
+      await this._client.setexAsync(redisKey, Math.ceil(durationInMs / 1000.0), electionKey);
+      return electionKey;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      const current = await this._client.getAsync(redisKey);
+      if (current === electionKey) {
+        await this._client.delAsync(redisKey);
+      } else if (current !== null) {
+        throw new Error('could not remove election');
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+}
+
+// If we don't have redis available and use a DummyDocWorker, it should be a singleton.
+let dummyDocWorkerMap: DummyDocWorkerMap|null = null;
+
+export function getDocWorkerMap(): IDocWorkerMap {
+  if (process.env.REDIS_URL) {
+    return new DocWorkerMap();
+  } else {
+    dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap();
+    return dummyDocWorkerMap;
+  }
+}

app/gen-server/lib/Permissions.ts

@@ -0,0 +1,21 @@
+export enum Permissions {
+  NONE          = 0x0,
+  // Note that the view permission bit provides view access ONLY to the resource to which
+  // the aclRule belongs - it does not allow listing that resource's children. A resource's
+  // children may only be listed if those children also have the view permission set.
+  VIEW          = 0x1,
+  UPDATE        = 0x2,
+  ADD           = 0x4,
+  // Note that the remove permission bit provides remove access to a resource AND all of
+  // its child resources/ACLs
+  REMOVE        = 0x8,
+  SCHEMA_EDIT   = 0x10,
+  ACL_EDIT      = 0x20,
+  EDITOR        = VIEW | UPDATE | ADD | REMOVE,   // tslint:disable-line:no-bitwise
+  ADMIN         = EDITOR | SCHEMA_EDIT,           // tslint:disable-line:no-bitwise
+  OWNER         = ADMIN | ACL_EDIT,               // tslint:disable-line:no-bitwise
+
+  // A virtual permission bit signifying that the general public has some access to
+  // the resource via ACLs involving the everyone@ user.
+  PUBLIC        = 0x80
+}

app/gen-server/lib/TypeORMPatches.ts

@@ -0,0 +1,196 @@
+// This contains two TypeORM patches.
+
+// Patch 1:
+// TypeORM Sqlite driver does not support using transactions in async code, if it is possible
+// for two transactions to get called (one of the whole point of transactions).  This
+// patch adds support for that, based on a monkey patch published in:
+//   https://gist.github.com/keenondrums/556f8c61d752eff730841170cd2bc3f1
+// Explanation at https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213
+
+// Patch 2:
+// TypeORM parameters are global, and collisions in setting them are not detected.
+// We add a patch to throw an exception if a parameter value is ever set and then
+// changed during construction of a query.
+
+import * as sqlite3 from '@gristlabs/sqlite3';
+import isEqual = require('lodash/isEqual');
+import {EntityManager, QueryRunner} from 'typeorm';
+import {SqliteDriver} from 'typeorm/driver/sqlite/SqliteDriver';
+import {SqliteQueryRunner} from 'typeorm/driver/sqlite/SqliteQueryRunner';
+import {
+  QueryRunnerProviderAlreadyReleasedError
+} from 'typeorm/error/QueryRunnerProviderAlreadyReleasedError';
+import {QueryBuilder} from 'typeorm/query-builder/QueryBuilder';
+
+
+/**********************
+ * Patch 1
+ **********************/
+
+type Releaser = () => void;
+type Worker<T> = () => Promise<T>|T;
+
+interface MutexInterface {
+  acquire(): Promise<Releaser>;
+  runExclusive<T>(callback: Worker<T>): Promise<T>;
+  isLocked(): boolean;
+}
+
+class Mutex implements MutexInterface {
+  private _queue: Array<(release: Releaser) => void> = [];
+  private _pending = false;
+
+  public isLocked(): boolean {
+    return this._pending;
+  }
+
+  public acquire(): Promise<Releaser> {
+    const ticket = new Promise<Releaser>(resolve => this._queue.push(resolve));
+    if (!this._pending) {
+      this._dispatchNext();
+    }
+    return ticket;
+  }
+
+  public runExclusive<T>(callback: Worker<T>): Promise<T> {
+    return this
+      .acquire()
+      .then(release => {
+        let result: T|Promise<T>;
+
+        try {
+          result = callback();
+        } catch (e) {
+          release();
+          throw(e);
+        }
+
+        return Promise
+          .resolve(result)
+          .then(
+            (x: T) => (release(), x),
+            e => {
+              release();
+              throw e;
+            }
+          );
+      }
+           );
+  }
+
+  private _dispatchNext(): void {
+    if (this._queue.length > 0) {
+      this._pending = true;
+      this._queue.shift()!(this._dispatchNext.bind(this));
+    } else {
+      this._pending = false;
+    }
+  }
+
+}
+
+// A singleton mutex for all sqlite transactions.
+const mutex = new Mutex();
+
+class SqliteQueryRunnerPatched extends SqliteQueryRunner {
+  private _releaseMutex: Releaser | null;
+
+  public async startTransaction(level?: any): Promise<void> {
+    this._releaseMutex = await mutex.acquire();
+    return super.startTransaction(level);
+  }
+
+  public async commitTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.commitTransaction -> mutex releaser unknown');
+    }
+    await super.commitTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async rollbackTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.rollbackTransaction -> mutex releaser unknown');
+    }
+    await super.rollbackTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async connect(): Promise<any> {
+    if (!this.isTransactionActive) {
+      const release = await mutex.acquire();
+      release();
+    }
+    return super.connect();
+  }
+}
+
+class SqliteDriverPatched extends SqliteDriver {
+  public createQueryRunner(): QueryRunner {
+    if (!this.queryRunner) {
+      this.queryRunner = new SqliteQueryRunnerPatched(this);
+    }
+    return this.queryRunner;
+  }
+  protected loadDependencies(): void {
+    // Use our own sqlite3 module, which is a fork of the original.
+    this.sqlite = sqlite3;
+  }
+}
+
+// Patch the underlying SqliteDriver, since it's impossible to convince typeorm to use only our
+// patched classes. (Previously we patched DriverFactory and Connection, but those would still
+// create an unpatched SqliteDriver and then overwrite it.)
+SqliteDriver.prototype.createQueryRunner = SqliteDriverPatched.prototype.createQueryRunner;
+(SqliteDriver.prototype as any).loadDependencies = (SqliteDriverPatched.prototype as any).loadDependencies;
+
+export function applyPatch() {
+  // tslint: disable-next-line
+  EntityManager.prototype.transaction = async function <T>(arg1: any,  arg2?: any): Promise<T> {
+    if (this.queryRunner && this.queryRunner.isReleased) {
+      throw new QueryRunnerProviderAlreadyReleasedError();
+    }
+    if (this.queryRunner && this.queryRunner.isTransactionActive) {
+      throw new Error(`Cannot start transaction because its already started`);
+    }
+    const queryRunner = this.connection.createQueryRunner();
+    const runInTransaction = typeof arg1 === "function" ? arg1 : arg2;
+    try {
+      await queryRunner.startTransaction();
+      const result = await runInTransaction(queryRunner.manager);
+      await queryRunner.commitTransaction();
+      return result;
+    } catch (err) {
+      try {
+        // we throw original error even if rollback thrown an error
+        await queryRunner.rollbackTransaction();
+        // tslint: disable-next-line
+      } catch (rollbackError) {
+        // tslint: disable-next-line
+      }
+      throw err;
+    } finally {
+      await queryRunner.release();
+    }
+  };
+}
+
+
+/**********************
+ * Patch 2
+ **********************/
+
+abstract class QueryBuilderPatched<T> extends QueryBuilder<T> {
+  public setParameter(key: string, value: any): this {
+    const prev = this.expressionMap.parameters[key];
+    if (prev !== undefined && !isEqual(prev, value)) {
+      throw new Error(`TypeORM parameter collision for key '${key}' ('${prev}' vs '${value}')`);
+    }
+    this.expressionMap.parameters[key] = value;
+    return this;
+  }
+}
+
+(QueryBuilder.prototype as any).setParameter = (QueryBuilderPatched.prototype as any).setParameter;

app/gen-server/lib/Usage.ts

@@ -0,0 +1,62 @@
+import {Document} from 'app/gen-server/entity/Document';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {User} from 'app/gen-server/entity/User';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import * as log from 'app/server/lib/log';
+
+// Frequency of logging usage information.  Not something we need
+// to track with much granularity.
+const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000;   // log every 1 hour
+
+/**
+ * Occasionally log usage information - number of users, orgs,
+ * docs, etc.
+ */
+export class Usage {
+  private _interval: NodeJS.Timeout;
+
+  public constructor(private _dbManager: HomeDBManager) {
+    this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS);
+    // Log once at beginning, in case we roll over servers faster than
+    // the logging period for an extended length of time,
+    // and to raise the visibility of this logging step so if it gets
+    // slow devs notice.
+    this.apply().catch(log.warn.bind(log));
+  }
+
+  public close() {
+    clearInterval(this._interval);
+  }
+
+  public async apply() {
+    const manager = this._dbManager.connection.manager;
+    // raw count of users
+    const userCount = await manager.count(User);
+    // users who have logged in at least once
+    const userWithLoginCount = await manager.createQueryBuilder()
+      .from(User, 'users')
+      .where('first_login_at is not null')
+      .getCount();
+    // raw count of organizations (excluding personal orgs)
+    const orgCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .where('owner_id is null')
+      .getCount();
+    // organizations with subscriptions that are in a non-terminated state
+    const orgInGoodStandingCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .where('owner_id is null')
+      .andWhere('billing_accounts.in_good_standing = true')
+      .getCount();
+    // raw count of documents
+    const docCount = await manager.count(Document);
+    log.rawInfo('activity', {
+      docCount,
+      orgCount,
+      orgInGoodStandingCount,
+      userCount,
+      userWithLoginCount,
+    });
+  }
+}

app/gen-server/lib/scrubUserFromOrg.ts

@@ -0,0 +1,209 @@
+import {EntityManager} from "typeorm";
+import * as roles from 'app/common/roles';
+import {Document} from "app/gen-server/entity/Document";
+import {Group} from "app/gen-server/entity/Group";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Workspace} from "app/gen-server/entity/Workspace";
+import pick = require('lodash/pick');
+
+/**
+ *
+ * Remove the given user from the given org and every resource inside the org.
+ * If the user being removed is an owner of any resources in the org, the caller replaces
+ * them as the owner. This is to prevent complete loss of access to any resource.
+ *
+ * This method transforms ownership without regard to permissions.  We all talked this
+ * over and decided this is what we wanted, but there's no denying it is funky and could
+ * be surprising.
+ * TODO: revisit user scrubbing when we can.
+ *
+ */
+export async function scrubUserFromOrg(
+  orgId: number,
+  removeUserId: number,
+  callerUserId: number,
+  manager: EntityManager
+): Promise<void> {
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+
+  // This will be a list of all mentions of removeUser and callerUser in any resource
+  // within the org.
+  const mentions: Mention[] = [];
+
+  // Base query for all group_users related to these two users and this org.
+  const q = manager.createQueryBuilder()
+    .select('group_users.group_id, group_users.user_id')
+    .from('group_users', 'group_users')
+    .leftJoin(Group, 'groups', 'group_users.group_id = groups.id')
+    .addSelect('groups.name as name')
+    .leftJoin('groups.aclRule', 'acl_rules')
+    .where('(group_users.user_id = :removeUserId or group_users.user_id = :callerUserId)',
+           {removeUserId, callerUserId})
+    .andWhere('orgs.id = :orgId', {orgId});
+
+  // Pick out group_users related specifically to the org resource, in 'mentions' format
+  // (including resource id, a tag for the kind of resource, the group name, the user
+  // id, and the group id).
+  const orgs = q.clone()
+    .addSelect(`'org' as kind, orgs.id`)
+    .innerJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id');
+  mentions.push(...await orgs.getRawMany());
+
+  // Pick out mentions related to any workspace within the org.
+  const wss = q.clone()
+    .innerJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id')
+    .addSelect(`'ws' as kind, workspaces.id`)
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await wss.getRawMany());
+
+  // Pick out mentions related to any doc within the org.
+  const docs = q.clone()
+    .innerJoin(Document, 'docs', 'docs.id = acl_rules.doc_id')
+    .addSelect(`'doc' as kind, docs.id`)
+    .innerJoin('docs.workspace', 'workspaces')
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await docs.getRawMany());
+
+  // Prepare to add and delete group_users.
+  const toDelete: Mention[] = [];
+  const toAdd: Mention[] = [];
+
+  // Now index the mentions by whether they are for the removeUser or the callerUser,
+  // and the resource they apply to.
+  const removeUserMentions = new Map<MentionKey, Mention>();
+  const callerUserMentions = new Map<MentionKey, Mention>();
+  for (const mention of mentions) {
+    const isGuest = mention.name === roles.GUEST;
+    if (mention.user_id === removeUserId) {
+      // We can safely remove any guest roles for the removeUser without any
+      // further inspection.
+      if (isGuest) { toDelete.push(mention); continue; }
+      removeUserMentions.set(getMentionKey(mention), mention);
+    } else {
+      if (isGuest) { continue; }
+      callerUserMentions.set(getMentionKey(mention), mention);
+    }
+  }
+  // Now iterate across the mentions of removeUser, and see what we need to do
+  // for each of them.
+  for (const [key, removeUserMention] of removeUserMentions) {
+    toDelete.push(removeUserMention);
+    if (removeUserMention.name !== roles.OWNER) {
+      // Nothing fancy needed for cases where the removeUser is not the owner.
+      // Just discard those.
+      continue;
+    }
+    // The removeUser was a direct owner on this resource, but the callerUser was
+    // not.  We set the callerUser as a direct owner on this resource, to preserve
+    // access to it.
+    // TODO: the callerUser might inherit sufficient access, in which case this
+    // step is unnecessary and could be skipped.  I believe it does no harm though.
+    const callerUserMention = callerUserMentions.get(key);
+    if (callerUserMention && callerUserMention.name === roles.OWNER) { continue; }
+    if (callerUserMention) { toDelete.push(callerUserMention); }
+    toAdd.push({...removeUserMention, user_id: callerUserId});
+  }
+  if (toDelete.length > 0) {
+    await manager.createQueryBuilder()
+      .delete()
+      .from('group_users')
+      .whereInIds(toDelete.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+  if (toAdd.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(toAdd.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+
+  // TODO: At this point, we've removed removeUserId from every mention in group_users.
+  // The user may still be mentioned in billing_account_managers.  If the billing_account
+  // is linked to just this single organization, perhaps it would make sense to remove
+  // the user there, if the callerUser is themselves a billing account manager?
+
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+}
+
+/**
+ * Adds specified user to any guest groups for the resources of an org where the
+ * user needs to be and is not already.
+ */
+export async function addMissingGuestMemberships(userId: number, orgId: number,
+                                                 manager: EntityManager) {
+  // For workspaces:
+  // User should be in guest group if mentioned in a doc within that workspace.
+  let groupUsers = await manager.createQueryBuilder()
+    .select('workspace_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Workspace, 'workspaces')
+    .where('workspaces.org_id = :orgId', {orgId})
+    .innerJoin('workspaces.docs', 'docs')
+    .innerJoin('docs.aclRules', 'doc_acl_rules')
+    .innerJoin('doc_acl_rules.group', 'doc_groups')
+    .innerJoin('doc_groups.memberUsers', 'doc_group_users')
+    .andWhere('doc_group_users.id = :userId', {userId})
+    .leftJoin('workspaces.aclRules', 'workspace_acl_rules')
+    .leftJoin('workspace_acl_rules.group', 'workspace_groups')
+    .leftJoin('group_users', 'workspace_group_users',
+              'workspace_group_users.group_id = workspace_groups.id and ' +
+              'workspace_group_users.user_id = :userId')
+    .andWhere('workspace_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('workspaces.id, workspace_groups.id, workspace_group_users.user_id')
+    .having('workspace_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For org:
+  // User should be in guest group if mentioned in a workspace within that org.
+  groupUsers = await manager.createQueryBuilder()
+    .select('org_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Organization, 'orgs')
+    .where('orgs.id = :orgId', {orgId})
+    .innerJoin('orgs.workspaces', 'workspaces')
+    .innerJoin('workspaces.aclRules', 'workspaces_acl_rules')
+    .innerJoin('workspaces_acl_rules.group', 'workspace_groups')
+    .innerJoin('workspace_groups.memberUsers', 'workspace_group_users')
+    .andWhere('workspace_group_users.id = :userId', {userId})
+    .leftJoin('orgs.aclRules', 'org_acl_rules')
+    .leftJoin('org_acl_rules.group', 'org_groups')
+    .leftJoin('group_users', 'org_group_users',
+              'org_group_users.group_id = org_groups.id and ' +
+              'org_group_users.user_id = :userId')
+    .andWhere('org_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('org_groups.id, org_group_users.user_id')
+    .having('org_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For doc:
+  // Guest groups are not used.
+}
+
+interface Mention {
+  id: string|number;       // id of resource
+  kind: 'org'|'ws'|'doc';  // type of resource
+  user_id: number;         // id of user in group
+  group_id: number;        // id of group
+  name: string;            // name of group
+}
+
+type MentionKey = string;
+
+function getMentionKey(mention: Mention): MentionKey {
+  return `${mention.kind} ${mention.id}`;
+}

app/gen-server/lib/values.ts

@@ -0,0 +1,36 @@
+/**
+ * This smoothes over some awkward differences between TypeORM treatment of
+ * booleans and json in sqlite and postgres.  Booleans and json work fine
+ * with each db, but have different levels of driver-level support.
+ */
+
+export interface NativeValues {
+  // Json columns are handled natively by the postgres driver, but for
+  // sqlite requires a typeorm wrapper (simple-json).
+  jsonEntityType: 'json' | 'simple-json';
+  jsonType: 'json' | 'varchar';
+  booleanType: 'boolean' | 'integer';
+  dateTimeType: 'timestamp with time zone' | 'datetime';
+  trueValue: boolean | number;
+  falseValue: boolean | number;
+}
+
+const sqliteNativeValues: NativeValues = {
+  jsonEntityType: 'simple-json',
+  jsonType: 'varchar',
+  booleanType: 'integer',
+  dateTimeType: 'datetime',
+  trueValue: 1,
+  falseValue: 0
+};
+
+const postgresNativeValues: NativeValues = {
+  jsonEntityType: 'json',
+  jsonType: 'json',
+  booleanType: 'boolean',
+  dateTimeType: 'timestamp with time zone',
+  trueValue: true,
+  falseValue: false
+};
+
+export const nativeValues = (process.env.TYPEORM_TYPE === 'postgres') ? postgresNativeValues : sqliteNativeValues;

app/gen-server/migration/1536634251710-Initial.ts

@@ -0,0 +1,304 @@
+import {MigrationInterface, QueryRunner, Table} from "typeorm";
+
+
+export class Initial1536634251710 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // TypeORM doesn't currently help with types of created tables:
+    //   https://github.com/typeorm/typeorm/issues/305
+    // so we need to do a little smoothing over postgres and sqlite.
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    const now = "now()";
+
+    await queryRunner.createTable(new Table({
+      name: "users",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "api_key",
+          type: "varchar",
+          isNullable: true,
+          isUnique: true
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "orgs",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "domain",
+          type: "varchar",
+          isNullable: true,
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "owner_id",
+          type: "integer",
+          isNullable: true,
+          isUnique: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["owner_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "users"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "workspaces",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "org_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["org_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "orgs"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "docs",
+      columns: [
+        {
+          name: "id",
+          type: "varchar",
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "workspace_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["workspace_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "workspaces"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "groups",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "acl_rules",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "permissions",
+          type: "integer"
+        },
+        {
+          name: "type",
+          type: "varchar"
+        },
+        {
+          name: "workspace_id",
+          type: "integer",
+          isNullable: true
+        },
+        {
+          name: "org_id",
+          type: "integer",
+          isNullable: true
+        },
+        {
+          name: "doc_id",
+          type: "varchar",
+          isNullable: true
+        },
+        {
+          name: "group_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["workspace_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "workspaces"
+        },
+        {
+          columnNames: ["org_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "orgs"
+        },
+        {
+          columnNames: ["doc_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "docs"
+        },
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "group_users",
+      columns: [
+        {
+          name: "group_id",
+          type: "integer",
+          isPrimary: true
+        },
+        {
+          name: "user_id",
+          type: "integer",
+          isPrimary: true
+        },
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        },
+        {
+          columnNames: ["user_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "users"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "group_groups",
+      columns: [
+        {
+          name: "group_id",
+          type: "integer",
+          isPrimary: true
+        },
+        {
+          name: "subgroup_id",
+          type: "integer",
+          isPrimary: true
+        },
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        },
+        {
+          columnNames: ["subgroup_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        }
+      ]
+    }), false);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.query(`DROP TABLE "group_groups"`);
+    await queryRunner.query(`DROP TABLE "group_users"`);
+    await queryRunner.query(`DROP TABLE "acl_rules"`);
+    await queryRunner.query(`DROP TABLE "groups"`);
+    await queryRunner.query(`DROP TABLE "docs"`);
+    await queryRunner.query(`DROP TABLE "workspaces"`);
+    await queryRunner.query(`DROP TABLE "orgs"`);
+    await queryRunner.query(`DROP TABLE "users"`);
+  }
+}

app/gen-server/migration/1539031763952-Login.ts

@@ -0,0 +1,39 @@
+import {MigrationInterface, QueryRunner, Table} from "typeorm";
+
+export class Login1539031763952 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.createTable(new Table({
+        name: 'logins',
+        columns: [
+          {
+            name: "id",
+            type: "integer",
+            isGenerated: true,
+            generationStrategy: 'increment',
+            isPrimary: true
+          },
+          {
+            name: 'user_id',
+            type: 'integer'
+          },
+          {
+            name: 'email',
+            type: 'varchar',
+            isUnique: true
+          }
+        ],
+        foreignKeys: [
+          {
+            columnNames: ["user_id"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "users"
+          }
+        ]
+      }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.query('DROP TABLE logins');
+    }
+}

app/gen-server/migration/1549313797109-PinDocs.ts

@@ -0,0 +1,17 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class PinDocs1549313797109 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    await queryRunner.addColumn('docs', new TableColumn({
+      name: 'is_pinned',
+      type: 'boolean',
+      default: sqlite ? 0 : false
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('docs', 'is_pinned');
+  }
+}

app/gen-server/migration/1549381727494-UserPicture.ts

@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class UserPicture1549381727494 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.addColumn("users", new TableColumn({
+        name: "picture",
+        type: "varchar",
+        isNullable: true,
+      }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.dropColumn("users", "picture");
+    }
+}

app/gen-server/migration/1551805156919-LoginDisplayEmail.ts

@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class LoginDisplayEmail1551805156919 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn('logins', new TableColumn({
+      name: 'display_email',
+      type: 'varchar',
+      isNullable: true,
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('logins', 'display_email');
+  }
+}

app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts

@@ -0,0 +1,32 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class LoginDisplayEmailNonNull1552416614755 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.query('update logins set display_email = email where display_email is null');
+    // if our db will already heavily loaded, it might be better to add a check constraint
+    // rather than modifying the column properties.  But for our case, this will be fast.
+
+    // To work correctly with RDS version of postgres, it is important to clone
+    // and change typeorm's settings for the column, rather than the settings specified
+    // in previous migrations.  Otherwise typeorm will fall back on a brutal method of
+    // drop-and-recreate that doesn't work for non-null in any case.
+    //
+    // The pg command is very simple, just alter table logins alter column display_email set not null
+    // but sqlite migration is tedious since table needs to be rebuilt, so still just
+    // marginally worthwhile letting typeorm deal with it.
+    const logins = (await queryRunner.getTable('logins'))!;
+    const displayEmail = logins.findColumnByName('display_email')!;
+    const displayEmailNonNull = displayEmail.clone();
+    displayEmailNonNull.isNullable = false;
+    await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('logins'))!;
+    const displayEmail = logins.findColumnByName('display_email')!;
+    const displayEmailNonNull = displayEmail.clone();
+    displayEmailNonNull.isNullable = true;
+    await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
+  }
+}

app/gen-server/migration/1553016106336-Indexes.ts

@@ -0,0 +1,66 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class Indexes1553016106336 implements MigrationInterface {
+
+   public async up(queryRunner: QueryRunner): Promise<any> {
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__org_id",
+       columnNames: ["org_id"]
+     }));
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__workspace_id",
+       columnNames: ["workspace_id"]
+     }));
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__doc_id",
+       columnNames: ["doc_id"]
+     }));
+
+     await queryRunner.createIndex("group_groups", new TableIndex({
+       name: "group_groups__group_id",
+       columnNames: ["group_id"]
+     }));
+     await queryRunner.createIndex("group_groups", new TableIndex({
+       name: "group_groups__subgroup_id",
+       columnNames: ["subgroup_id"]
+     }));
+
+     await queryRunner.createIndex("group_users", new TableIndex({
+       name: "group_users__group_id",
+       columnNames: ["group_id"]
+     }));
+     await queryRunner.createIndex("group_users", new TableIndex({
+       name: "group_users__user_id",
+       columnNames: ["user_id"]
+     }));
+
+     await queryRunner.createIndex("workspaces", new TableIndex({
+       name: "workspaces__org_id",
+       columnNames: ["org_id"]
+     }));
+
+     await queryRunner.createIndex("docs", new TableIndex({
+       name: "docs__workspace_id",
+       columnNames: ["workspace_id"]
+     }));
+
+     await queryRunner.createIndex("logins", new TableIndex({
+       name: "logins__user_id",
+       columnNames: ["user_id"]
+     }));
+   }
+
+   public async down(queryRunner: QueryRunner): Promise<any> {
+     await queryRunner.dropIndex("acl_rules", "acl_rules__org_id");
+     await queryRunner.dropIndex("acl_rules", "acl_rules__workspace_id");
+     await queryRunner.dropIndex("acl_rules", "acl_rules__doc_id");
+     await queryRunner.dropIndex("group_groups", "group_groups__group_id");
+     await queryRunner.dropIndex("group_groups", "group_groups__subgroup_id");
+     await queryRunner.dropIndex("group_users", "group_users__group_id");
+     await queryRunner.dropIndex("group_users", "group_users__user_id");
+     await queryRunner.dropIndex("workspaces", "workspaces__org_id");
+     await queryRunner.dropIndex("docs", "docs__workspace_id");
+     await queryRunner.dropIndex("logins", "logins__user_id");
+   }
+
+}

app/gen-server/migration/1556726945436-Billing.ts

@@ -0,0 +1,225 @@
+import {MigrationInterface, QueryRunner, Table, TableColumn, TableForeignKey} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Product} from 'app/gen-server/entity/Product';
+import {nativeValues} from 'app/gen-server/lib/values';
+
+export class Billing1556726945436 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // Create table for products.
+    await queryRunner.createTable(new Table({
+      name: 'products',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'name',
+          type: 'varchar'
+        },
+        {
+          name: 'stripe_product_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'features',
+          type: nativeValues.jsonType
+        }
+      ]
+    }));
+
+    // Create a basic free product that existing orgs can use.
+    const product = new Product();
+    product.name = 'Free';
+    product.features = {};
+    await queryRunner.manager.save(product);
+
+    // Create billing accounts and billing account managers.
+    await queryRunner.createTable(new Table({
+      name: 'billing_accounts',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'product_id',
+          type: 'integer'
+        },
+        {
+          name: 'individual',
+          type: nativeValues.booleanType
+        },
+        {
+          name: 'in_good_standing',
+          type: nativeValues.booleanType,
+          default: nativeValues.trueValue
+        },
+        {
+          name: 'status',
+          type: nativeValues.jsonType,
+          isNullable: true
+        },
+        {
+          name: 'stripe_customer_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'stripe_subscription_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'stripe_plan_id',
+          type: 'varchar',
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['product_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'products'
+        }
+      ]
+    }));
+
+    await queryRunner.createTable(new Table({
+      name: 'billing_account_managers',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'billing_account_id',
+          type: 'integer'
+        },
+        {
+          name: 'user_id',
+          type: 'integer'
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['billing_account_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'billing_accounts',
+          onDelete: 'CASCADE'  // delete manager if referenced billing_account goes away
+        },
+        {
+          columnNames: ['user_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'users',
+          onDelete: 'CASCADE'  // delete manager if referenced user goes away
+        }
+      ]
+    }));
+
+    // Add a reference to billing accounts from orgs.
+    await queryRunner.addColumn('orgs', new TableColumn({
+        name: 'billing_account_id',
+        type: 'integer',
+        isNullable: true
+    }));
+    await queryRunner.createForeignKey('orgs', new TableForeignKey({
+      columnNames: ['billing_account_id'],
+      referencedColumnNames: ['id'],
+      referencedTableName: 'billing_accounts'
+    }));
+
+    // Let's add billing accounts to all existing orgs.
+    // Personal orgs are put on an individual billing account.
+    // Other orgs are put on a team billing account, with the
+    // list of payment managers seeded by owners of that account.
+    const query =
+      queryRunner.manager.createQueryBuilder()
+      .select('orgs.id')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.owner', 'owners')
+      .addSelect('orgs.owner.id')
+      .leftJoinAndSelect('orgs.aclRules', 'acl_rules')
+      .leftJoinAndSelect('acl_rules.group', 'groups')
+      .leftJoin('groups.memberUsers', 'users')
+      .addSelect('users.id')
+      .where('permissions & 8 = 8');  // seed managers with owners+editors, omitting guests+viewers
+                                      // (permission 8 is "Remove")
+    const orgs = await query.getMany();
+    for (const org of orgs) {
+      const individual = Boolean(org.owner);
+      const billingAccountInsert = await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(BillingAccount)
+        .values([{product, individual}])
+        .execute();
+      const billingAccountId = billingAccountInsert.identifiers[0].id;
+      if (individual) {
+        await queryRunner.manager.createQueryBuilder()
+          .insert()
+          .into(BillingAccountManager)
+          .values([{billingAccountId, userId: org.owner.id}])
+          .execute();
+      } else {
+        for (const rule of org.aclRules) {
+          for (const user of rule.group.memberUsers) {
+            await queryRunner.manager.createQueryBuilder()
+              .insert()
+              .into(BillingAccountManager)
+              .values([{billingAccountId, userId: user.id}])
+              .execute();
+          }
+        }
+      }
+      await queryRunner.manager.createQueryBuilder()
+        .update(Organization)
+        .set({billingAccountId})
+        .where('id = :id', {id: org.id})
+        .execute();
+    }
+
+    // TODO: in a future migration, orgs.billing_account_id could be constrained
+    // to be non-null.  All code deployments linked to a database that will be
+    // migrated must have code that sets orgs.billing_account_id by that time,
+    // otherwise they would fail to create orgs (and remember creating a user
+    // involves creating an org).
+    /*
+    // Now that all orgs have a billing account (and this migration is running within
+    // a transaction), we can constrain orgs.billing_account_id to be non-null.
+    const orgTable = (await queryRunner.getTable('orgs'))!;
+    const billingAccountId = orgTable.findColumnByName('billing_account_id')!;
+    const billingAccountIdNonNull = billingAccountId.clone();
+    billingAccountIdNonNull.isNullable = false;
+    await queryRunner.changeColumn('orgs', billingAccountId, billingAccountIdNonNull);
+    */
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    // this is a bit ugly, but is the documented way to remove a foreign key
+    const table = await queryRunner.getTable('orgs');
+    const foreignKey = table!.foreignKeys.find(fk => fk.columnNames.indexOf('billing_account_id') !== -1);
+    await queryRunner.dropForeignKey('orgs', foreignKey!);
+
+    await queryRunner.dropColumn('orgs', 'billing_account_id');
+    await queryRunner.dropTable('billing_account_managers');
+    await queryRunner.dropTable('billing_accounts');
+    await queryRunner.dropTable('products');
+  }
+
+}

app/gen-server/migration/1557157922339-OrgDomainUnique.ts

@@ -0,0 +1,27 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class OrgDomainUnique1557157922339 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('orgs'))!;
+    const domain = logins.findColumnByName('domain')!;
+    const domainUnique = domain.clone();
+    domainUnique.isUnique = true;
+    await queryRunner.changeColumn('orgs', domain, domainUnique);
+
+    // On postgres, all of the above amounts to:
+    //   ALTER TABLE "orgs" ADD CONSTRAINT "..." UNIQUE ("domain")
+    // On sqlite, the table gets regenerated.
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('orgs'))!;
+    const domain = logins.findColumnByName('domain')!;
+    const domainNonUnique = domain.clone();
+    domainNonUnique.isUnique = false;
+    await queryRunner.changeColumn('orgs', domain, domainNonUnique);
+
+    // On postgres, all of the above amount to:
+    //   ALTER TABLE "orgs" DROP CONSTRAINT "..."
+  }
+}

app/gen-server/migration/1561589211752-Aliases.ts

@@ -0,0 +1,67 @@
+import {MigrationInterface, QueryRunner, Table, TableColumn, TableIndex} from 'typeorm';
+import {datetime, now} from 'app/gen-server/sqlUtils';
+
+export class Aliases1561589211752 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const dbType = queryRunner.connection.driver.options.type;
+
+    // Make a table for document aliases.
+    await queryRunner.createTable(new Table({
+      name: 'aliases',
+      columns: [
+        {
+          name: 'url_id',
+          type: 'varchar',
+          isPrimary: true
+        },
+        {
+          name: 'org_id',
+          type: 'integer',
+          isPrimary: true
+        },
+        {
+          name: 'doc_id',
+          type: 'varchar',
+          isNullable: true   // nullable in case in future we make aliases for other resources
+        },
+        {
+          name: "created_at",
+          type: datetime(dbType),
+          default: now(dbType)
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['doc_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'docs',
+          onDelete: 'CASCADE'  // delete alias if doc goes away
+        },
+        {
+          columnNames: ['org_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'orgs'
+          // no CASCADE set - let deletions be triggered via docs
+        }
+      ]
+    }));
+
+    // Add preferred alias to docs.  Not quite a foreign key (we'd need org as well)
+    await queryRunner.addColumn('docs', new TableColumn({
+        name: 'url_id',
+        type: 'varchar',
+        isNullable: true
+    }));
+    await queryRunner.createIndex("docs", new TableIndex({
+      name: "docs__url_id",
+      columnNames: ["url_id"]
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropIndex('docs', 'docs__url_id');
+    await queryRunner.dropColumn('docs', 'url_id');
+    await queryRunner.dropTable('aliases');
+  }
+}

app/gen-server/migration/1568238234987-TeamMembers.ts

@@ -0,0 +1,56 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+import * as roles from "app/common/roles";
+import {AclRuleOrg} from "app/gen-server/entity/AclRule";
+import {Group} from "app/gen-server/entity/Group";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Permissions} from "app/gen-server/lib/Permissions";
+
+export class TeamMembers1568238234987 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // Get all orgs and add a team member ACL (with group) to each.
+    const orgs = await queryRunner.manager.createQueryBuilder()
+      .select("orgs.id")
+      .from(Organization, "orgs")
+      .getMany();
+    for (const org of orgs) {
+      const groupInsert = await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(Group)
+        .values([{name: roles.MEMBER}])
+        .execute();
+      const groupId = groupInsert.identifiers[0].id;
+      await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(AclRuleOrg)
+        .values([{
+          permissions: Permissions.VIEW,
+          organization: {id: org.id},
+          group: groupId
+        }])
+        .execute();
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    // Remove all team member groups and corresponding ACLs.
+    const groups = await queryRunner.manager.createQueryBuilder()
+      .select("groups")
+      .from(Group, "groups")
+      .where('name = :name', {name: roles.MEMBER})
+      .getMany();
+    for (const group of groups) {
+      await queryRunner.manager.createQueryBuilder()
+        .delete()
+        .from(AclRuleOrg)
+        .where("group_id = :id", {id: group.id})
+        .execute();
+    }
+    await queryRunner.manager.createQueryBuilder()
+      .delete()
+      .from(Group)
+      .where("name = :name", {name: roles.MEMBER})
+      .execute();
+  }
+
+}

app/gen-server/migration/1569593726320-FirstLogin.ts

@@ -0,0 +1,18 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class FirstLogin1569593726320 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    await queryRunner.addColumn('users', new TableColumn({
+      name: 'first_login_at',
+      type: datetime,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('users', 'first_login_at');
+  }
+}

app/gen-server/migration/1569946508569-FirstTimeUser.ts

@@ -0,0 +1,18 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+import {nativeValues} from 'app/gen-server/lib/values';
+
+export class FirstTimeUser1569946508569 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn("users", new TableColumn({
+      name: "is_first_time_user",
+      type: nativeValues.booleanType,
+      default: nativeValues.falseValue,
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn("users", "is_first_time_user");
+  }
+
+}

app/gen-server/migration/1573569442552-CustomerIndex.ts

@@ -0,0 +1,15 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class CustomerIndex1573569442552 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.createIndex("billing_accounts", new TableIndex({
+      name: "billing_accounts__stripe_customer_id",
+      columnNames: ["stripe_customer_id"]
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropIndex("billing_accounts", "billing_accounts__stripe_customer_id");
+  }
+}

app/gen-server/migration/1579559983067-ExtraIndexes.ts

@@ -0,0 +1,45 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class ExtraIndexes1579559983067 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.createIndex("acl_rules", new TableIndex({
+      name: "acl_rules__group_id",
+      columnNames: ["group_id"]
+    }));
+    await queryRunner.createIndex("orgs", new TableIndex({
+      name: "orgs__billing_account_id",
+      columnNames: ["billing_account_id"]
+    }));
+    await queryRunner.createIndex("billing_account_managers", new TableIndex({
+      name: "billing_account_managers__billing_account_id",
+      columnNames: ["billing_account_id"]
+    }));
+    await queryRunner.createIndex("billing_account_managers", new TableIndex({
+      name: "billing_account_managers__user_id",
+      columnNames: ["user_id"]
+    }));
+    await queryRunner.createIndex("billing_accounts", new TableIndex({
+      name: "billing_accounts__product_id",
+      columnNames: ["product_id"]
+    }));
+    await queryRunner.createIndex("aliases", new TableIndex({
+      name: "aliases__org_id",
+      columnNames: ["org_id"]
+    }));
+    await queryRunner.createIndex("aliases", new TableIndex({
+      name: "aliases__doc_id",
+      columnNames: ["doc_id"]
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropIndex("acl_rules", "acl_rules__group_id");
+    await queryRunner.dropIndex("orgs", "orgs__billing_account_id");
+    await queryRunner.dropIndex("billing_account_managers", "billing_account_managers__billing_account_id");
+    await queryRunner.dropIndex("billing_account_managers", "billing_account_managers__user_id");
+    await queryRunner.dropIndex("billing_accounts", "billing_accounts__product_id");
+    await queryRunner.dropIndex("aliases", "aliases__org_id");
+    await queryRunner.dropIndex("aliases", "aliases__doc_id");
+  }
+}

app/gen-server/migration/1591755411755-OrgHost.ts

@@ -0,0 +1,17 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class OrgHost1591755411755 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn('orgs', new TableColumn({
+      name: 'host',
+      type: 'varchar',
+      isNullable: true,
+      isUnique: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('orgs', 'host');
+  }
+}

app/gen-server/migration/1592261300044-DocRemovedAt.ts

@@ -0,0 +1,26 @@
+import {nativeValues} from "app/gen-server/lib/values";
+import {MigrationInterface, QueryRunner, TableColumn, TableIndex} from "typeorm";
+
+export class DocRemovedAt1592261300044 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    for (const table of ['docs', 'workspaces']) {
+      await queryRunner.addColumn(table, new TableColumn({
+        name: 'removed_at',
+        type: nativeValues.dateTimeType,
+        isNullable: true
+      }));
+      await queryRunner.createIndex(table, new TableIndex({
+        name: `${table}__removed_at`,
+        columnNames: ['removed_at']
+      }));
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    for (const table of ['docs', 'workspaces']) {
+      await queryRunner.dropIndex(table, `${table}__removed_at`);
+      await queryRunner.dropColumn(table, 'removed_at');
+    }
+  }
+}

app/gen-server/sqlUtils.ts

@@ -0,0 +1,90 @@
+import {DatabaseType} from 'typeorm';
+
+/**
+ *
+ * Generates an expression to simulate postgres's bit_or
+ * aggregate function in sqlite.  The expression is verbose,
+ * and has a term for each bit in the permission bitmap,
+ * but this seems ok since sqlite is only used in the dev
+ * environment.
+ * @param column: the sql column to aggregate
+ * @param bits: the maximum number of bits to consider
+ *
+ */
+export function sqliteBitOr(column: string, bits: number): string {
+  const parts: string[] = [];
+  let mask: number = 1;
+  for (let b = 0; b < bits; b++) {
+    parts.push(`((sum(${column}&${mask})>0)<<${b})`);
+    mask *= 2;
+  }
+  return `(${parts.join('+')})`;
+}
+
+/**
+ * Generates an expression to aggregate the named column
+ * by taking the bitwise-or of all the values it takes on.
+ * @param dbType: the type of database (sqlite and postgres are supported)
+ * @param column: the sql column to aggregate
+ * @param bits: the maximum number of bits to consider (used for sqlite variant)
+ */
+export function bitOr(dbType: DatabaseType, column: string, bits: number): string {
+  switch (dbType) {
+    case 'postgres':
+      return `bit_or(${column})`;
+    case 'sqlite':
+      return sqliteBitOr(column, bits);
+    default:
+      throw new Error(`bitOr not implemented for ${dbType}`);
+  }
+}
+
+/**
+ * Convert a json value returned by the database into a javascript
+ * object.  For postgres, the value is already unpacked, but for sqlite
+ * it is a string.
+ */
+export function readJson(dbType: DatabaseType, selection: any) {
+  switch (dbType) {
+    case 'postgres':
+      return selection;
+    case 'sqlite':
+      return JSON.parse(selection);
+    default:
+      throw new Error(`readJson not implemented for ${dbType}`);
+  }
+}
+
+export function now(dbType: DatabaseType) {
+  switch (dbType) {
+    case 'postgres':
+      return 'now()';
+    case 'sqlite':
+      return "datetime('now')";
+    default:
+      throw new Error(`now not implemented for ${dbType}`);
+  }
+}
+
+// Understands strings like: "-30 days" or "1 year"
+export function fromNow(dbType: DatabaseType, relative: string) {
+  switch (dbType) {
+    case 'postgres':
+      return `(now() + interval '${relative}')`;
+    case 'sqlite':
+      return `datetime('now','${relative}')`;
+    default:
+      throw new Error(`fromNow not implemented for ${dbType}`);
+  }
+}
+
+export function datetime(dbType: DatabaseType) {
+  switch (dbType) {
+    case 'postgres':
+      return 'timestamp with time zone';
+    case 'sqlite':
+      return "datetime";
+    default:
+      throw new Error(`now not implemented for ${dbType}`);
+  }
+}