(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552

app/common/ActionBundle.ts

@@ -0,0 +1,74 @@
+/**
+ * Basic definitions of types needed for ActionBundles.
+ * See also EncActionBundle for how these are packaged for encryption.
+ */
+
+import {DocAction, UserAction} from 'app/common/DocActions';
+
+// Metadata about the action.
+export interface ActionInfo {
+  time: number;       // Milliseconds since epoch.
+  user: string;
+  inst: string;
+  desc?: string;
+  otherId: number;
+  linkId: number;
+}
+
+// Envelope contains information about recipients. In EncActionBundle, it's augmented with
+// information about the symmetric key that encrypts this envelope's contents.
+export interface Envelope {
+  recipients: string[];       // sorted array of recipient instanceIds
+}
+
+// EnvContent packages arbitrary content with the index of the envelope to which it belongs.
+export type EnvContent<Content> = [number, Content];
+
+// ActionBundle contains actions arranged into envelopes, i.e. split up by sets of recipients.
+// Note that different Envelopes contain different sets of recipients (which may overlap however).
+// ActionBundle is what gets encrypted/decrypted and then sent between hub and instance.
+export interface ActionBundle {
+  actionNum: number;
+  actionHash: string|null;        // a checksum of bundle, (not including actionHash and other parts).
+  parentActionHash: string|null;  // a checksum of the parent action bundle, if there is one.
+  envelopes: Envelope[];
+  info: EnvContent<ActionInfo>;           // Should be in the envelope addressed to all peers.
+  stored: Array<EnvContent<DocAction>>;
+  calc: Array<EnvContent<DocAction>>;
+}
+
+export function getEnvContent<Content>(items: Array<EnvContent<Content>>): Content[] {
+  return items.map((item) => item[1]);
+}
+
+// ======================================================================
+// Types for ActionBundles used locally inside an instance.
+
+// Local action received from the browser, that is not yet applied. It is usually one UserAction,
+// but when multiple actions are sent by the browser in one call, they will form one bundle.
+export interface UserActionBundle {
+  info: ActionInfo;
+  userActions: UserAction[];
+}
+
+// ActionBundle as received from the sandbox. It does not have some action metadata, but does have
+// undo information and a retValue for each input UserAction. Note that it is satisfied by the
+// ActionBundle structure defined in sandbox/grist/action_obj.py.
+export interface SandboxActionBundle {
+  envelopes: Envelope[];
+  stored: Array<EnvContent<DocAction>>;
+  calc: Array<EnvContent<DocAction>>;
+  undo: Array<EnvContent<DocAction>>;   // Inverse actions for all 'stored' actions.
+  retValues: any[];                     // Contains retValue for each of userActions.
+}
+
+// Local action that's been applied. It now has an actionNum, and includes doc actions packaged
+// into envelopes, as well as undo, and userActions, which allow rebasing.
+export interface LocalActionBundle extends ActionBundle {
+  userActions: UserAction[];
+
+  // Inverse actions for all 'stored' actions. These aren't shared and not split by envelope.
+  // Applying 'undo' is governed by EDIT rather than READ permissions, so we always apply all undo
+  // actions. (It is the result of applying 'undo' that may be addressed to different recipients).
+  undo: DocAction[];
+}

app/common/ActionDispatcher.ts

@@ -0,0 +1,71 @@
+import mapValues = require('lodash/mapValues');
+import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction} from "./DocActions";
+
+// TODO this replaces modelUtil's ActionDispatcher and bulkActionExpand. Those should be removed.
+
+/**
+ * Helper class which provides a `dispatchAction` method that dispatches DocActions received from
+ * the server to methods `this.on{ActionType}`, e.g. `this.onUpdateRecord`.
+ *
+ * Implementation methods `on*` are called with the action as the first argument, and with
+ * the action arguments as additional method arguments, for convenience.
+ *
+ * Methods for bulk actions may be implemented directly, or will iterate through each record in
+ * the action, and call the single-record methods for each one.
+ */
+export abstract class ActionDispatcher {
+  public dispatchAction(action: DocAction): void {
+    // In node 6 testing, this switch is 5+ times faster than looking up "on"+action[0].
+    const a: any[] = action;
+    switch (action[0]) {
+      case "AddRecord":        return this.onAddRecord       (action, a[1], a[2], a[3]);
+      case "UpdateRecord":     return this.onUpdateRecord    (action, a[1], a[2], a[3]);
+      case "RemoveRecord":     return this.onRemoveRecord    (action, a[1], a[2]);
+      case "BulkAddRecord":    return this.onBulkAddRecord   (action, a[1], a[2], a[3]);
+      case "BulkUpdateRecord": return this.onBulkUpdateRecord(action, a[1], a[2], a[3]);
+      case "BulkRemoveRecord": return this.onBulkRemoveRecord(action, a[1], a[2]);
+      case "ReplaceTableData": return this.onReplaceTableData(action, a[1], a[2], a[3]);
+      case "AddColumn":        return this.onAddColumn       (action, a[1], a[2], a[3]);
+      case "RemoveColumn":     return this.onRemoveColumn    (action, a[1], a[2]);
+      case "RenameColumn":     return this.onRenameColumn    (action, a[1], a[2], a[3]);
+      case "ModifyColumn":     return this.onModifyColumn    (action, a[1], a[2], a[3]);
+      case "AddTable":         return this.onAddTable        (action, a[1], a[2]);
+      case "RemoveTable":      return this.onRemoveTable     (action, a[1]);
+      case "RenameTable":      return this.onRenameTable     (action, a[1], a[2]);
+      default: throw new Error(`Received unknown action ${action[0]}`);
+    }
+  }
+
+  protected abstract onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void;
+  protected abstract onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void;
+  protected abstract onRemoveRecord(action: DocAction, tableId: string, rowId: number): void;
+
+  // If not overridden, these will make multiple calls to single-record action methods.
+  protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
+    for (let i = 0; i < rowIds.length; i++) {
+      this.onAddRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i]));
+    }
+  }
+  protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
+    for (let i = 0; i < rowIds.length; i++) {
+      this.onUpdateRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i]));
+    }
+  }
+  protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) {
+    for (const r of rowIds) {
+      this.onRemoveRecord(action, tableId, r);
+    }
+  }
+
+  protected abstract onReplaceTableData(
+    action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void;
+
+  protected abstract onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void;
+  protected abstract onRemoveColumn(action: DocAction, tableId: string, colId: string): void;
+  protected abstract onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void;
+  protected abstract onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void;
+
+  protected abstract onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void;
+  protected abstract onRemoveTable(action: DocAction, tableId: string): void;
+  protected abstract onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void;
+}

app/common/ActionGroup.ts

@@ -0,0 +1,19 @@
+import {ActionSummary} from 'app/common/ActionSummary';
+
+/** This is the action representation the client works with. */
+export interface ActionGroup {
+  actionNum: number;
+  actionHash: string;
+  desc?: string;
+  actionSummary: ActionSummary;
+  fromSelf: boolean;
+  linkId: number;
+  otherId: number;
+  time: number;
+  user: string;
+  rowIdHint: number;      // If non-zero, this is a rowId that would be a good place to put
+                          // the cursor after an undo.
+  primaryAction: string;  // The name of the first user action in the ActionGroup.
+  isUndo: boolean;        // True if the first user action is ApplyUndoActions.
+  internal: boolean;      // True if it is inappropriate to log/undo the action.
+}

app/common/ActionRouter.ts

@@ -0,0 +1,49 @@
+import { Rpc } from "grain-rpc";
+
+/**
+ * ActionRouter allows to choose what actions to send over rpc. Action are posted as message `{type:
+ * "docAction", action }` over rpc.
+ */
+export class ActionRouter {
+
+  private _subscribedTables: Set<string> = new Set();
+
+  constructor(private _rpc: Rpc) {}
+
+  /**
+   * Subscribe to send all actions related to a table. Keeps sending actions if table is renamed.
+   */
+  public subscribeTable(tableId: string): Promise<void> {
+    this._subscribedTables.add(tableId);
+    return Promise.resolve();
+  }
+
+  /**
+   * Stop sending all message related to a table.
+   */
+  public unsubscribeTable(tableId: string): Promise<void> {
+    this._subscribedTables.delete(tableId);
+    return Promise.resolve();
+  }
+
+  /**
+   * Process a action updates subscription set in case of table rename and table remove, and post
+   * action if it matches a subscriptions.
+   */
+  public process(action: any[]): Promise<void> {
+    const tableId = action[1];
+    if (!this._subscribedTables.has(tableId)) {
+      return Promise.resolve();
+    }
+    switch (action[0]) {
+      case "RemoveTable":
+        this._subscribedTables.delete(tableId);
+        break;
+      case "RenameTable":
+        this._subscribedTables.delete(tableId);
+        this._subscribedTables.add(action[2]);
+        break;
+    }
+    return this._rpc.postMessage({type: "docAction", action});
+  }
+}

app/common/ActionSummary.ts

@@ -0,0 +1,235 @@
+import {CellDelta, TabularDiff, TabularDiffs} from 'app/common/TabularDiff';
+import toPairs = require('lodash/toPairs');
+
+/**
+ * An ActionSummary represents the overall effect of changes that took place
+ * during a period of history.
+ *   - Only net changes are represented.  Intermediate changes within the period are
+ *     not represented.  Changes that are done and undone within the period are not
+ *     represented.
+ *   - Net addition, removal, and renaming of tables is represented.  The names
+ *     of tables, for ActionSummary purposes are their tableIds, the database-safe
+ *     version of their names.
+ *   - Net addition, removal, and renaming of columns is represented.  As for tables,
+ *     the names of columns for ActionSummary purposes are their colIds.
+ *   - Net additions and removals of rows are partially represented.  The rowIds of added
+ *     and removed rows are represented fully.  The *values* of cells in the rows that
+ *     were added or removed are stored in some cases.  There is a threshold on the
+ *     number of rows whose values will be cached for each DocAction scanned.
+ *   - Net updates of rows are partially represented.  The rowIds of updated rows are
+ *     represented fully, but the *values* of updated cells partially, as for additions/
+ *     removals.
+ *   - Cell value changes affecting _grist_* tables are always represented in full,
+ *     even if they are bulk changes.
+ *
+ * The representation of table name changes and column name changes is the same,
+ * simply a list of name pairs [before, after].  We represent the addition of a
+ * a table (or column) as the special name pair [null, initialName], and the
+ * removal of a table (or column) as the special name pair [finalName, null].
+ *
+ * An ActionSummary contains two fields:
+ *   - tableRenames: a list of table name changes (incuding addition/removal).
+ *   - tableDeltas: a dictionary of changes within a table.
+ *
+ * The key of the tableDeltas dictionary is the name of a table at the end of the
+ * period of history covered by the ActionSummary.
+ *   - For example, if we add a table called N, we use the key N for it.
+ *   - If we rename a table from N1 to N2, we use the key N2 for it.
+ *   - If we add a table called N1, then rename it to N2, we use the key N2 for it.
+ * If the table was removed during that period, we use its name at the beginning
+ * of the period, preceded by "-".
+ *   - If we remove a table called N, we use the key -N for it.
+ *   - If we add a table called N, then remove it, there is no net change to represent.
+ *   - If we remove a table called N, then add a new table called N, we use the key -N
+ *     for the first, and the key N for the second.
+ *
+ * The changes within a table are represented as a TableDelta, which has the following
+ * fields:
+ *   - columnRenames: a list of column name changes (incuding addition/removal).
+ *   - columnDeltas: a dictionary of changes within a column.
+ *   - updateRows, removeRows, addRows: lists of affected rows.
+ *
+ * The columnRenames/columnDeltas pair work just like tableRenames/tableDeltas, just
+ * on the scope of columns within a table rather than tables within a document.
+ *
+ * The changes within a column are represented as a ColumnDelta, which is a dictionary
+ * keyed by rowIds.  It contains CellDelta values.  CellDelta values represent before
+ * and after values of a particular cell.
+ *   - a CellDelta of [null, [value]] represents a cell that was non-existent coming into
+ *     existence with the given value.
+ *   - a CellDelta of [[value], null] represents an existing cell with the given value that
+ *     is removed.
+ *   - a CellDelta of [[value1], [value2]] represents a change in value of a cell between
+ *     two known values.
+ *   - a CellDelta of ['?', [value2]] represents a change in value of a cell from an
+ *     unknown value to a known value.  Unknown values happen when we know a cell was
+ *     implicated in a bulk change but its value didn't happen to be stored.
+ *   - a CellDelta of [[value1], '?'] represents a change in value of a cell from an
+ *     known value to an unknown value.
+ * The CellDelta itself does not tell you whether the rowId has the same identity before
+ * and after -- for example it may have been removed and then added.  That information
+ * is available by consulting the removeRows and addRows fields.
+ *
+ */
+
+/**
+ * A collection of changes related to a set of tables.
+ */
+export interface ActionSummary {
+  tableRenames: LabelDelta[];  /** a list of table renames/additions/removals */
+  tableDeltas: {[tableId: string]: TableDelta};  /** changes within an individual table */
+}
+
+/**
+ * A collection of changes related to rows and columns of a single table.
+ */
+export interface TableDelta {
+  updateRows: number[];  /** rowIds of rows that exist before+after and were changed during */
+  removeRows: number[];  /** rowIds of rows that existed before but were removed during */
+  addRows: number[];     /** rowIds of rows that were added during, and exist after */
+  /** Partial record of cell-level changes - large bulk changes not included. */
+  columnDeltas: {[colId: string]: ColumnDelta};
+  columnRenames: LabelDelta[];  /** a list of column renames/additions/removals */
+}
+
+/**
+ * Pairs of before/after names of tables and columns.  Null represents non-existence,
+ * so the addition and removal of tables/columns can be represented.
+ */
+export type LabelDelta = [string|null, string|null];
+
+/**
+ * A collection of changes related to cells in a specific column.
+ */
+export interface ColumnDelta {
+  [rowId: number]: CellDelta;
+}
+
+
+/** Create an ActionSummary for a period with no action */
+export function createEmptyActionSummary(): ActionSummary {
+  return { tableRenames: [], tableDeltas: {} };
+}
+
+/** Create a TableDelta for a period with no action */
+export function createEmptyTableDelta(): TableDelta {
+  return {
+    updateRows: [],
+    removeRows: [],
+    addRows: [],
+    columnDeltas: {},
+    columnRenames: []
+  };
+}
+
+
+/**
+ * Distill a summary further, into tabular form, for ease of rendering.
+ */
+export function asTabularDiffs(summary: ActionSummary): TabularDiffs {
+  const allChanges: TabularDiffs = {};
+  for (const [tableId, td] of toPairs(summary.tableDeltas)) {
+    const tableChanges: TabularDiff = allChanges[tableId] = {
+      header: [],
+      cells: [],
+    };
+    // swap order to row-dominant for visualization purposes
+    const perRow: {[row: number]: {[name: string]: any}} = {};
+    const activeCols = new Set<string>();
+    // iterate through the column-dominant representation grist prefers internally
+    for (const [col, perCol] of toPairs(td.columnDeltas)) {
+      activeCols.add(col);
+      // iterate through the rows for that column, writing out the row-dominant
+      // results we want for visualization.
+      for (const row of Object.keys(perCol)) {
+        if (!perRow[row as any]) { perRow[row as any] = {}; }
+        perRow[row as any][col] = perCol[row as any];
+      }
+    }
+    // TODO: recover order of columns; recover row numbers (as opposed to rowIds)
+    const activeColsWithoutManualSort = [...activeCols].filter(c => c !== 'manualSort');
+    tableChanges.header = activeColsWithoutManualSort;
+    const addedRows = new Set(td.addRows);
+    const removedRows = new Set(td.removeRows);
+    const updatedRows = new Set(td.updateRows);
+    const rowIds = Object.keys(perRow).map(row => parseInt(row, 10));
+    const presentRows = new Set(rowIds);
+    const droppedRows = [...addedRows, ...removedRows, ...updatedRows]
+      .filter(x => !presentRows.has(x))
+      .sort((a, b) => a - b);
+
+    // Now that we have pulled together rows of changes, we will add a summary cell
+    // to each row to show whether they were caused by row updates, additions or removals.
+    // We also at this point make sure the cells of the row are output in a consistent
+    // order with a header.
+    for (const rowId of rowIds) {
+      if (droppedRows.length > 0) {
+        // Bulk additions/removals/updates may result in just some rows being saved.
+        // We signal this visually with a "..." row.  The order of where this should
+        // go isn't well defined at this point (there's a row number TODO above).
+        if (rowId > droppedRows[0]) {
+          tableChanges.cells.push(['...', droppedRows[0],
+                                   activeColsWithoutManualSort.map(x => [null, null] as [null, null])]);
+          while (rowId > droppedRows[0]) {
+            droppedRows.shift();
+          }
+        }
+      }
+      // For each rowId, we need to issue either 1 or 2 rows.  We issue 2 rows
+      // if the rowId is both added and removed - in this scenario, the rows
+      // before and after are unrelated.  In all other cases, the before and
+      // after values refer to the same row.
+      const versions: Array<[string, (diff: CellDelta) => CellDelta]> = [];
+      if (addedRows.has(rowId) && removedRows.has(rowId)) {
+        versions.push(['-', (diff) => [diff[0], null]]);
+        versions.push(['+', (diff) => [null, diff[1]]]);
+      } else {
+        let code: string = '...';
+        if (updatedRows.has(rowId)) { code = '→'; }
+        if (addedRows.has(rowId))   { code = '+';  }
+        if (removedRows.has(rowId)) { code = '-';  }
+        versions.push([code, (diff) => diff]);
+      }
+      for (const [code, transform] of versions) {
+        const acc: CellDelta[] = [];
+        const perCol = perRow[rowId];
+        activeColsWithoutManualSort.forEach(col => {
+          const diff = perCol ? perCol[col] : null;
+          if (!diff) {
+            acc.push([null, null]);
+          } else {
+            acc.push(transform(diff));
+          }
+        });
+        tableChanges.cells.push([code, rowId, acc]);
+      }
+    }
+  }
+  return allChanges;
+}
+
+/**
+ * Return a suitable key for a removed table/column.  We cannot use their id directly
+ * since it could clash with an added table/column of the same name.
+ */
+export function defunctTableName(id: string): string {
+  return `-${id}`;
+}
+
+export function rootTableName(id: string): string {
+  return id.replace('-', '');
+}
+
+/**
+ * Returns a list of all tables changed by the summarized action.  Changes include
+ * schema or data changes.  Tables are identified by their post-action name.
+ * Deleted tables are identified by their pre-action name, with "-" prepended.
+ */
+export function getAffectedTables(summary: ActionSummary): string[] {
+  return [
+    // Tables added, renamed, or removed in this action.
+    ...summary.tableRenames.map(pair => pair[1] || defunctTableName(pair[0] || "")),
+    // Tables modified in this action.
+    ...Object.keys(summary.tableDeltas)
+  ];
+}

app/common/ActiveDocAPI.ts

@@ -0,0 +1,226 @@
+import {ActionGroup} from 'app/common/ActionGroup';
+import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
+import {Peer} from 'app/common/sharing';
+import {UploadResult} from 'app/common/uploads';
+import {ParseOptions} from 'app/plugin/FileParserAPI';
+import {IMessage} from 'grain-rpc';
+
+export interface ApplyUAOptions {
+  desc?: string;      // Overrides the description of the action.
+  otherId?: number;   // For undo/redo; the actionNum of the original action to which it applies.
+  linkId?: number;    // For bundled actions, actionNum of the previous action in the bundle.
+}
+
+export interface ApplyUAResult {
+  actionNum: number;      // number of the action that got recorded.
+  retValues: any[];       // array of return values, one for each of the passed-in user actions.
+  isModification: boolean; // true if document was modified.
+}
+
+export interface DataSourceTransformed {
+  // Identifies the upload, which may include multiple files.
+  uploadId: number;
+
+  // For each file in the upload, the transform rules for that file.
+  transforms: TransformRuleMap[];
+}
+
+export interface TransformRuleMap {
+  [origTableName: string]: TransformRule;
+}
+
+export interface TransformRule {
+  destTableId: string|null;
+  destCols: TransformColumn[];
+  sourceCols: string[];
+}
+
+export interface TransformColumn {
+  label: string;
+  colId: string|null;
+  type: string;
+  formula: string;
+}
+
+export interface ImportResult {
+  options: ParseOptions;
+  tables: ImportTableResult[];
+}
+
+export interface ImportTableResult {
+  hiddenTableId: string;
+  uploadFileIndex: number;      // Index into upload.files array, for the file reponsible for this table.
+  origTableName: string;
+  transformSectionRef: number;
+  destTableId: string|null;
+}
+
+/**
+ * Represents a query for Grist data. The tableId is required. An empty set of filters indicates
+ * the full table. Examples:
+ *    {tableId: "Projects", filters: {}}
+ *    {tableId: "Employees", filters: {Status: ["Active"], Dept: ["Sales", "HR"]}}
+ */
+export interface Query {
+  tableId: string;
+  filters: {
+    [colId: string]: any[];
+  };
+
+  // Queries to server for onDemand tables will set a limit to avoid bringing down the browser.
+  limit?: number;
+}
+
+/**
+ * Response from useQuerySet(). A query returns data AND creates a subscription to receive
+ * DocActions that affect this data. The querySubId field identifies this subscription, and must
+ * be used in a disposeQuerySet() call to unsubscribe.
+ */
+export interface QueryResult {
+  querySubId: number;     // ID of the subscription, to use with disposeQuerySet.
+  tableData: TableDataAction;
+}
+
+/**
+ * Result of a fork operation, with newly minted ids.
+ * For a document with docId XXXXX and urlId UUUUU, the fork will have a
+ * docId of XXXXX~FORKID[~USERID] and a urlId of UUUUU~FORKID[~USERID].
+ */
+export interface ForkResult {
+  docId: string;
+  urlId: string;
+}
+
+export interface ActiveDocAPI {
+  /**
+   * Closes a document, and unsubscribes from its userAction events.
+   */
+  closeDoc(): Promise<void>;
+
+  /**
+   * Fetches a particular table from the data engine to return to the client.
+   */
+  fetchTable(tableId: string): Promise<TableDataAction>;
+
+  /**
+   * Fetches the generated Python code for this document. (TODO rename this misnomer.)
+   */
+  fetchTableSchema(): Promise<string>;
+
+  /**
+   * Makes a query (documented elsewhere) and subscribes to it, so that the client receives
+   * docActions that affect this query's results. The subscription remains functional even when
+   * tables or columns get renamed.
+   */
+  useQuerySet(query: Query): Promise<QueryResult>;
+
+  /**
+   * Removes the subscription to a Query, identified by QueryResult.querySubId, so that the
+   * client stops receiving docActions relevant only to that query.
+   */
+  disposeQuerySet(querySubId: number): Promise<void>;
+
+  /**
+   * Applies an array of user actions to the document.
+   */
+  applyUserActions(actions: UserAction[], options?: ApplyUAOptions): Promise<ApplyUAResult>;
+
+  /**
+   * A variant of applyUserActions where actions are passed in by ids (actionNum, actionHash)
+   * rather than by value.
+   */
+  applyUserActionsById(actionNums: number[], actionHashes: string[],
+                       undo: boolean, options?: ApplyUAOptions): Promise<ApplyUAResult>;
+
+  /**
+   * Imports files, removes previously created temporary hidden tables and creates the new ones.
+   */
+  importFiles(dataSource: DataSourceTransformed,
+              parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
+
+  /**
+   * Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads.
+   */
+  finishImportFiles(dataSource: DataSourceTransformed,
+                    parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
+
+  /**
+   * Cancels import files, cleans up temporary hidden tables and uploads.
+   */
+  cancelImportFiles(dataSource: DataSourceTransformed, prevTableIds: string[]): Promise<void>;
+
+  /**
+   * Saves attachments from a given upload and creates an entry for them in the database. It
+   * returns the list of rowIds for the rows created in the _grist_Attachments table.
+   */
+  addAttachments(uploadId: number): Promise<number[]>;
+
+  /**
+   * Returns up to n columns in the document, or a specific table, which contain the given values.
+   * Columns are returned ordered from best to worst based on an estimate for number of matches.
+   */
+  findColFromValues(values: any[], n: number, optTableId?: string): Promise<number[]>;
+
+  /**
+   * Returns cell value with an error message (traceback) for one invalid formula cell.
+   */
+  getFormulaError(tableId: string, colId: string, rowId: number): Promise<CellValue>;
+
+  /**
+   * Fetch content at a url.
+   */
+  fetchURL(url: string): Promise<UploadResult>;
+
+  /**
+   * Find and return a list of auto-complete suggestions that start with `txt`, when editing a
+   * formula in table `tableId`.
+   */
+  autocomplete(txt: string, tableId: string): Promise<string[]>;
+
+  /**
+   * Shares the doc and invites peers.
+   */
+  shareDoc(peers: Peer[]): Promise<void>;
+
+  /**
+   * Removes the current instance from the doc.
+   */
+  removeInstanceFromDoc(): Promise<void>;
+
+  /**
+   * Get recent actions in ActionGroup format with summaries included.
+   */
+  getActionSummaries(): Promise<ActionGroup[]>;
+
+  /**
+   *  Initiates user actions bandling for undo.
+   */
+  startBundleUserActions(): Promise<void>;
+
+  /**
+   *  Stopes user actions bandling for undo.
+   */
+  stopBundleUserActions(): Promise<void>;
+
+  /**
+   * Forward a grain-rpc message to a given plugin.
+   */
+  forwardPluginRpc(pluginId: string, msg: IMessage): Promise<any>;
+
+  /**
+   * Reload documents plugins.
+   */
+  reloadPlugins(): Promise<void>;
+
+  /**
+   * Immediately close the document and data engine, to be reloaded from scratch, and cause all
+   * browser clients to reopen it.
+   */
+  reloadDoc(): Promise<void>;
+
+  /**
+   * Prepare a fork of the document, and return the id(s) of the fork.
+   * TODO: remove string option here, it is present to ease transition.
+   */
+  fork(): Promise<string | ForkResult>;
+}

app/common/ApiError.ts

@@ -0,0 +1,40 @@
+/**
+ * A tip for fixing an error.
+ */
+export interface ApiTip {
+  action: 'add-members' | 'upgrade' |'ask-for-help';
+  message: string;
+}
+
+/**
+ * Documentation of a limit relevant to an API error.
+ */
+export interface ApiLimit {
+  quantity: 'collaborators' | 'docs' | 'workspaces';  // what are we counting
+  subquantity?: string;    // a nuance to what we are counting
+  maximum: number;         // maximum allowed
+  value: number;           // current value of quantity for user
+  projectedValue: number;  // value of quantity expected if request had been allowed
+}
+
+/**
+ * Structured details about an API error.
+ */
+export interface ApiErrorDetails {
+  limit?: ApiLimit;
+
+  // If set, this is the more user-friendly message to show to the user than error.message.
+  userError?: string;
+
+  // If set, contains suggestions for fixing a problem.
+  tips?: ApiTip[];
+}
+
+/**
+ * An error with an http status code.
+ */
+export class ApiError extends Error {
+  constructor(message: string, public status: number, public details?: ApiErrorDetails) {
+    super(message);
+  }
+}

app/common/AsyncCreate.ts

@@ -0,0 +1,147 @@
+/**
+ * Implements a pattern for creating objects requiring asynchronous construction. The given
+ * asynchronous createFunc() is called on the .get() call, and the result is cached on success.
+ * On failure, the result is cleared, so that subsequent calls attempt the creation again.
+ *
+ * Usage:
+ *  this._obj = AsyncCreate<MyObject>(asyncCreateFunc);
+ *  obj = await this._obj.get();    // calls asyncCreateFunc
+ *  obj = await this._obj.get();    // uses cached object if asyncCreateFunc succeeded, else calls it again.
+ *
+ * Note that multiple calls while createFunc() is running will return the same promise, and will
+ * succeed or fail together.
+ */
+export class AsyncCreate<T> {
+  private _value?: Promise<T> = undefined;
+
+  constructor(private _createFunc: () => Promise<T>) {}
+
+  /**
+   * Returns createFunc() result, returning the cached promise if createFunc() succeeded, or if
+   * another call to it is currently pending.
+   */
+  public get(): Promise<T> {
+    return this._value || (this._value = this._clearOnError(this._createFunc.call(null)));
+  }
+
+  /** Clears the cached promise, forcing createFunc to be called again on next get(). */
+  public clear(): void {
+    this._value = undefined;
+  }
+
+  /** Returns a boolean indicating whether the object is created. */
+  public isSet(): boolean {
+    return Boolean(this._value);
+  }
+
+  /** Returns the value if it's set and successful, or undefined otherwise. */
+  public async getIfValid(): Promise<T|undefined> {
+    return this._value ? this._value.catch(() => undefined) : undefined;
+  }
+
+  // Helper which clears this AsyncCreate if the given promise is rejected.
+  private _clearOnError(p: Promise<T>): Promise<T> {
+    p.catch(() => this.clear());
+    return p;
+  }
+}
+
+/**
+ * Supports a usage similar to AsyncCreate in a Map. Returns map.get(key) if it is set to a
+ * resolved or pending promise. Otherwise, calls creator(key) to create and return a new promise,
+ * and sets the key to it. If the new promise is rejected, the key will be removed from the map,
+ * so that subsequent calls would call creator() again.
+ *
+ * As with AsyncCreate, while the promise for a key is pending, multiple calls to that key will
+ * return the same promise, and will succeed or fail together.
+ */
+export function mapGetOrSet<K, V>(map: Map<K, Promise<V>>, key: K, creator: (key: K) => Promise<V>): Promise<V> {
+  return map.get(key) || mapSetOrClear(map, key, creator(key));
+}
+
+/**
+ * Supports a usage similar to AsyncCreate in a Map. Sets the given key in a map to the given
+ * promise, and removes it later if the promise is rejected. Returns the same promise.
+ */
+export function mapSetOrClear<K, V>(map: Map<K, Promise<V>>, key: K, pvalue: Promise<V>): Promise<V> {
+  pvalue.catch(() => map.delete(key));
+  map.set(key, pvalue);
+  return pvalue;
+}
+
+/**
+ * A Map implementation that allows for expiration of old values.
+ */
+export class MapWithTTL<K, V> extends Map<K, V> {
+  private _timeouts = new Map<K, NodeJS.Timer>();
+
+  /**
+   * Create a map with keys that will be automatically deleted _ttlMs
+   * milliseconds after they have been last set.  Precision of timing
+   * may vary.
+   */
+  constructor(private _ttlMs: number) {
+    super();
+  }
+
+  /**
+   * Set a key, with expiration.
+   */
+  public set(key: K, value: V): this {
+    return this.setWithCustomTTL(key, value, this._ttlMs);
+  }
+
+  /**
+   * Set a key, with custom expiration.
+   */
+  public setWithCustomTTL(key: K, value: V, ttlMs: number): this {
+    const curr = this._timeouts.get(key);
+    if (curr) { clearTimeout(curr); }
+    super.set(key, value);
+    this._timeouts.set(key, setTimeout(this.delete.bind(this, key), ttlMs));
+    return this;
+  }
+
+  /**
+   * Remove a key.
+   */
+  public delete(key: K): boolean {
+    const result = super.delete(key);
+    const timeout = this._timeouts.get(key);
+    if (timeout) {
+      clearTimeout(timeout);
+      this._timeouts.delete(key);
+    }
+    return result;
+  }
+
+  /**
+   * Forcibly expire everything.
+   */
+  public clear(): void {
+    for (const timeout of this._timeouts.values()) {
+      clearTimeout(timeout);
+    }
+    this._timeouts.clear();
+    super.clear();
+  }
+}
+
+/**
+ * Sometimes it is desirable to cache either fulfilled or rejected
+ * outcomes.  This method wraps a promise so that it never throws.
+ * The result has an unfreeze method which, when called, is either
+ * fulfilled or rejected.
+ */
+export async function freezeError<T>(promise: Promise<T>): Promise<ErrorOrValue<T>> {
+  try {
+    const value = await promise;
+    return { unfreeze: async () => value };
+  } catch (error) {
+    return { unfreeze: async () => { throw error; } };
+  }
+}
+
+export interface ErrorOrValue<T> {
+  unfreeze(): Promise<T>;
+}

app/common/AsyncFlow.ts

@@ -0,0 +1,83 @@
+/**
+ * This module is a helper for asynchronous work. It allows resources acquired asynchronously to
+ * be conveniently and reliably released.
+ *
+ * Usage:
+ * (1) Implement a function `myFunc(flow: AsyncFlow)`. The `flow` argument provides some helpers:
+ *
+ *      // Create a disposable, making it owned by the flow. It will be disposed when the flow
+ *      // ends, whether successfully, on error, or by being cancelled.
+ *      const foo = Foo.create(flow, ...);
+ *
+ *      // As with Disposables in general, schedule a callback to be called when the flow ends.
+ *      flow.onDispose(...);
+ *
+ *      // Release foo from the flow's ownership, and give its ownership to another object. This way
+ *      // `other` will be responsible for disposing foo, and not flow.
+ *      other.autoDispose(flow.release(foo))
+ *
+ *      // Abort the flow (by throwing CancelledError) if cancellation is requested. This should
+ *      // be called after async work, in case the flow shouldn't be continued.
+ *      checkIfCancelled();
+ *
+ * (2) Call `runner = FlowRunner.create(owner, myFunc)`. The flow will start. Once myFunc's
+ *     promise resolves (including on failure), the objects owned by the flow will be disposed.
+ *
+ *     The runner exposes the promise for when the flow ends as `runner.resultPromise`.
+ *
+ *     If the runner itself is disposed, the flow will be cancelled, and disposed once it notices
+ *     the cancellation.
+ *
+ * To replace one FlowRunner with another, put it in a grainjs Holder.
+ */
+import {Disposable, IDisposable} from 'grainjs';
+
+type DisposeListener = ReturnType<Disposable["onDispose"]>;
+
+export class CancelledError extends Error {}
+
+export class FlowRunner extends Disposable {
+  public resultPromise: Promise<void>;
+
+  constructor(func: (flow: AsyncFlow) => Promise<void>) {
+    super();
+    const flow = AsyncFlow.create(null);
+    async function runFlow() {
+      try {
+        return await func(flow);
+      } finally {
+        flow.dispose();
+      }
+    }
+    this.resultPromise = runFlow();
+    this.onDispose(flow.cancel, flow);
+  }
+}
+
+export class AsyncFlow extends Disposable {
+  private _handles = new Map<IDisposable, DisposeListener>();
+  private _isCancelled = false;
+
+  public autoDispose<T extends IDisposable>(obj: T): T {
+    const lis = this.onDispose(obj.dispose, obj);
+    this._handles.set(obj, lis);
+    return obj;
+  }
+
+  public release<T extends IDisposable>(obj: T): T {
+    const h = this._handles.get(obj);
+    if (h) { h.dispose(); }
+    this._handles.delete(obj);
+    return obj;
+  }
+
+  public checkIfCancelled() {
+    if (this._isCancelled) {
+      throw new CancelledError('cancelled');
+    }
+  }
+
+  public cancel() {
+    this._isCancelled = true;
+  }
+}

app/common/BaseAPI.ts

@@ -0,0 +1,124 @@
+import {ApiError, ApiErrorDetails} from 'app/common/ApiError';
+import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
+import {tbind} from './tbind';
+
+export type ILogger = Pick<Console, 'log'|'debug'|'info'|'warn'|'error'>;
+
+export interface IOptions {
+  headers?: Record<string, string>;
+  fetch?: typeof fetch;
+  newFormData?: () => FormData;  // constructor for FormData depends on platform.
+  logger?: ILogger;
+  extraParameters?: Map<string, string>;  // if set, add query parameters to requests.
+}
+
+/**
+ * Base setup class for creating a REST API client interface.
+ */
+export class BaseAPI {
+  // Count of pending requests. It is relied on by tests.
+  public static numPendingRequests(): number { return this._numPendingRequests; }
+
+  // Wrap a promise to add to the count of pending requests until the promise is resolved.
+  public static async countPendingRequest<T>(promise: Promise<T>): Promise<T> {
+    try {
+      BaseAPI._numPendingRequests++;
+      return await promise;
+    } finally {
+      BaseAPI._numPendingRequests--;
+    }
+  }
+
+  // Define a decorator for methods in BaseAPI or derived classes.
+  public static countRequest(target: unknown, propertyKey: string, descriptor: PropertyDescriptor) {
+    const originalMethod = descriptor.value;
+    descriptor.value = async function(...args: any[]) {
+      return BaseAPI.countPendingRequest(originalMethod.apply(this, args));
+    };
+  }
+
+  private static _numPendingRequests: number = 0;
+
+  protected fetch: typeof fetch;
+  protected newFormData: () => FormData;
+  private _headers: Record<string, string>;
+  private _logger: ILogger;
+  private _extraParameters?: Map<string, string>;
+
+  constructor(options: IOptions = {}) {
+    this.fetch = options.fetch || tbind(window.fetch, window);
+    this.newFormData = options.newFormData || (() => new FormData());
+    this._logger = options.logger || console;
+    this._headers = {
+      'Content-Type': 'application/json',
+      ...options.headers
+    };
+    this._extraParameters = options.extraParameters;
+  }
+
+  // Make a modified request, exposed for test convenience.
+  public async testRequest(url: string, init: RequestInit = {}): Promise<Response> {
+    return this.request(url, init);
+  }
+
+  // Similar to request, but uses the axios library, and supports progress indicator.
+  @BaseAPI.countRequest
+  protected async requestAxios(url: string, config: AxiosRequestConfig): Promise<AxiosResponse> {
+    // If using with FormData in node, axios needs the headers prepared by FormData.
+    let headers = config.headers;
+    if (config.data && typeof config.data.getHeaders === 'function') {
+      headers = {...config.data.getHeaders(), ...headers};
+    }
+    const resp = await axios.request({
+      url,
+      withCredentials: true,
+      validateStatus: (status) => true,     // This is more like fetch
+      ...config,
+      headers,
+    });
+    if (resp.status !== 200) {
+      throwApiError(url, resp, resp.data);
+    }
+    return resp;
+  }
+
+  @BaseAPI.countRequest
+  protected async request(input: string, init: RequestInit = {}): Promise<Response> {
+    init = Object.assign({ headers: this._headers, credentials: 'include' }, init);
+    if (this._extraParameters) {
+      const url = new URL(input);
+      for (const [key, val] of this._extraParameters.entries()) {
+        url.searchParams.set(key, val);
+        input = url.href;
+      }
+    }
+    const resp = await this.fetch(input, init);
+    this._logger.log("Fetched", input);
+    if (resp.status !== 200) {
+      const body = await resp.json().catch(() => ({}));
+      throwApiError(input, resp, body);
+    }
+    return resp;
+  }
+
+  /**
+   * Make a request, and read the response as JSON. This allows counting the request as pending
+   * until it has been read, which is relied on by tests.
+   */
+  @BaseAPI.countRequest
+  protected async requestJson(input: string, init: RequestInit = {}): Promise<any> {
+    return (await this.request(input, init)).json();
+  }
+}
+
+function throwApiError(url: string, resp: Response | AxiosResponse, body: any) {
+  // If the response includes details, include them into the ApiError we construct. Include
+  // also the error message from the server as details.userError. It's used by the Notifier.
+  if (!body) { body = {}; }
+  const details: ApiErrorDetails = body.details && typeof body.details === 'object' ? body.details : {};
+  if (body.error) {
+    details.userError = body.error;
+  }
+  throw new ApiError(`Request to ${url} failed with status ${resp.status}: ` +
+    `${resp.statusText} (${body.error || 'unknown cause'})`, resp.status, details);
+}

app/common/BasketClientAPI.ts

@@ -0,0 +1,11 @@
+export interface BasketClientAPI {
+  /**
+   * Returns an array of all tableIds in this basket.
+   */
+  getBasketTables(): Promise<string[]>;
+
+  /**
+   * Adds, updates or deletes a table's data to/from Grist Basket.
+   */
+  embedTable(tableId: string, action: "add"|"update"|"delete"): Promise<void>;
+}

app/common/BigInt.ts

@@ -0,0 +1,72 @@
+/**
+ * A minimal library to represent arbitrarily large integers. Unlike the many third party
+ * libraries, which are big, this only implements a representation and conversion to string (such
+ * as base 10 or base 16), so it's tiny in comparison.
+ *
+ * Big integers
+ *    base: number - the base for the digits
+ *    digits: number[] - digits, from least significant to most significant, in [0, base) range.
+ *    sign: number - 1 or -1
+ */
+export class BigInt {
+  constructor(
+    private _base: number,      // Base for the digits
+    private _digits: number[],  // Digits from least to most significant, in [0, base) range.
+    private _sign: number,      // +1 or -1
+  ) {}
+
+  public copy() { return new BigInt(this._base, this._digits, this._sign); }
+
+  /** Convert to Number if there is no loss of precision, or string (base 10) otherwise. */
+  public toNative(): number|string {
+    const num = this.toNumber();
+    return Number.isSafeInteger(num) ? num : this.toString(10);
+  }
+
+  /** Convert to Number as best we can. This will lose precision beying 53 bits. */
+  public toNumber(): number {
+    let res = 0;
+    let baseFactor = 1;
+    for (const digit of this._digits) {
+      res += digit * baseFactor;
+      baseFactor *= this._base;
+    }
+    return res * (this._sign < 0 ? -1 : 1);
+  }
+
+  /** Like Number.toString(). Radix (or base) is an integer between 2 and 36, defaulting to 10. */
+  public toString(radix: number = 10): string {
+    const copy = this.copy();
+    const decimals = [];
+    while (copy._digits.length > 0) {
+      decimals.push(copy._mod(radix).toString(radix));
+      copy._divide(radix);
+    }
+    if (decimals.length === 0) { return "0"; }
+    return (this._sign < 0 ? "-" : "") + decimals.reverse().join("");
+  }
+
+  /** Returns the remainder when this number is divided by divisor. */
+  private _mod(divisor: number): number {
+    let res = 0;
+    let baseFactor = 1;
+    for (const digit of this._digits) {
+      res = (res + (digit % divisor) * baseFactor) % divisor;
+      baseFactor = (baseFactor * this._base) % divisor;
+    }
+    return res;
+  }
+
+  /** Divides this number in-place. */
+  private _divide(divisor: number): void {
+    if (this._digits.length === 0) { return; }
+    for (let i = this._digits.length - 1; i > 0; i--) {
+      this._digits[i - 1] += (this._digits[i] % divisor) * this._base;
+      this._digits[i] = Math.floor(this._digits[i] / divisor);
+    }
+    this._digits[0] = Math.floor(this._digits[0] / divisor);
+    while (this._digits.length > 0 && this._digits[this._digits.length - 1] === 0) {
+      this._digits.pop();
+    }
+  }
+}

app/common/BillingAPI.ts

@@ -0,0 +1,212 @@
+import {BaseAPI, IOptions} from 'app/common/BaseAPI';
+import {FullUser} from 'app/common/LoginSessionAPI';
+import {StringUnion} from 'app/common/StringUnion';
+import {addCurrentOrgToPath} from 'app/common/urlUtils';
+import {BillingAccount, ManagerDelta, OrganizationWithoutAccessInfo} from 'app/common/UserAPI';
+
+export const BillingSubPage = StringUnion('payment', 'plans');
+export type BillingSubPage = typeof BillingSubPage.type;
+
+export const BillingPage = StringUnion(...BillingSubPage.values, 'billing');
+export type BillingPage = typeof BillingPage.type;
+
+export const BillingTask = StringUnion('signUp', 'updatePlan', 'addCard', 'updateCard', 'updateAddress');
+export type BillingTask = typeof BillingTask.type;
+
+// Note that IBillingPlan includes selected fields from the Stripe plan object along with
+// custom metadata fields that are present on plans we store in Stripe.
+// For reference: https://stripe.com/docs/api/plans/object
+export interface IBillingPlan {
+  id: string;                 // the Stripe plan id
+  nickname: string;
+  currency: string;           // lowercase three-letter ISO currency code
+  interval: string;           // billing frequency - one of day, week, month or year
+  amount: number;             // amount in cents charged at each interval
+  metadata: {
+    family?: string;          // groups plans for filtering by GRIST_STRIPE_FAMILY env variable
+    isStandard: boolean;      // indicates that the plan should be returned by the API to be offered.
+    supportAvailable: boolean;
+    gristProduct: string;     // name of grist product that should be used with this plan.
+    unthrottledApi: boolean;
+    customSubdomain: boolean;
+    workspaces: boolean;
+    maxDocs?: number;         // if given, limit of docs that can be created
+    maxUsersPerDoc?: number;  // if given, limit of users each doc can be shared with
+  };
+  trial_period_days: number|null;  // Number of days in the trial period, or null if there is none.
+  product: string;         // the Stripe product id.
+}
+
+// Stripe customer address information. Used to maintain the company address.
+// For reference: https://stripe.com/docs/api/customers/object#customer_object-address
+export interface IBillingAddress {
+  line1: string;
+  line2?: string;
+  city?: string;
+  state?: string;
+  postal_code?: string;
+  country?: string;
+}
+
+export interface IBillingCard {
+  funding?: 'credit'|'debit'|'prepaid'|'unknown';
+  brand?: string;
+  country?: string;         // uppercase two-letter ISO country code
+  last4?: string;           // last 4 digits of the card number
+  name?: string|null;
+}
+
+export interface IBillingSubscription {
+  // All standard plan options.
+  plans: IBillingPlan[];
+  // Index in the plans array of the plan currently in effect.
+  planIndex: number;
+  // Index in the plans array of the plan to be in effect after the current period end.
+  // Equal to the planIndex when the plan has not been downgraded or cancelled.
+  upcomingPlanIndex: number;
+  // Timestamp in milliseconds indicating when the current plan period ends.
+  // Null if the account is not signed up with Stripe.
+  periodEnd: number|null;
+  // Whether the subscription is in the trial period.
+  isInTrial: boolean;
+  // Value in cents remaining for the current subscription. This indicates the amount that
+  // will be discounted from a subscription upgrade.
+  valueRemaining: number;
+  // The payment card, or null if none is attached.
+  card: IBillingCard|null;
+  // The company address.
+  address: IBillingAddress|null;
+  // The effective tax rate of the customer for the given address.
+  taxRate: number;
+  // The current number of users with whom the paid org is shared.
+  userCount: number;
+  // The next total in cents that Stripe is going to charge (includes tax and discount).
+  nextTotal: number;
+  // Name of the discount if any.
+  discountName: string|null;
+  // Last plan we had a subscription for, if any.
+  lastPlanId: string|null;
+  // Whether there is a valid plan in effect
+  isValidPlan: boolean;
+}
+
+export interface IBillingOrgSettings {
+  name: string;
+  domain: string;
+}
+
+// Full description of billing account, including nested list of orgs and managers.
+export interface FullBillingAccount extends BillingAccount {
+  orgs: OrganizationWithoutAccessInfo[];
+  managers: FullUser[];
+}
+
+export interface BillingAPI {
+  isDomainAvailable(domain: string): Promise<boolean>;
+  getTaxRate(address: IBillingAddress): Promise<number>;
+  getPlans(): Promise<IBillingPlan[]>;
+  getSubscription(): Promise<IBillingSubscription>;
+  getBillingAccount(): Promise<FullBillingAccount>;
+  // The signUp function takes the tokenId generated when card data is submitted to Stripe.
+  // See: https://stripe.com/docs/stripe-js/reference#stripe-create-token
+  signUp(planId: string, tokenId: string, address: IBillingAddress,
+         settings: IBillingOrgSettings): Promise<OrganizationWithoutAccessInfo>;
+  setCard(tokenId: string): Promise<void>;
+  removeCard(): Promise<void>;
+  setSubscription(planId: string, tokenId?: string): Promise<void>;
+  updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise<void>;
+  updateBillingManagers(delta: ManagerDelta): Promise<void>;
+}
+
+export class BillingAPIImpl extends BaseAPI implements BillingAPI {
+  constructor(private _homeUrl: string, options: IOptions = {}) {
+    super(options);
+  }
+
+  public async isDomainAvailable(domain: string): Promise<boolean> {
+    const resp = await this.request(`${this._url}/api/billing/domain`, {
+      method: 'POST',
+      body: JSON.stringify({ domain })
+    });
+    return resp.json();
+  }
+
+  public async getTaxRate(address: IBillingAddress): Promise<number> {
+    const resp = await this.request(`${this._url}/api/billing/tax`, {
+      method: 'POST',
+      body: JSON.stringify({ address })
+    });
+    return resp.json();
+  }
+
+  public async getPlans(): Promise<IBillingPlan[]> {
+    const resp = await this.request(`${this._url}/api/billing/plans`, {method: 'GET'});
+    return resp.json();
+  }
+
+  // Returns an IBillingSubscription
+  public async getSubscription(): Promise<IBillingSubscription> {
+    const resp = await this.request(`${this._url}/api/billing/subscription`, {method: 'GET'});
+    return resp.json();
+  }
+
+  public async getBillingAccount(): Promise<FullBillingAccount> {
+    const resp = await this.request(`${this._url}/api/billing`, {method: 'GET'});
+    return resp.json();
+  }
+
+  // Returns the new Stripe customerId.
+  public async signUp(
+    planId: string,
+    tokenId: string,
+    address: IBillingAddress,
+    settings: IBillingOrgSettings
+  ): Promise<OrganizationWithoutAccessInfo> {
+    const resp = await this.request(`${this._url}/api/billing/signup`, {
+      method: 'POST',
+      body: JSON.stringify({ tokenId, planId, address, settings })
+    });
+    const parsed = await resp.json();
+    return parsed.data;
+  }
+
+  public async setSubscription(planId: string, tokenId?: string): Promise<void> {
+    await this.request(`${this._url}/api/billing/subscription`, {
+      method: 'POST',
+      body: JSON.stringify({ tokenId, planId })
+    });
+  }
+
+  public async removeSubscription(): Promise<void> {
+    await this.request(`${this._url}/api/billing/subscription`, {method: 'DELETE'});
+  }
+
+  public async setCard(tokenId: string): Promise<void> {
+    await this.request(`${this._url}/api/billing/card`, {
+      method: 'POST',
+      body: JSON.stringify({ tokenId })
+    });
+  }
+
+  public async removeCard(): Promise<void> {
+    await this.request(`${this._url}/api/billing/card`, {method: 'DELETE'});
+  }
+
+  public async updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise<void> {
+    await this.request(`${this._url}/api/billing/address`, {
+      method: 'POST',
+      body: JSON.stringify({ address, settings })
+    });
+  }
+
+  public async updateBillingManagers(delta: ManagerDelta): Promise<void> {
+    await this.request(`${this._url}/api/billing/managers`, {
+      method: 'PATCH',
+      body: JSON.stringify({delta})
+    });
+  }
+
+  private get _url(): string {
+    return addCurrentOrgToPath(this._homeUrl);
+  }
+}

app/common/BinaryIndexedTree.js

@@ -0,0 +1,258 @@
+/**
+ * Implements a binary indexed tree, aka Fenwick tree. See
+ * http://en.wikipedia.org/wiki/Fenwick_tree
+ */
+function BinaryIndexedTree(optSize) {
+  this.tree = [];
+  if (optSize > 0) {
+    this.tree.length = optSize;
+    for (var i = 0; i < optSize; i++) {
+      this.tree[i] = 0;
+    }
+    // The last valid index rounded down to the nearest power of 2.
+    this.mask = mostSignificantOne(this.tree.length - 1);
+  }
+}
+
+/**
+ * Returns a number that contains only the least significant one in `num`.
+ * @param {Number} num - Positive integer.
+ * @returns {Number} The least significant one in `num`, e.g. for 10110, returns 00010.
+ */
+function leastSignificantOne(num) {
+  return num & (-num);
+}
+BinaryIndexedTree.leastSignificantOne = leastSignificantOne;
+
+
+/**
+ * Strips the least significant one from `num`.
+ * @param {Number} num - Positive integer.
+ * @returns {Number} `num` with the least significant one removed, e.g. for 10110, returns 10100.
+ */
+function stripLeastSignificantOne(num) {
+  return num & (num - 1);
+}
+BinaryIndexedTree.stripLeastSignificantOne = stripLeastSignificantOne;
+
+
+function mostSignificantOne(num) {
+  if (num === 0) {
+    return 0;
+  }
+  var msb = 1;
+  while ((num >>>= 1)) {
+    msb <<= 1;
+  }
+  return msb;
+}
+BinaryIndexedTree.mostSignificantOne = mostSignificantOne;
+
+/**
+ * Converts in-place an array of cumulative values to the original values.
+ * @param {Array[Number]} values - Array of cumulative values, or partial sums.
+ * @returns {Array[Number]} - same `values` array, with elements replaced by deltas.
+ *      E.g. [1,3,6,10] is converted to [1,2,3,4].
+ */
+function cumulToValues(values) {
+  for (var i = values.length - 1; i >= 1; i--) {
+    values[i] -= values[i - 1];
+  }
+  return values;
+}
+BinaryIndexedTree.cumulToValues = cumulToValues;
+
+
+/**
+ * Converts in-place an array of values to cumulative values, or partial sums.
+ * @param {Array[Number]} values - Array of numerical values.
+ * @returns {Array[Number]} - same `values` array, with elements replaced by partial sums.
+ *      E.g. [1,2,3,4] is converted to [1,3,6,10].
+ */
+function valuesToCumul(values) {
+  for (var i = 1; i < values.length; i++) {
+    values[i] += values[i - 1];
+  }
+  return values;
+}
+BinaryIndexedTree.valuesToCumul = valuesToCumul;
+
+
+/**
+ * @returns {Number} length of the tree.
+ */
+BinaryIndexedTree.prototype.size = function() {
+  return this.tree.length;
+};
+
+
+/**
+ * Converts the BinaryIndexedTree to a cumulative array.
+ * Takes time linear in the size of the array.
+ * @returns {Array[Number]} - array with each element a partial sum.
+ */
+BinaryIndexedTree.prototype.toCumulativeArray = function() {
+  var cumulValues = [this.tree[0]];
+  var len = cumulValues.length = this.tree.length;
+  for (var i = 1; i < len; i++) {
+    cumulValues[i] = this.tree[i] + cumulValues[stripLeastSignificantOne(i)];
+  }
+  return cumulValues;
+};
+
+
+/**
+ * Converts the BinaryIndexedTree to an array of individual values.
+ * Takes time linear in the size of the array.
+ * @returns {Array[Number]} - array with each element containing the value that was inserted.
+ */
+BinaryIndexedTree.prototype.toValueArray = function() {
+  return cumulToValues(this.toCumulativeArray());
+};
+
+
+/**
+ * Creates a tree from an array of cumulative values.
+ * Takes time linear in the size of the array.
+ * @param {Array[Number]} - array with each element a partial sum.
+ */
+BinaryIndexedTree.prototype.fillFromCumulative = function(cumulValues) {
+  var len = this.tree.length = cumulValues.length;
+  if (len > 0) {
+    this.tree[0] = cumulValues[0];
+    for (var i = 1; i < len; i++) {
+      this.tree[i] = cumulValues[i] - cumulValues[stripLeastSignificantOne(i)];
+    }
+    // The last valid index rounded down to the nearest power of 2.
+    this.mask = mostSignificantOne(this.tree.length - 1);
+  } else {
+    this.mask = 0;
+  }
+};
+
+
+/**
+ * Creates a tree from an array of invididual values.
+ * Takes time linear in the size of the array.
+ * @param {Array[Number]} - array with each element containing the value to insert.
+ */
+BinaryIndexedTree.prototype.fillFromValues = function(values) {
+  this.fillFromCumulative(valuesToCumul(values.slice()));
+};
+
+
+/**
+ * Reads the cumulative value at the given index. Takes time O(log(index)).
+ * @param {Number} index - index in the array.
+ * @returns {Number} - cumulative values up to and including `index`.
+ */
+BinaryIndexedTree.prototype.getCumulativeValue = function(index) {
+  var sum = this.tree[0];
+  while (index > 0) {
+    sum += this.tree[index];
+    index = stripLeastSignificantOne(index);
+  }
+  return sum;
+};
+
+/**
+ * Reads the cumulative value from start(inclusive) to end(exclusive). Takes time O(log(end)).
+ * @param {Number} start - start index
+ * @param {Number} end - end index
+ * @returns {Number} - cumulative values between start(inclusive) and end(exclusive)
+ */
+BinaryIndexedTree.prototype.getCumulativeValueRange = function(start, end) {
+  return this.getSumTo(end) - this.getSumTo(start);
+};
+
+/**
+ * Returns the sum of values up to the given index. Takes time O(log(index)).
+ * @param {Number} index - index in the array.
+ * @returns {Number} - cumulative values up to but not including `index`.
+ */
+BinaryIndexedTree.prototype.getSumTo = function(index) {
+  return (index > 0 ? this.getCumulativeValue(index - 1) : 0);
+};
+
+
+/**
+ * Returns the total of all values in the tree. Takes time O(log(N)).
+ * @returns {Number} - sum of all values.
+ */
+BinaryIndexedTree.prototype.getTotal = function() {
+  return this.getCumulativeValue(this.tree.length - 1);
+};
+
+
+/**
+ * Reads a single value at the given index. Takes time O(log(index)).
+ * @param {Number} index - index in the array.
+ * @returns {Number} - the value that was inserted at `index`.
+ */
+BinaryIndexedTree.prototype.getValue = function(index) {
+  var value = this.tree[index];
+  if (index > 0) {
+    var parent = stripLeastSignificantOne(index);
+    index--;
+    while (index !== parent) {
+      value -= this.tree[index];
+      index = stripLeastSignificantOne(index);
+    }
+  }
+  return value;
+};
+
+
+/**
+ * Updates a value at an index. Takes time O(log(table size)).
+ * @param {Number} index - index in the array.
+ * @param {Number} delta - value to add to the previous value at `index`.
+ */
+BinaryIndexedTree.prototype.addValue = function(index, delta) {
+  if (index === 0) {
+    this.tree[0] += delta;
+  } else {
+    while (index < this.tree.length) {
+      this.tree[index] += delta;
+      index += leastSignificantOne(index);
+    }
+  }
+};
+
+
+/**
+ * Sets a value at an index. Takes time O(log(table size)).
+ * @param {Number} index - index in the array.
+ * @param {Number} value - new value to set at `index`.
+ */
+BinaryIndexedTree.prototype.setValue = function(index, value) {
+  this.addValue(index, value - this.getValue(index));
+};
+
+
+/**
+ * Given a cumulative value, finds the first element whose inclusion reaches the value.
+ * E.g. for values [1,2,3,4] (cumulative [1,3,6,10]), getIndex(3) = 1, getIndex(3.1) = 2.
+ * @param {Number} cumulValue - cumulative value to exceed.
+ * @returns {Number} index - the first index such that getCumulativeValue(index) >= cumulValue.
+ *    If cumulValue is too large, return one more than the highest valid index.
+ */
+BinaryIndexedTree.prototype.getIndex = function(cumulValue) {
+  if (this.tree.length === 0 || this.tree[0] >= cumulValue) {
+    return 0;
+  }
+  var index = 0;
+  var mask = this.mask;
+  var sum = this.tree[0];
+  while (mask !== 0) {
+    var testIndex = index + mask;
+    if (testIndex < this.tree.length && sum + this.tree[testIndex] < cumulValue) {
+      index = testIndex;
+      sum += this.tree[index];
+    }
+    mask >>>= 1;
+  }
+  return index + 1;
+};
+
+module.exports = BinaryIndexedTree;

app/common/BrowserSettings.ts

@@ -0,0 +1,7 @@
+/**
+ * Describes the settings that a browser sends to the server.
+ */
+export interface BrowserSettings {
+  // The browser's timezone, must be one of `momet.tz.names()`.
+  timezone?: string;
+}

app/common/ColumnGetters.ts

@@ -0,0 +1,26 @@
+/**
+ *
+ * An interface for accessing the columns of a table by their
+ * ID in _grist_Tables_column, which is the ID used in sort specifications.
+ * Implementations of this interface can be supplied to SortFunc to
+ * sort the rows of a table according to such a specification.
+ *
+ */
+export interface ColumnGetters {
+
+  /**
+   *
+   * Takes a _grist_Tables_column ID and returns a function that maps
+   * rowIds to values for that column.  Those values should be display
+   * values if available, drawn from a corresponding display column.
+   *
+   */
+  getColGetter(colRef: number): ((rowId: number) => any) | null;
+
+  /**
+   *
+   * Returns a getter for the manual sort column if it is available.
+   *
+   */
+  getManualSortGetter(): ((rowId: number) => any) | null;
+}

app/common/DisposableWithEvents.ts

@@ -0,0 +1,28 @@
+/**
+ * A base class which combines grainjs Disposable with mixed-in backbone Events. It includes the
+ * backbone Events methods, and when disposed, stops backbone listeners started with listenTo().
+ */
+import {Events as BackboneEvents, EventsHash} from 'backbone';
+import {Disposable} from 'grainjs';
+
+// In Typescript, mixins are awkward. This follows the recommendation here
+// https://www.typescriptlang.org/docs/handbook/mixins.html
+export class DisposableWithEvents extends Disposable implements BackboneEvents {
+  public on: (eventName: string|EventsHash, callback?: (...args: any[]) => void, context?: any) => any;
+  public off: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any;
+  public trigger: (eventName: string, ...args: any[]) => any;
+  public bind: (eventName: string, callback: (...args: any[]) => void, context?: any) => any;
+  public unbind: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any;
+
+  public once: (events: string, callback: (...args: any[]) => void, context?: any) => any;
+  public listenTo: (object: any, events: string, callback: (...args: any[]) => void) => any;
+  public listenToOnce: (object: any, events: string, callback: (...args: any[]) => void) => any;
+  public stopListening: (object?: any, events?: string, callback?: (...args: any[]) => void) => any;
+
+  // DisposableWithEvents knows also how to stop any backbone listeners started with listenTo().
+  constructor() {
+    super();
+    this.onDispose(this.stopListening, this);
+  }
+}
+Object.assign(DisposableWithEvents.prototype, BackboneEvents);

app/common/DocActions.ts

@@ -0,0 +1,148 @@
+/**
+ * This mirrors action definitions from sandbox/grist/actions.py
+ */
+
+import map = require('lodash/map');
+
+export type AddRecord = ['AddRecord', string, number, ColValues];
+export type BulkAddRecord = ['BulkAddRecord', string, number[], BulkColValues];
+export type RemoveRecord = ['RemoveRecord', string, number];
+export type BulkRemoveRecord = ['BulkRemoveRecord', string, number[]];
+export type UpdateRecord = ['UpdateRecord', string, number, ColValues];
+export type BulkUpdateRecord = ['BulkUpdateRecord', string, number[], BulkColValues];
+
+export type ReplaceTableData = ['ReplaceTableData', string, number[], BulkColValues];
+
+// This is the format in which data comes when we fetch a table from the sandbox.
+export type TableDataAction = ['TableData', string, number[], BulkColValues];
+
+export type AddColumn = ['AddColumn', string, string, ColInfo];
+export type RemoveColumn = ['RemoveColumn', string, string];
+export type RenameColumn = ['RenameColumn', string, string, string];
+export type ModifyColumn = ['ModifyColumn', string, string, ColInfo];
+
+export type AddTable = ['AddTable', string, ColInfoWithId[]];
+export type RemoveTable = ['RemoveTable', string];
+export type RenameTable = ['RenameTable', string, string];
+
+export type DocAction = (
+  AddRecord |
+  BulkAddRecord |
+  RemoveRecord |
+  BulkRemoveRecord |
+  UpdateRecord |
+  BulkUpdateRecord |
+  ReplaceTableData |
+  TableDataAction |
+  AddColumn |
+  RemoveColumn |
+  RenameColumn |
+  ModifyColumn |
+  AddTable |
+  RemoveTable |
+  RenameTable
+);
+
+// type guards for convenience - see:
+//   https://www.typescriptlang.org/docs/handbook/advanced-types.html#user-defined-type-guards
+export function isAddRecord(act: DocAction): act is AddRecord { return act[0] === 'AddRecord'; }
+export function isBulkAddRecord(act: DocAction): act is BulkAddRecord { return act[0] === 'BulkAddRecord'; }
+export function isRemoveRecord(act: DocAction): act is RemoveRecord { return act[0] === 'RemoveRecord'; }
+export function isBulkRemoveRecord(act: DocAction): act is BulkRemoveRecord { return act[0] === 'BulkRemoveRecord'; }
+export function isUpdateRecord(act: DocAction): act is UpdateRecord { return act[0] === 'UpdateRecord'; }
+export function isBulkUpdateRecord(act: DocAction): act is BulkUpdateRecord { return act[0] === 'BulkUpdateRecord'; }
+
+export function isReplaceTableData(act: DocAction): act is ReplaceTableData { return act[0] === 'ReplaceTableData'; }
+
+export function isAddColumn(act: DocAction): act is AddColumn { return act[0] === 'AddColumn'; }
+export function isRemoveColumn(act: DocAction): act is RemoveColumn { return act[0] === 'RemoveColumn'; }
+export function isRenameColumn(act: DocAction): act is RenameColumn { return act[0] === 'RenameColumn'; }
+export function isModifyColumn(act: DocAction): act is ModifyColumn { return act[0] === 'ModifyColumn'; }
+
+export function isAddTable(act: DocAction): act is AddTable { return act[0] === 'AddTable'; }
+export function isRemoveTable(act: DocAction): act is RemoveTable { return act[0] === 'RemoveTable'; }
+export function isRenameTable(act: DocAction): act is RenameTable { return act[0] === 'RenameTable'; }
+
+
+const SCHEMA_ACTIONS = new Set(['AddTable', 'RemoveTable', 'RenameTable', 'AddColumn',
+  'RemoveColumn', 'RenameColumn', 'ModifyColumn']);
+
+/**
+ * Determines whether a given action is a schema action or not.
+ */
+export function isSchemaAction(action: DocAction): boolean {
+  return SCHEMA_ACTIONS.has(action[0]);
+}
+
+/**
+ * Returns the tableId from the action.
+ */
+export function getTableId(action: DocAction): string {
+  return action[1];   // It happens to always be in the same position in the action tuple.
+}
+
+// Helper types used in the definitions above.
+
+export type CellValue = number|string|boolean|null|[string, any?];
+export interface ColValues { [colId: string]: CellValue; }
+export interface BulkColValues { [colId: string]: CellValue[]; }
+export interface ColInfoMap { [colId: string]: ColInfo; }
+
+export interface ColInfo {
+  type: string;
+  isFormula: boolean;
+  formula: string;
+}
+
+export interface ColInfoWithId extends ColInfo {
+  id: string;
+}
+
+export interface RowRecord {
+  id: number;
+  [colId: string]: CellValue;
+}
+
+// Multiple records in column-oriented format, i.e. same as BulkColValues but with a mandatory
+// 'id' column. This is preferred over TableDataAction in external APIs.
+export interface TableColValues {
+  id: number[];
+  [colId: string]: CellValue[];
+}
+
+// Both UserActions and DocActions are represented as [ActionName, ...actionArgs].
+// TODO I think it's better to represent DocAction as a Buffer containing the marshalled action.
+
+export type UserAction = Array<string|number|object|boolean|null|undefined>;
+
+/**
+ * Gives a description for an action which involves setting values to a selection.
+ * @param {Array} action - The (Bulk)AddRecord/(Bulk)UpdateRecord action to describe.
+ * @param {Boolean} optExcludeVals - Indicates whether the values should be excluded from
+ *  the description.
+ */
+export function getSelectionDesc(action: UserAction, optExcludeVals: boolean): string {
+  const table   = action[1];
+  const rows    = action[2];
+  const colValues: number[]  = action[3] as any;  // TODO: better typing - but code may evaporate
+  const columns = map(colValues, (values, col) => optExcludeVals ? col : `${col}: ${values}`);
+  const s = typeof rows === 'object' ? 's' : '';
+  return `table ${table}, row${s} ${rows}; ${columns.join(", ")}`;
+}
+
+// Convert from TableColValues (used by DocStorage and external APIs) to TableDataAction (used
+// mainly by the sandbox).
+export function toTableDataAction(tableId: string, colValues: TableColValues): TableDataAction {
+  const colData = {...colValues};   // Make a copy to avoid changing passed-in arguments.
+  const rowIds: number[] = colData.id;
+  delete colData.id;
+  return ['TableData', tableId, rowIds, colData];
+}
+
+// Convert from TableDataAction (used mainly by the sandbox) to TableColValues (used by DocStorage
+// and external APIs).
+export function fromTableDataAction(tableData: TableDataAction): TableColValues {
+  const rowIds: number[] = tableData[2];
+  const colValues: BulkColValues = tableData[3];
+  return {id: rowIds, ...colValues};
+}

app/common/DocData.ts

@@ -0,0 +1,121 @@
+/**
+ * DocData maintains all underlying data for a Grist document, knows how to load it,
+ * subscribes to actions which change it, and forwards those actions to individual tables.
+ * It also provides the interface to apply actions to data.
+ */
+import {schema} from 'app/common/schema';
+import fromPairs = require('lodash/fromPairs');
+import groupBy = require('lodash/groupBy');
+import {ActionDispatcher} from './ActionDispatcher';
+import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction,
+        RowRecord, TableDataAction} from './DocActions';
+import {ColTypeMap, TableData} from './TableData';
+
+type FetchTableFunc = (tableId: string) => Promise<TableDataAction>;
+
+export class DocData extends ActionDispatcher {
+  private _tables: Map<string, TableData> = new Map();
+
+  constructor(private _fetchTableFunc: FetchTableFunc, metaTableData: {[tableId: string]: TableDataAction}) {
+    super();
+    // Create all meta tables, and populate data we already have.
+    for (const tableId in schema) {
+      if (schema.hasOwnProperty(tableId)) {
+        const colTypes: ColTypeMap = (schema as any)[tableId];
+        this._tables.set(tableId, this.createTableData(tableId, metaTableData[tableId], colTypes));
+      }
+    }
+
+    // Build a map from tableRef to [columnRecords]
+    const colsByTable = groupBy(this._tables.get('_grist_Tables_column')!.getRecords(), 'parentId');
+    for (const t of this._tables.get('_grist_Tables')!.getRecords()) {
+      const tableId = t.tableId as string;
+      const colRecords: RowRecord[] = colsByTable[t.id] || [];
+      const colTypes = fromPairs(colRecords.map(c => [c.colId, c.type]));
+      this._tables.set(tableId, this.createTableData(tableId, null, colTypes));
+    }
+  }
+
+  /**
+   * Creates a new TableData object. A derived class may override to return an object derived from TableData.
+   */
+  public createTableData(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap): TableData {
+    return new TableData(tableId, tableData, colTypes);
+  }
+
+  /**
+   * Returns the TableData object for the requested table.
+   */
+  public getTable(tableId: string): TableData|undefined {
+    return this._tables.get(tableId);
+  }
+
+  /**
+   * Returns an unsorted list of all tableIds in this doc, including both metadata and user tables.
+   */
+  public getTables(): ReadonlyMap<string, TableData> {
+    return this._tables;
+  }
+
+  /**
+   * Fetches the data for tableId if needed, and returns a promise that is fulfilled when the data
+   * is loaded.
+   */
+  public fetchTable(tableId: string, force?: boolean): Promise<void> {
+    const table = this._tables.get(tableId);
+    if (!table) { throw new Error(`DocData.fetchTable: unknown table ${tableId}`); }
+    return (!table.isLoaded || force) ? table.fetchData(this._fetchTableFunc) : Promise.resolve();
+  }
+
+  /**
+   * Handles an action received from the server, by forwarding it to the appropriate TableData
+   * object.
+   */
+  public receiveAction(action: DocAction): void {
+    // Look up TableData before processing the action in case we rename or remove it.
+    const tableId: string = action[1];
+    const table = this._tables.get(tableId);
+
+    this.dispatchAction(action);
+
+    // Forward all actions to per-table TableData objects.
+    if (table) {
+      table.receiveAction(action);
+    }
+  }
+
+  // ---- The following methods implement ActionDispatcher interface ----
+
+  protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void {
+    const colTypes = fromPairs(columns.map(c => [c.id, c.type]));
+    this._tables.set(tableId, this.createTableData(tableId, null, colTypes));
+  }
+
+  protected onRemoveTable(action: DocAction, tableId: string): void {
+    this._tables.delete(tableId);
+  }
+
+  protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void {
+    const table = this._tables.get(oldTableId);
+    if (table) {
+      this._tables.set(newTableId, table);
+      this._tables.delete(oldTableId);
+    }
+  }
+
+  // tslint:disable:no-empty
+  protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {}
+  protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {}
+  protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {}
+
+  protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
+  protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
+  protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) {}
+
+  protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
+
+  protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {}
+  protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {}
+  protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {}
+  protected onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {}
+}

app/common/DocListAPI.ts

@@ -0,0 +1,83 @@
+import {ActionGroup} from 'app/common/ActionGroup';
+import {TableDataAction} from 'app/common/DocActions';
+import {LocalPlugin} from 'app/common/plugin';
+import {StringUnion} from 'app/common/StringUnion';
+
+// Possible flavors of items in a list of documents.
+export type DocEntryTag = ''|'sample'|'invite'|'shared';
+
+export const OpenDocMode = StringUnion(
+  'default',  // open doc with user's maximal access level
+  'fork',     // open doc limited to view access (if user has at least that level of access)
+  'view'     // as for 'view', but suggest a fork on any attempt to edit
+);
+export type OpenDocMode = typeof OpenDocMode.type;
+
+/**
+ * Represents an entry in the DocList.
+ */
+export interface DocEntry {
+  docId?: string;       // Set for shared docs and invites
+  name: string;
+  mtime?: Date;
+  size?: number;
+  tag: DocEntryTag;
+  senderName?: string;
+  senderEmail?: string;
+}
+
+export interface DocCreationInfo {
+  id: string;
+  title: string;
+}
+
+/**
+ * This documents the members of the structure returned when a local
+ * grist document is opened.
+ */
+export interface OpenLocalDocResult {
+  docFD: number;
+  clientId: string;  // the docFD is meaningful only in the context of this session
+  doc: {[tableId: string]: TableDataAction};
+  log: ActionGroup[];
+  plugins: LocalPlugin[];
+}
+
+export interface DocListAPI {
+  /**
+   * Returns a all known Grist documents and document invites to show in the doc list.
+   */
+  getDocList(): Promise<{docs: DocEntry[], docInvites: DocEntry[]}>;
+
+  /**
+   * Creates a new document, fetches it, and adds a table to it. Returns its name.
+   */
+  createNewDoc(): Promise<string>;
+
+  /**
+   * Makes a copy of the given sample doc. Returns the name of the new document.
+   */
+  importSampleDoc(sampleDocName: string): Promise<string>;
+
+  /**
+   * Processes an upload, containing possibly multiple files, to create a single new document, and
+   * returns the new document's name.
+   */
+  importDoc(uploadId: number): Promise<string>;
+
+  /**
+   * Deletes a Grist document. Returns the name of the deleted document. If `deletePermanently` is
+   * true, the doc is deleted permanently rather than just moved to the trash.
+   */
+  deleteDoc(docName: string, deletePermanently: boolean): Promise<string>;
+
+  /**
+   * Renames a document.
+   */
+  renameDoc(oldName: string, newName: string): Promise<void>;
+
+  /**
+   * Opens a document, loads it, subscribes to its userAction events, and returns its metadata.
+   */
+  openDoc(userDocName: string, openMode?: OpenDocMode): Promise<OpenLocalDocResult>;
+}

app/common/EncActionBundle.ts

@@ -0,0 +1,73 @@
+/**
+ * Types for encrypted ActionBundles that get sent between instances and hub.
+ */
+
+import {ActionInfo, Envelope} from 'app/common/ActionBundle';
+import {DocAction} from 'app/common/DocActions';
+
+// Type representing a point in time as milliseconds since Epoch.
+export type Timestamp = number;
+
+// Type representing binary data encoded as a base64 string.
+export type Base64String = string;
+
+// Metadata about a symmetric encryption key.
+export interface KeyInfo {
+  firstActionNum: number;     // ActionNum of first action for which this key was used.
+  firstUsedTime: Timestamp;   // Timestamp of first action for which this key was used.
+}
+
+// Encrypted symmetric key with metadata, sent from hub to instance with each envelope.
+export interface EncKeyInfo extends KeyInfo {
+  encryptedKey: Base64String; // Symmetric key encrypted with the recipient's public key.
+}
+
+// Bundle of encryptions of the symmetric key. Note that the hub will store EncKeyBundles for
+// lookup, indexed by the combination {recipients: string[], firstActionNum: number}.
+export interface EncKeyBundle extends KeyInfo {
+  encryptedKeys: {
+    // Map of instanceId to the symmetric key encrypted with that instance's public key.
+    // A single symmetric key is used for all, and only present here in encrypted form.
+    [instanceId: string]: Base64String;
+  };
+}
+
+// This allows reorganizing ActionBundle by envelope while preserving order information for
+// actions. E.g. if ActionBundle contains {stored: [(0,A), (1,B), (2,C), (0,D)], then we'll have:
+//    - in envelopes 0: {stored: [[0, A], [3, D]]}
+//    - in envelopes 1: {stored: [[1, B]]}
+//    - in envelopes 2: {stored: [[2, C]]}
+// Then recipients of multiple envelopes can sort actions by index to get their correct order.
+export interface DecryptedEnvelopeContent {
+  info?: ActionInfo;
+  // number is the index into the bundle-wide array of 'stored' or 'calc' DocActions.
+  stored: Array<[number, DocAction]>;
+  calc: Array<[number, DocAction]>;
+}
+
+export type DecryptedEnvelope = Envelope & DecryptedEnvelopeContent;
+
+// Sent from instance to hub.
+export interface EncEnvelopeToHub extends Envelope {
+  encKeyReused?: number;        // If reusing a key, firstActionNum of the key being reused.
+  encKeyBundle?: EncKeyBundle;  // If created a new key, its encryption for all recipients.
+  content: Base64String;        // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string.
+}
+
+// Sent from hub to instance.
+export interface EncEnvelopeFromHub extends Envelope {
+  encKeyInfo: EncKeyInfo;
+  content: Base64String;        // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string.
+}
+
+// EncActionBundle is an encrypted version of ActionBundle. It comes in two varieties, one for
+// sending ActionBundle to the hub, and one for receiving from the hub.
+export interface EncActionBundle<EncEnvelope> {
+  actionNum: number;
+  actionHash: string|null;
+  parentActionHash: string|null;
+  envelopes: EncEnvelope[];
+}
+
+export type EncActionBundleToHub = EncActionBundle<EncEnvelopeToHub>;
+export type EncActionBundleFromHub = EncActionBundle<EncEnvelopeFromHub>;

app/common/ErrorWithCode.ts

@@ -0,0 +1,23 @@
+import {OpenDocMode} from 'app/common/DocListAPI';
+
+interface ErrorDetails {
+  status?: number;
+  accessMode?: OpenDocMode;
+}
+
+/**
+ *
+ * An error with a human-readable message and a machine-readable code.
+ * Makes it easier to change the human-readable message without breaking
+ * error handlers.
+ *
+ */
+export class ErrorWithCode extends Error {
+  public accessMode?: OpenDocMode;
+  public status?: number;
+  constructor(public code: string, message: string, details: ErrorDetails = {}) {
+    super(message);
+    this.status = details.status;
+    this.accessMode = details.accessMode;
+  }
+}

app/common/Features.ts

@@ -0,0 +1,45 @@
+// A product is essentially a list of flags and limits that we may enforce/support.
+export interface Features {
+  vanityDomain?: boolean;   // are user-selected domains allowed (unenforced) (default: true)
+
+  workspaces?: boolean;     // are workspaces shown in web interface (default: true)
+                            // (this was intended as something we can turn off to shut down
+                            // web access to content while leaving access to billing)
+
+  /**
+   * Some optional limits.  Since orgs can change plans, limits will typically be checked
+   * at the point of creation.  E.g. adding someone new to a document, or creating a
+   * new document.  If, after an operation, the limit would be exceeded, that operation
+   * is denied.  That means it is possible to exceed limits if the limits were not in
+   * place when shares/docs were originally being added.  The action that would need
+   * to be taken when infringement is pre-existing is not so obvious.
+   */
+
+  maxSharesPerDoc?: number; // Maximum number of users that can be granted access to a
+                            // particular doc.  Doesn't count users granted access at
+                            // workspace or organization level.  Doesn't count billable
+                            // users if applicable (default: unlimited)
+
+  maxSharesPerDocPerRole?: {[role: string]: number};  // As maxSharesPerDoc, but
+                            // for specific roles.  Roles are named as in app/common/roles.
+                            // Applied independently to maxSharesPerDoc.
+                            // (default: unlimited)
+  maxSharesPerWorkspace?: number;  // Maximum number of users that can be granted access to
+                            // a particular workspace.  Doesn't count users granted access
+                            // at organizational level, or billable users (default: unlimited)
+
+  maxDocsPerOrg?: number;   // Maximum number of documents allowed per org.
+                            // (default: unlimited)
+  maxWorkspacesPerOrg?: number;   // Maximum number of workspaces allowed per org.
+                            // (default: unlimited)
+
+  readOnlyDocs?: boolean;   // if set, docs can only be read, not written.
+}
+
+// Check whether it is possible to add members at the org level.  There's no flag
+// for this right now, it isn't enforced at the API level, it is just a bluff.
+// For now, when maxWorkspacesPerOrg is 1, we should assume members can't be added
+// to org (even though this is not enforced).
+export function canAddOrgMembers(features: Features): boolean {
+  return features.maxWorkspacesPerOrg !== 1;
+}

app/common/Formula.ts

@@ -0,0 +1,72 @@
+/**
+ *
+ * This represents a formula supported under SQL for on-demand tables.  This is currently
+ * a very small subset of the formulas supported by the data engine for regular tables.
+ *
+ * The following kinds of formula are supported:
+ *   $refColId.colId    [where colId is not itself a formula]
+ *   $colId             [where colId is not itself a formula]
+ *   NNN                [a non-negative integer]
+ * TODO: support a broader range of formula, by adding a parser or reusing Python parser.
+ * An argument for reusing Python parser: wwe already do substantial parsing of the formula code.
+ * E.g. Python does such amazing things as handle updating the formula when any of the columns
+ * referred to in Foo.lookup(bar=$baz).blah get updated.
+ *
+ */
+export type Formula = LiteralNumberFormula | ColumnFormula | ForeignColumnFormula | FormulaError;
+
+// A simple copy of another column.  E.g. "$Person"
+export interface ColumnFormula {
+  kind: 'column';
+  colId: string;
+}
+
+// A copy of a column in another table (via a reference column).  E.g. "$Person.FirstName"
+export interface ForeignColumnFormula {
+  kind: 'foreignColumn';
+  colId: string;
+  refColId: string;
+}
+
+export interface LiteralNumberFormula {
+  kind: 'literalNumber';
+  value: number;
+}
+
+// A formula that couldn't be parsed.
+export interface FormulaError {
+  kind: 'error';
+  msg: string;
+}
+
+/**
+ * Convert a string to a parsed formula.  Regexes are adequate for the very few
+ * supported formulas, but once the syntax is at all flexible a proper parser will
+ * be needed.  In principle, it might make sense to support python syntax, for
+ * compatibility with the data engine, but compatibility in corner cases will be
+ * fiddly given underlying differences between sqlite and python.
+ */
+export function parseFormula(txt: string): Formula {
+  // Formula of form: $x.y
+  let m = txt.match(/^\$([a-z]\w*)\.([a-z]\w*)$/i);
+  if (m) {
+    return {kind: 'foreignColumn', refColId: m[1], colId: m[2]};
+  }
+
+  // Formula of form: $x
+  m = txt.match(/^\$([a-z][a-z_0-9]*)$/i);
+  if (m) {
+    return {kind: 'column', colId: m[1]};
+  }
+
+  // Formula of form: NNN
+  m = txt.match(/^[0-9]+$/);
+  if (m) {
+    const value = parseInt(txt, 10);
+    if (isNaN(value)) { return {kind: 'error', msg: 'Cannot parse integer'}; }
+    return {kind: 'literalNumber', value};
+  }
+
+  // Everything else is an error.
+  return {kind: 'error', msg: 'Formula not supported'};
+}

app/common/GristServerAPI.ts

@@ -0,0 +1,84 @@
+import {BasketClientAPI} from 'app/common/BasketClientAPI';
+import {DocListAPI} from 'app/common/DocListAPI';
+import {LoginSessionAPI} from 'app/common/LoginSessionAPI';
+import {EmailResult, Invite} from 'app/common/sharing';
+import {UserConfig} from 'app/common/UserConfig';
+
+export interface GristServerAPI extends
+  DocListAPI,
+  LoginSessionAPI,
+  BasketClientAPI,
+  ServerMetricsAPI,
+  UserAPI,
+  SharingAPI,
+  MiscAPI {}
+
+
+interface ServerMetricsAPI {
+  /**
+   * Registers the list of client metric names. The calls to pushClientMetrics() send metric
+   * values as an array parallel to this list of names.
+   */
+  registerClientMetrics(clientMetricsList: string[]): Promise<void>;
+
+  /**
+   * Sends bucketed client metric data to the server. The .values arrays contain one value for
+   * each of the registered metric names, as a parallel array.
+   */
+  pushClientMetrics(clientBuckets: Array<{startTime: number, values: number[]}>): Promise<void>;
+}
+
+interface UserAPI {
+  /**
+   * Gets the Grist configuration from the server.
+   */
+  getConfig(): Promise<UserConfig>;
+
+  /**
+   * Updates the user configuration and saves it to the server.
+   * @param {Object} config - Configuration object to save.
+   * @returns {Promise:Object} Configuration object as persisted by the server. You can use it to
+   * validate the configuration.
+   */
+  updateConfig(config: UserConfig): Promise<UserConfig>;
+
+  /**
+   * Re-load plugins.
+   */
+  reloadPlugins(): Promise<void>;
+}
+
+interface SharingAPI {
+  /**
+   * Looks up a user account by email, return an object with basic user profile information.
+   */
+  lookupEmail(email: string): Promise<EmailResult|null>;
+
+  /**
+   * Fetches and saves invites from the hub which are not already present locally.
+   * Returns the new invites from the hub only.
+   */
+  getNewInvites(): Promise<Invite[]>;
+
+  /**
+   * Fetches local invites and marks them all as read.
+   */
+  getLocalInvites(): Promise<Invite[]>;
+
+  /**
+   * Marks the stored local invite belonging to the calling instance as ignored.
+   * Called when the user declines an invite.
+   */
+  ignoreLocalInvite(docId: string): Promise<void>;
+
+  /**
+   * Downloads a shared doc by creating a new doc and applying the snapshot actions associated
+   * with the given docId on the sharing hub. Must be called from a logged in account and instance
+   * invited to download the doc. Returns the actual non-conflicting docName used.
+   */
+  downloadSharedDoc(docId: string, docName: string): Promise<string>;
+}
+
+interface MiscAPI {
+  showItemInFolder(docName: string): Promise<void>;
+}

app/common/InactivityTimer.ts

@@ -0,0 +1,119 @@
+/**
+ * InactivityTimer allows to set a function that executes after a certain time of
+ * inactivity. Activities can be of two kinds: synchronous or asynchronous. Asynchronous activities,
+ * are handle with the `disableUntiFinish` method that takes in a Promise and makes sure that the
+ * timer does not start before the promise resolves. Synchroneous activities are monitored with the
+ * `ping` method which resets the timer if called during inactivity.
+ *
+ * Timer won't start before any activity happens, but you may simply call ping() after construction
+ * to start it. After cb is called, timer is disabled but enabled again if there is more activity.
+ *
+ * Example usage: InactivityTimer is used internally for implementing the plugins' component
+ * deactivation after a certain time of inactivity.
+ *
+ */
+
+// important to explicitly import this, or webpack --watch gets confused.
+import {clearTimeout, setTimeout} from "timers";
+
+export class InactivityTimer {
+
+  private _timeout?: NodeJS.Timer | null;
+  private _counter: number = 0;
+  private _enabled: boolean = true;
+
+  constructor(private _callback: () => void, private _delay: number) {}
+
+  // Returns the delay used by InactivityTimer, in ms.
+  public getDelay(): number {
+    return this._delay;
+  }
+
+  // Sets a different delay to use, in ms.
+  public setDelay(delayMs: number): void {
+    this._delay = delayMs;
+    this.ping();
+  }
+
+  /**
+   * Enable the InactivityTimer and schedule the callback.
+   */
+  public enable(): void {
+    this._enabled = true;
+    this.ping();
+  }
+
+  /**
+   * Clears the timeout and prevents the callback from being called until enable() is called.
+   */
+  public disable(): void {
+    this._enabled = false;
+    this._clearTimeout();
+  }
+
+  /**
+   * Returns whether the InactivityTimer is enabled. If not, the callback will not be scheduled.
+   */
+  public isEnabled(): boolean {
+    return this._enabled;
+  }
+
+  /**
+   * Whether the callback is currently scheduled, and would trigger if there is no activity and if
+   * it's not disabled before it triggers.
+   */
+  public isScheduled(): boolean {
+    return Boolean(this._timeout);
+  }
+
+  /**
+   * Resets the timer if called during inactivity.
+   */
+  public ping() {
+    if (!this._counter && this._enabled) {
+      this._setTimeout();
+    }
+  }
+
+  /**
+   * The `disableUntilFinish` method takes in a promise and makes sure the timer won't start before
+   * it resolves. It returns a promise that resolves to the same object.
+   */
+  public async disableUntilFinish<T>(promise: Promise<T>): Promise<T> {
+    this._beginActivity();
+    try {
+      return await promise;
+    } finally {
+      this._endActivity();
+    }
+  }
+
+  private _beginActivity() {
+    this._counter++;
+    this._clearTimeout();
+  }
+
+  private _endActivity() {
+    this._counter = Math.max(this._counter - 1, 0);
+    this.ping();
+  }
+
+  private _clearTimeout() {
+    if (this._timeout) {
+      clearTimeout(this._timeout);
+      this._timeout = null;
+    }
+  }
+
+  private _setTimeout() {
+    this._clearTimeout();
+    this._timeout = setTimeout(() => this._onTimeoutTriggered(), this._delay);
+  }
+
+  private _onTimeoutTriggered() {
+    this._clearTimeout();
+    // _counter is set to 0, even if there's no reason why it should be any thing else.
+    this._counter = 0;
+    this._callback();
+  }
+}

app/common/KeyedOps.ts

@@ -0,0 +1,222 @@
+/**
+ * A class for scheduling a particular operation on resources
+ * identified by a key.  For operations which should be applied
+ * some time after an event.
+ */
+export class KeyedOps {
+  private _operations = new Map<string, OperationStatus>();  // status of operations
+  private _history = new Map<string, OperationHistory>();    // history of operations
+                                                             // (will accumulate without limit)
+  private _changed = new Set<string>();    // set when key needs an operation
+  private _operating = new Set<string>();  // set when operation is in progress for key
+
+  /**
+   * Provide a function to apply operation, and some optional
+   * parameters.
+   *
+   *   - delayBeforeOperationMs: if set, a call to addOperation(key) will have
+   *     a delayed effect.  It will schedule (or reschedule) the operation to occur
+   *     after this interval.  If the operation is currently in progress, it will
+   *     get rerun after it completes.
+   *
+   *   - minDelaybetweenOperationsMs: is set, scheduling for operations will have
+   *     additional delays inserted as necessary to keep this minimal delay between
+   *     the start of successive operations.
+   *
+   *   - retry: if `retry` is set, the operation will be retried
+   *     indefinitely with a rather primitive retry mechanism -
+   *     otherwise no attempt is made to retry failures.
+   *
+   *   - logError: called when errors occur, with a count of number of failures so
+   *     far.
+   */
+  constructor(private _op: (key: string) => Promise<void>, private _options: {
+    delayBeforeOperationMs?: number,
+    minDelayBetweenOperationsMs?: number,
+    retry?: boolean,
+    logError?: (key: string, failureCount: number, err: Error) => void
+  }) {
+  }
+
+  /**
+   * Request an operation be done (eventually) on the specified resourse.
+   */
+  public addOperation(key: string) {
+    this._changed.add(key);
+    this._schedule(key);
+  }
+
+  /**
+   * Check whether any work is scheduled or in progress.
+   */
+  public hasPendingOperations() {
+    return this._changed.size > 0 || this._operating.size > 0;
+  }
+
+  /**
+   * Check whether any work is scheduled or in progress for a specific resource.
+   */
+  public hasPendingOperation(key: string) {
+    return this._changed.has(key) || this._operating.has(key);
+  }
+
+  /**
+   * Take all scheduled operations and re-schedule them for right now.  Useful
+   * when shutting down.  Affects retries.  Cannot be undone.  Returns immediately.
+   */
+  public expediteOperations() {
+    this._options.delayBeforeOperationMs = 0;
+    this._options.minDelayBetweenOperationsMs = 0;
+    for (const op of this._operations.values()) {
+      if (op.timeout) {
+        this._schedule(op.key, true);
+      }
+    }
+  }
+
+  /**
+   * Wait for all operations to complete.  This makes most sense to use during
+   * shutdown - otherwise it might be a very long wait to reach a moment where
+   * there are no operations.
+   */
+  public async wait(logRepeat?: (count: number) => void) {
+    let repeats: number = 0;
+    while (this.hasPendingOperations()) {
+      if (repeats && logRepeat) { logRepeat(repeats); }
+      await Promise.all([...this._operating.keys(), ...this._changed.keys()]
+                        .map(key => this.expediteOperationAndWait(key)));
+      repeats++;
+    }
+  }
+
+  /**
+   * Re-schedules any pending operation on a resource for right now.  Returns
+   * when operations on the resource are complete.  Does not affect retries.
+   */
+  public async expediteOperationAndWait(key: string) {
+    const status = this._getOperationStatus(key);
+    if (status.promise) {
+      await status.promise;
+      return;
+    }
+    const callback = new Promise((resolve) => {
+      status.callbacks.push(resolve);
+    });
+    this._schedule(key, true);
+    await callback;
+  }
+
+  /**
+   * Schedule an operation for a resource.
+   * If the operation is already in progress, we do nothing.
+   * If the operation has not yet happened, it is rescheduled.
+   * If `immediate` is set, the operation is scheduled with no delay.
+   */
+  private _schedule(key: string, immediate: boolean = false) {
+    const status = this._getOperationStatus(key);
+    if (status.promise) { return; }
+    if (status.timeout) {
+      clearTimeout(status.timeout);
+      delete status.timeout;
+    }
+    let ticks = this._options.delayBeforeOperationMs || 0;
+    const {lastStart} = this._getOperationHistory(key);
+    if (lastStart && this._options.minDelayBetweenOperationsMs && !immediate) {
+      ticks = Math.max(ticks, lastStart + this._options.minDelayBetweenOperationsMs - Date.now());
+    }
+    // Primitive slow-down on retries.
+    // Will do nothing if neither delayBeforeOperationMs nor minDelayBetweenOperationsMs
+    // are set.
+    ticks *= 1 + Math.min(5, status.failures);
+    status.timeout = setTimeout(() => this._update(key), immediate ? 0 : ticks);
+  }
+
+  private _getOperationStatus(key: string): OperationStatus {
+    let status = this._operations.get(key);
+    if (!status) {
+      status = {
+        key,
+        failures: 0,
+        callbacks: []
+      };
+      this._operations.set(key, status);
+    }
+    return status;
+  }
+
+  private _getOperationHistory(key: string): OperationHistory {
+    let hist = this._history.get(key);
+    if (!hist) {
+      hist = {};
+      this._history.set(key, hist);
+    }
+    return hist;
+  }
+
+  // Implement the next scheduled operation for a resource.
+  private _update(key: string) {
+    const status = this._getOperationStatus(key);
+    delete status.timeout;
+
+    // We don't have to do anything if there have been no changes.
+    if (!this._changed.has(key)) { return; }
+    // We don't have to do anything (yet) if an operation is already in progress.
+    if (status.promise) { return; }
+
+    // Switch status from changed to operating.
+    this._changed.delete(key);
+    this._operating.add(key);
+    const history = this._getOperationHistory(key);
+    history.lastStart = Date.now();
+
+    // Store a promise for the operation.
+    status.promise = this._op(key).then(() => {
+      // Successful push!  Reset failure count, notify callbacks.
+      status.failures = 0;
+      status.callbacks.forEach(callback => callback());
+      status.callbacks = [];
+    }).catch(err => {
+      // Operation failed.  Increment failure count, notify callbacks.
+      status.failures++;
+      if (this._options.retry) {
+        this._changed.add(key);
+      }
+      if (this._options.logError) {
+        this._options.logError(key, status.failures, err);
+      }
+      status.callbacks.forEach(callback => callback(err));
+      status.callbacks = [];
+    }).then(() => {
+      // Clean up and schedule follow-up if necessary.
+      this._operating.delete(key);
+      delete status.promise;
+      if (this._changed.has(key)) {
+        this._schedule(key);
+      } else {
+        // No event information left to track, we can delete our OperationStatus entry.
+        if (status.failures === 0 && !status.timeout) {
+          this._operations.delete(key);
+        }
+      }
+    });
+  }
+}
+
+/**
+ * Status of an operation.
+ */
+interface OperationStatus {
+  timeout?: NodeJS.Timeout;  // a timeout for a scheduled future operation
+  promise?: Promise<void>;   // a promise for an operation that is under way
+  key: string;               // the operation key
+  failures: number;          // consecutive number of times the operation has failed
+  callbacks: Array<(err?: Error) => void>;  // callbacks for notifications when op is done/fails
+}
+
+
+/**
+ * History of an operation.
+ */
+interface OperationHistory {
+  lastStart?: number;        // last time operation was started, in ms since epoch
+}

app/common/LoginSessionAPI.ts

@@ -0,0 +1,27 @@
+// User profile info for the user. When using Cognito, it is fetched during login.
+export interface UserProfile {
+  email: string;
+  name: string;
+  picture?: string|null; // when present, a url to a public image of unspecified dimensions.
+  anonymous?: boolean;   // when present, asserts whether user is anonymous (not authorized).
+  loginMethod?: 'Google'|'Email + Password';
+}
+
+// User profile including user id.  All information in it should
+// have been validated against database.
+export interface FullUser extends UserProfile {
+  id: number;
+}
+
+export interface LoginSessionAPI {
+  /**
+   * Logs out by clearing all data in the session store besides the session cookie itself.
+   * Broadcasts the logged out state to all clients.
+   */
+  logout(): Promise<void>;
+
+  /**
+   * Replaces the user profile object in the session and broadcasts the new profile to all clients.
+   */
+  updateProfile(profile: UserProfile): Promise<void>;
+}

app/common/LoginState.ts

@@ -0,0 +1,31 @@
+import {parseSubdomain} from 'app/common/gristUrls';
+
+// This interface is used by the standalone login-connect tool for knowing where to redirect to,
+// by Client.ts to construct this info, and by CognitoClient to decide what to do.
+
+export interface LoginState {
+  // Locally-running Grist uses localPort, while hosted uses subdomain. Login-connect uses this to
+  // redirect back to the localhost or to the subdomain.
+  localPort?: number;
+  subdomain?: string;
+  baseDomain?: string;  // the domain with the (left-most) subdomain removed, e.g. ".getgrist.com".
+                        // undefined on localhost.
+
+  // Standalone version sets clientId, used later to find the LoginSession. Hosted and dev
+  // versions rely on the browser cookies instead, specifically on the session cookie.
+  clientId?: string;
+
+  // Hosted and dev versions set redirectUrl and redirect to it when login or logout completes.
+  // Standalone version omits redirectUrl, and serves a page which closes the window.
+  redirectUrl?: string;
+}
+
+/// Allowed localhost addresses.
+export const localhostRegex = /^localhost(?::(\d+))?$/i;
+
+export function getLoginState(reqHost: string): LoginState|null {
+  const {org, base} = parseSubdomain(reqHost);
+  const matchPort = localhostRegex.exec(reqHost);
+  return org ? {subdomain: org, baseDomain: base} :
+    matchPort ? {localPort: matchPort[1] ? parseInt(matchPort[1], 10) : 80} : null;
+}

app/common/MemBuffer.js

@@ -0,0 +1,294 @@
+const gutil = require('./gutil');
+const {arrayToString, stringToArray} = require('./arrayToString');
+
+
+/**
+ * Class for a dynamic memory buffer. You can optionally pass the number of bytes
+ * to reserve initially.
+ */
+function MemBuffer(optBytesToReserve) {
+  this.buffer = new ArrayBuffer(optBytesToReserve || 64);
+  this.asArray = new Uint8Array(this.buffer);
+  this.asDataView = new DataView(this.buffer);
+  this.startPos = 0;
+  this.endPos = 0;
+}
+
+// These are defined in gutil now because they are used there (and to avoid a circular import),
+// but were originally defined in MemBuffer and various code still uses them as MemBuffer members.
+MemBuffer.arrayToString = arrayToString;
+MemBuffer.stringToArray = stringToArray;
+
+/**
+ * Returns the number of bytes in the buffer.
+ */
+MemBuffer.prototype.size = function() {
+  return this.endPos - this.startPos;
+};
+
+/**
+ * Returns the number of bytes reserved in the buffer for data. This is at least size().
+ */
+MemBuffer.prototype.reserved = function() {
+  return this.buffer.byteLength - this.startPos;
+};
+
+/**
+ * Reserves enough space in the buffer to hold a nbytes of data, counting the data already in the
+ * buffer.
+ */
+MemBuffer.prototype.reserve = function(nbytes) {
+  if (this.startPos + nbytes > this.buffer.byteLength) {
+    var origArray = new Uint8Array(this.buffer, this.startPos, this.size());
+    if (nbytes > this.buffer.byteLength) {
+      // At least double the size of the buffer.
+      var newBytes = Math.max(nbytes, this.buffer.byteLength * 2);
+      this.buffer = new ArrayBuffer(newBytes);
+      this.asArray = new Uint8Array(this.buffer);
+      this.asDataView = new DataView(this.buffer);
+    }
+    // If we did not allocate more space, this line will just move data to the beginning.
+    this.asArray.set(origArray);
+    this.endPos = this.size();
+    this.startPos = 0;
+  }
+};
+
+/**
+ * Clears the buffer.
+ */
+MemBuffer.prototype.clear = function() {
+  this.startPos = this.endPos = 0;
+  // If the buffer has grown somewhat big, use this chance to free the memory.
+  if (this.buffer.byteLength >= 256 * 1024) {
+    this.buffer = new ArrayBuffer(64);
+    this.asArray = new Uint8Array(this.buffer);
+    this.asDataView = new DataView(this.buffer);
+  }
+};
+
+/**
+ * Returns a Uint8Array viewing all the data in the buffer. It is the caller's responsibility to
+ * make a copy if needed to avoid it being affected by subsequent changes to the buffer.
+ */
+MemBuffer.prototype.asByteArray = function() {
+  return new Uint8Array(this.buffer, this.startPos, this.size());
+};
+
+/**
+ * Converts all buffer data to string using UTF8 encoding.
+ * This is mainly for testing.
+ */
+MemBuffer.prototype.toString = function() {
+  return arrayToString(this.asByteArray());
+};
+
+/*
+ * (Dmitry 2017/03/20. Some unittests that include timing (e.g. Sandbox.js measuring serializing
+ * of data using marshal.js) indicated that gutil.arrayCopyForward gets deoptimized. Narrowing it
+ * down, I found it was because it was used with different argument types (Arrays, Buffers,
+ * Uint8Arrays). To keep it optimized, we'll use a cloned copy of arrayCopyForward (for copying to
+ * a Uint8Array) in this module.
+ */
+let arrayCopyForward = gutil.cloneFunc(gutil.arrayCopyForward);
+
+/**
+ * Appends an array of bytes to this MemBuffer.
+ * @param {Uint8Array|Buffer} bytes: Array of bytes to append. May be a Node Buffer.
+ */
+MemBuffer.prototype.writeByteArray = function(bytes) {
+  // Note that the implementation is identical for Uint8Array and a Node Buffer.
+  this.reserve(this.size() + bytes.length);
+  arrayCopyForward(this.asArray, this.endPos, bytes, 0, bytes.length);
+  this.endPos += bytes.length;
+};
+
+/**
+ * Encodes the given string in UTF8 and appends to the buffer.
+ */
+if (typeof TextDecoder !== 'undefined') {
+  MemBuffer.prototype.writeString = function(string) {
+    this.writeByteArray(stringToArray(string));
+  };
+} else {
+  // We can write faster without using stringToArray, to avoid allocating new buffers.
+  // We'll encode data in chunks reusing a single buffer. The buffer is a multiple of chunk size
+  // to have enough space for multi-byte characters.
+  var encodeChunkSize = 1024;
+  var encodeBufferPad = Buffer.alloc(encodeChunkSize * 4);
+
+  MemBuffer.prototype.writeString = function(string) {
+    // Reserve one byte per character initially (common case), but we'll reserve more below as
+    // needed.
+    this.reserve(this.size() + string.length);
+    for (var i = 0; i < string.length; i += encodeChunkSize) {
+      var bytesWritten = encodeBufferPad.write(string.slice(i, i + encodeChunkSize));
+      this.reserve(this.size() + bytesWritten);
+      arrayCopyForward(this.asArray, this.endPos, encodeBufferPad, 0, bytesWritten);
+      this.endPos += bytesWritten;
+    }
+  };
+}
+
+
+function makeWriteFunc(typeName, bytes, optLittleEndian) {
+  var setter = DataView.prototype['set' + typeName];
+  return function(value) {
+    this.reserve(this.size() + bytes);
+    setter.call(this.asDataView, this.endPos, value, optLittleEndian);
+    this.endPos += bytes;
+  };
+}
+
+/**
+ * The following methods append a value of the given type to the buffer.
+ * These are analogous to Node Buffer's write* family of methods.
+ */
+MemBuffer.prototype.writeInt8 = makeWriteFunc('Int8', 1);
+MemBuffer.prototype.writeUint8 = makeWriteFunc('Uint8', 1);
+MemBuffer.prototype.writeInt16LE = makeWriteFunc('Int16', 2, true);
+MemBuffer.prototype.writeInt16BE = makeWriteFunc('Int16', 2, false);
+MemBuffer.prototype.writeUint16LE = makeWriteFunc('Uint16', 2, true);
+MemBuffer.prototype.writeUint16BE = makeWriteFunc('Uint16', 2, false);
+MemBuffer.prototype.writeInt32LE = makeWriteFunc('Int32', 4, true);
+MemBuffer.prototype.writeInt32BE = makeWriteFunc('Int32', 4, false);
+MemBuffer.prototype.writeUint32LE = makeWriteFunc('Uint32', 4, true);
+MemBuffer.prototype.writeUint32BE = makeWriteFunc('Uint32', 4, false);
+MemBuffer.prototype.writeFloat32LE = makeWriteFunc('Float32', 4, true);
+MemBuffer.prototype.writeFloat32BE = makeWriteFunc('Float32', 4, false);
+MemBuffer.prototype.writeFloat64LE = makeWriteFunc('Float64', 8, true);
+MemBuffer.prototype.writeFloat64BE = makeWriteFunc('Float64', 8, false);
+
+/**
+ * To consume data from an mbuf, the following pattern is recommended:
+ *    var consumer = mbuf.makeConsumer();
+ *    try {
+ *      mbuf.readInt8(consumer);
+ *      mbuf.readByteArray(consumer, len);
+ *      ...
+ *    } catch (e) {
+ *      if (e.needMoreData) {
+ *        ...
+ *      }
+ *    }
+ *    mbuf.consume(consumer);
+ */
+MemBuffer.prototype.makeConsumer = function() {
+  return new Consumer(this);
+};
+
+/**
+ * After some data has been read via a consumer, mbuf.consume(consumer) will clear out the
+ * consumed data from the buffer.
+ */
+MemBuffer.prototype.consume = function(consumer) {
+  this.startPos = consumer.pos;
+  if (this.size() === 0) {
+    this.clear();
+    consumer.pos = this.startPos;
+  }
+};
+
+/**
+ * Helper class for reading data from the buffer. It keeps track of an offset into the buffer
+ * without changing anything in the MemBuffer itself. To affect the MemBuffer,
+ * mbuf.consume(consumer) should be called.
+ */
+function Consumer(mbuf) {
+  this.mbuf = mbuf;
+  this.pos = mbuf.startPos;
+}
+
+/**
+ * Helper for reading data, used by MemBuffer's read* methods.
+ */
+Consumer.prototype._consume = function(nbytes) {
+  var offset = this.pos;
+  if (this.pos + nbytes > this.mbuf.endPos) {
+    var err = new RangeError("MemBuffer: read past end");
+    err.needMoreData = true;
+    err.consumedData = this.pos - this.mbuf.startPos;
+    throw err;
+  }
+  this.pos += nbytes;
+  return offset;
+};
+
+/**
+ * Reads length bytes from the buffer using the passed-in consumer, as created by
+ * mbuf.makeConsumer(). Returns a view on the underlying data.
+ * @returns {Uint8Array} array of bytes viewing underlying MemBuffer data.
+ */
+MemBuffer.prototype.readByteArraySlice = function(cons, length) {
+  return new Uint8Array(this.buffer, cons._consume(length), length);
+};
+
+/**
+ * Reads length bytes from the buffer using the passed-in consumer.
+ * @returns {Uint8Array} array of bytes that's a copy of the underlying data.
+ */
+MemBuffer.prototype.readByteArray = function(cons, length) {
+  return new Uint8Array(this.readByteArraySlice(cons, length));
+};
+
+/**
+ * Reads length bytes from the buffer using the passed-in consumer.
+ * @returns {Buffer} copy of data as a Node Buffer.
+ */
+MemBuffer.prototype.readBuffer = function(cons, length) {
+  return Buffer.from(this.readByteArraySlice(cons, length));
+};
+
+/**
+ * Decodes byteLength bytes from the buffer using UTF8 and returns the resulting string. Uses the
+ * passed-in consumer, as created by mbuf.makeConsumer().
+ * @returns {string}
+ */
+if (typeof TextDecoder !== 'undefined') {
+  MemBuffer.prototype.readString = function(cons, byteLength) {
+    return arrayToString(this.readByteArraySlice(cons, byteLength));
+  };
+} else {
+  var decodeBuffer = Buffer.alloc(1024);
+  MemBuffer.prototype.readString = function(cons, byteLength) {
+    var offset = cons._consume(byteLength);
+    if (byteLength <= decodeBuffer.length) {
+      gutil.arrayCopyForward(decodeBuffer, 0, this.asArray, offset, byteLength);
+      return decodeBuffer.toString('utf8', 0, byteLength);
+    } else {
+      return Buffer.from(new Uint8Array(this.buffer, offset, byteLength)).toString();
+    }
+  };
+}
+
+function makeReadFunc(typeName, bytes, optLittleEndian) {
+  var getter = DataView.prototype['get' + typeName];
+  return function(cons) {
+    return getter.call(this.asDataView, cons._consume(bytes), optLittleEndian);
+  };
+}
+
+/**
+ * The following methods read and return a value of the given type from the buffer using the
+ * passed-in consumer, as created by mbuf.makeConsumer(). E.g.
+ *    var consumer = mbuf.makeConsumer();
+ *    mbuf.readInt8(consumer);
+ *    mbuf.consume(consumer);
+ * These are analogous to Node Buffer's read* family of methods.
+ */
+MemBuffer.prototype.readInt8 = makeReadFunc('Int8', 1);
+MemBuffer.prototype.readUint8 = makeReadFunc('Uint8', 1);
+MemBuffer.prototype.readInt16LE = makeReadFunc('Int16', 2, true);
+MemBuffer.prototype.readUint16LE = makeReadFunc('Uint16', 2, true);
+MemBuffer.prototype.readInt16BE = makeReadFunc('Int16', 2, false);
+MemBuffer.prototype.readUint16BE = makeReadFunc('Uint16', 2, false);
+MemBuffer.prototype.readInt32LE = makeReadFunc('Int32', 4, true);
+MemBuffer.prototype.readUint32LE = makeReadFunc('Uint32', 4, true);
+MemBuffer.prototype.readInt32BE = makeReadFunc('Int32', 4, false);
+MemBuffer.prototype.readUint32BE = makeReadFunc('Uint32', 4, false);
+MemBuffer.prototype.readFloat32LE = makeReadFunc('Float32', 4, true);
+MemBuffer.prototype.readFloat32BE = makeReadFunc('Float32', 4, false);
+MemBuffer.prototype.readFloat64LE = makeReadFunc('Float64', 8, true);
+MemBuffer.prototype.readFloat64BE = makeReadFunc('Float64', 8, false);
+
+module.exports = MemBuffer;

app/common/MetricCollector.js

@@ -0,0 +1,101 @@
+const _ = require('underscore');
+const metricConfig = require('./metricConfig');
+const metricTools = require('./metricTools');
+const gutil = require('app/common/gutil');
+
+/**
+ * Base class for metrics collection used by both the server metrics collector, ServerMetrics.js,
+ *  and the client metrics collector, ClientMetrics.js. Should not be instantiated.
+ * Establishes interval attempts to push metrics to the server on creation.
+ */
+function MetricsCollector() {
+  this.startTime = metricTools.getBucketStartTime(Date.now());
+  this.readyToExport = [];
+  // used (as a protected member) by the derived ServerMetrics class.
+  this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now()));
+}
+
+// Should return a map from metric names (as entered in metricConfig.js) to their metricTools.
+MetricsCollector.prototype.getMetrics = function() {
+  throw new Error("Not implemented");
+};
+
+// Should return a promise that is resolved when the metrics have been pushed.
+MetricsCollector.prototype.pushMetrics = function() {
+  throw new Error("Not implemented");
+};
+
+// Should return a bucket of metric data, formatted for either the client or server.
+MetricsCollector.prototype.createBucket = function(bucketStart) {
+  throw new Error("Not implemented");
+};
+
+// Takes a list of metrics specifications and creates an object mapping metric names to
+//  a new instance of the metric gathering tool matching that metric's type.
+MetricsCollector.prototype.initMetricTools = function(metricsList) {
+  var metrics = {};
+  metricsList.forEach(metricInfo => {
+    metrics[metricInfo.name] = new metricTools[metricInfo.type](metricInfo.name);
+  });
+  return metrics;
+};
+
+// Called each push interval.
+MetricsCollector.prototype.attemptPush = function() {
+  this.pushMetrics(this.readyToExport);
+  this.readyToExport = [];
+};
+
+// Pushes bucket to the end of the readyToExport queue. Should be called sequentially, since it
+//  handles deletion of buckets older than the export memory limit.
+MetricsCollector.prototype.queueBucket = function(bucket) {
+  // If readyToExport is at maximum length, delete the oldest element
+  this.readyToExport.push(bucket);
+  var length = this.readyToExport.length;
+  if (length > metricConfig.MAX_PENDING_BUCKETS) {
+    this.readyToExport.splice(0, length - metricConfig.MAX_PENDING_BUCKETS);
+  }
+};
+
+MetricsCollector.prototype.scheduleBucketPreparation = function() {
+  this.prepareCompletedBuckets(Date.now());
+  this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now()));
+};
+
+/**
+ * Checks if each bucket since the last update is completed and for each one adds all data and
+ *  pushes it to the export ready array.
+ */
+MetricsCollector.prototype.prepareCompletedBuckets = function(now) {
+  var bucketStart = metricTools.getBucketStartTime(now);
+  while (bucketStart > this.startTime) {
+    this.queueBucket(this.createBucket(this.startTime));
+    this.startTime += metricConfig.BUCKET_SIZE;
+  }
+};
+
+/**
+ * Collects primitive metrics tools into a list.
+ */
+MetricsCollector.prototype.collectPrimitiveMetrics = function() {
+  var metricTools = [];
+  _.forEach(this.getMetrics(), metricTool => {
+    gutil.arrayExtend(metricTools, metricTool.getPrimitiveMetrics());
+  });
+  return metricTools;
+};
+
+/**
+ * Loops through metric tools for a chosen bucket and performs the provided callback on each.
+ * Resets each tool after the callback is performed.
+ * @param {Number} bucketStart - The desired bucket's start time in milliseconds
+ * @param {Function} callback - The callback to perform on each metric tool.
+ */
+MetricsCollector.prototype.forEachBucketMetric = function(bucketEnd, callback) {
+  this.collectPrimitiveMetrics().forEach(tool => {
+    callback(tool);
+    tool.reset(bucketEnd);
+  });
+};
+
+module.exports = MetricsCollector;

app/common/NumberFormat.ts

@@ -0,0 +1,76 @@
+/**
+ * Here are the most relevant formats we want to support.
+ *   -1234.56     Plain
+ *   -1,234.56    Number (with separators)
+ *   12.34%       Percent
+ *   1.23E3       Scientific
+ *   $(1,234.56)  Accounting
+ *   (1,234.56)   Financial
+ *   -$1,234.56   Currency
+ *
+ * We implement a button-based UI, using one selector button to choose mode:
+ *    none  = NumMode undefined (plain number, no thousand separators)
+ *    `$`   = NumMode 'currency'
+ *    `,`   = NumMode 'decimal' (plain number, with thousand separators)
+ *    `%`   = NumMode 'percent'
+ *    `Exp` = NumMode 'scientific'
+ * A second toggle button is `(-)` for Sign, to use parentheses rather than "-" for negative
+ * numbers. It is Ignored and disabled when mode is 'scientific'.
+ */
+
+import {clamp} from 'app/common/gutil';
+
+// Options for number formatting.
+export type NumMode = 'currency' | 'decimal' | 'percent' | 'scientific';
+export type NumSign = 'parens';
+
+// TODO: In the future, locale should be a value associated with the document or the user.
+const defaultLocale = 'en-US';
+
+// TODO: The currency to use for currency formatting could be made configurable.
+const defaultCurrency = 'USD';
+
+export interface NumberFormatOptions {
+  numMode?: NumMode;
+  numSign?: NumSign;
+  decimals?: number;      // aka minimum fraction digits
+  maxDecimals?: number;
+}
+
+export function buildNumberFormat(options: NumberFormatOptions): Intl.NumberFormat {
+  const nfOptions: Intl.NumberFormatOptions = parseNumMode(options.numMode);
+
+  // numSign is implemented outside of Intl.NumberFormat since the latter's similar 'currencySign'
+  // option is not well-supported, and doesn't apply to non-currency formats.
+
+  if (options.decimals !== undefined) {
+    // Should be at least 0
+    nfOptions.minimumFractionDigits = clamp(Number(options.decimals), 0, 20);
+  }
+
+  // maximumFractionDigits must not be less than the minimum, so we need to know the minimum
+  // implied by numMode.
+  const tmp = new Intl.NumberFormat(defaultLocale, nfOptions).resolvedOptions();
+
+  if (options.maxDecimals !== undefined) {
+    // Should be at least 0 and at least minimumFractionDigits.
+    nfOptions.maximumFractionDigits = clamp(Number(options.maxDecimals), tmp.minimumFractionDigits || 0, 20);
+  } else if (!options.numMode) {
+    // For the default format, keep max digits at 10 as we had before.
+    nfOptions.maximumFractionDigits = clamp(10, tmp.minimumFractionDigits || 0, 20);
+  }
+
+  return new Intl.NumberFormat(defaultLocale, nfOptions);
+}
+
+function parseNumMode(numMode?: NumMode): Intl.NumberFormatOptions {
+  switch (numMode) {
+    case 'currency': return {style: 'currency', currency: defaultCurrency};
+    case 'decimal': return {useGrouping: true};
+    case 'percent': return {style: 'percent'};
+    // TODO 'notation' option (and therefore numMode 'scientific') works on recent Firefox and
+    // Chrome, not on Safari or Node 10.
+    case 'scientific': return {notation: 'scientific'} as Intl.NumberFormatOptions;
+    default: return {useGrouping: false};
+  }
+}

app/common/PluginInstance.ts

@@ -0,0 +1,213 @@
+import {IForwarderDest, IMessage, IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc} from 'grain-rpc';
+import {Checker} from "ts-interface-checker";
+
+import {InactivityTimer} from 'app/common/InactivityTimer';
+import {LocalPlugin} from 'app/common/plugin';
+import {BarePlugin} from 'app/plugin/PluginManifest';
+
+import {Implementation} from 'app/plugin/PluginManifest';
+import {RenderOptions, RenderTarget} from 'app/plugin/RenderOptions';
+
+
+export type ComponentKind = "safeBrowser" | "safePython" | "unsafeNode";
+
+// Describes a function that appends some html content to `containerElement` given some
+// options. Usefull for provided by a plugin.
+export type TargetRenderFunc = (containerElement: HTMLElement, options?: RenderOptions) => void;
+
+/**
+ * The `BaseComponent` is the base implementation for a plugins' component. It exposes methods
+ * related to its activation. It provides basic features including the inactivity timer, activated
+ * state for the component. A custom component must override the `deactivateImplementation`,
+ * `activeImplementation` and `useRemoteAPI` methods.
+ */
+export abstract class BaseComponent implements IForwarderDest {
+
+  public inactivityTimer: InactivityTimer;
+  private _activated: boolean = false;
+
+  constructor(plugin: BarePlugin, private _logger: IRpcLogger) {
+    const deactivate = plugin.components.deactivate;
+    const delay = (deactivate && deactivate.inactivitySec) ? deactivate.inactivitySec : 300;
+    this.inactivityTimer = new InactivityTimer(() => this.deactivate(), delay * 1000);
+  }
+
+  /**
+   * Wether the Component component have been activated.
+   */
+  public activated(): boolean {
+    return this._activated;
+  }
+
+  /**
+   * Activates the component.
+   */
+  public async activate(): Promise<void> {
+    if (this._logger.info) { this._logger.info("Activating plugin component"); }
+    await this.activateImplementation();
+    this._activated = true;
+    this.inactivityTimer.enable();
+  }
+
+  /**
+   * Force deactivate the component.
+   */
+  public async deactivate(): Promise<void> {
+    if (this._activated) {
+      if (this._logger.info) { this._logger.info("Deactivating plugin component"); }
+      this._activated = false;
+      // Cancel the timer to ensure we don't have an unnecessary hanging timeout (in tests it will
+      // prevent node from exiting, but also it's just wasteful).
+      this.inactivityTimer.disable();
+      try {
+        await this.deactivateImplementation();
+      } catch (e) {
+        // If it fails, we warn and swallow the exception (or it would be an unhandled rejection).
+        if (this._logger.warn) { this._logger.warn(`Deactivate failed: ${e.message}`); }
+      }
+    }
+  }
+
+  public async forwardCall(c: IMsgRpcCall): Promise<any> {
+    if (!this._activated) { await this.activate(); }
+    return await this.inactivityTimer.disableUntilFinish(this.doForwardCall(c));
+  }
+
+  public async forwardMessage(msg: IMsgCustom): Promise<any> {
+    if (!this._activated) { await this.activate(); }
+    this.inactivityTimer.ping();
+    this.doForwardMessage(msg); // tslint:disable-line:no-floating-promises TODO
+  }
+
+  protected abstract doForwardCall(c: IMsgRpcCall): Promise<any>;
+
+  protected abstract doForwardMessage(msg: IMsgCustom): Promise<any>;
+
+  protected abstract deactivateImplementation(): Promise<void>;
+
+  protected abstract activateImplementation(): Promise<void>;
+}
+
+
+/**
+ * Node Implementation for the PluginElement interface. A PluginInstance take care of activation of
+ * the the plugins's components (activating, timing and deactivating), and create the api's for each contributions.
+ *
+ * Do not try to instanciate yourself, PluginManager does it for you. Instead use the
+ * PluginManager.getPlugin(id) method that get instances for you.
+ *
+ */
+export class PluginInstance {
+
+  public rpc: Rpc;
+  public safeBrowser?: BaseComponent;
+  public unsafeNode?: BaseComponent;
+  public safePython?: BaseComponent;
+
+  private  _renderTargets: Map<RenderTarget, TargetRenderFunc> = new Map();
+
+  private _nextRenderTargetId = 0;
+
+  constructor(public definition: LocalPlugin, rpcLogger: IRpcLogger) {
+
+    const rpc = this.rpc = new Rpc({logger: rpcLogger});
+    rpc.setSendMessage((mssg: any) => rpc.receiveMessage(mssg));
+
+    this._renderTargets.set("fullscreen", renderFullScreen);
+  }
+
+  /**
+   * Create an instance for the implementation, this implementation is specific to node environment.
+   */
+  public getStub<Iface>(implementation: Implementation, checker: Checker): Iface {
+    const components: any = this.definition.manifest.components;
+    // the component forwarder was registered under the same relative path that was used to declare
+    // it in the manifest
+    const forwardName = components[implementation.component];
+    return this.rpc.getStubForward<Iface>(forwardName, implementation.name, checker);
+  }
+
+  /**
+   * Stop and clean up all components of this plugin.
+   */
+  public async shutdown(): Promise<void> {
+    await Promise.all([
+      this.safeBrowser && this.safeBrowser.deactivate(),
+      this.safePython && this.safePython.deactivate(),
+      this.unsafeNode && this.unsafeNode.deactivate(),
+      ]);
+  }
+
+  /**
+   * Create a render target and return its identifier. When a plugin calls `render` with `inline`
+   * mode and this identifier, it will append the safe browser process to `element`.
+   */
+  public addRenderTarget(renderPluginContent: TargetRenderFunc): number {
+    const id = this._nextRenderTargetId++;
+    this._renderTargets.set(id, renderPluginContent);
+    return id;
+  }
+
+  /**
+   * Get the function that render an HTML element based on RenderTarget and RenderOptions.
+   */
+  public getRenderTarget(target: RenderTarget, options?: RenderOptions): TargetRenderFunc {
+    const targetRenderPluginContent =  this._renderTargets.get(target);
+    if (!targetRenderPluginContent) {
+      throw new Error(`Unknown render target ${target}`);
+    }
+    return (containerElement, opts) => targetRenderPluginContent(containerElement, opts || options);
+  }
+
+  /**
+   * Removes the render target.
+   */
+  public removeRenderTarget(target: RenderTarget): boolean {
+    return this._renderTargets.delete(target);
+  }
+
+}
+
+/**
+ * Renders safe browser plugin in fullscreen.
+ */
+function renderFullScreen(element: Element) {
+  element.classList.add("plugin_instance_fullscreen");
+  document.body.appendChild(element);
+}
+
+// Basically the union of relevant interfaces of console and server log.
+export interface BaseLogger {
+  log?(message: string, ...args: any[]): void;
+  debug?(message: string, ...args: any[]): void;
+  warn?(message: string, ...args: any[]): void;
+}
+
+/**
+ * Create IRpcLogger which logs to console or server log with the given prefix. Specifically will
+ * warn using baseLog.warn, and log info using baseLog.debug or baseLog.log, as available.
+ */
+export function createRpcLogger(baseLog: BaseLogger, prefix: string): IRpcLogger {
+  const info = baseLog.debug || baseLog.log;
+  const warn = baseLog.warn;
+  return {
+    warn: warn && ((msg: string) => warn("%s %s", prefix, msg)),
+    info: info && ((msg: string) => info("%s %s", prefix, msg)),
+  };
+}
+
+/**
+ * If msec milliseconds pass without receiving a Ready message, print the given message as a
+ * warning.
+ * TODO: I propose making it a method of rpc itself, as rpc.warnIfNotReady(msec, message). Until
+ * we have that, this implements it via an ugly hack.
+ */
+export function warnIfNotReady(rpc: Rpc, msec: number, message: string): void {
+  if (!(rpc as any)._logger.warn) { return; }
+  const timer = setTimeout(() => (rpc as any)._logger.warn(message), msec);
+  const origDispatch = (rpc as any)._dispatch;
+  (rpc as any)._dispatch = (msg: IMessage) => {
+    if (msg.mtype === MsgType.Ready) { clearTimeout(timer); }
+    origDispatch.call(rpc, msg);
+  };
+}

app/common/RefCountMap.ts

@@ -0,0 +1,123 @@
+/**
+ * RefCountMap maintains a reference-counted key-value map. Its sole method is use(key) which
+ * increments the counter for the key, and returns a disposable object which exposes the value via
+ * the get() method, and decrements the counter back on disposal.
+ *
+ * The value is constructed on first reference using options.create(key) callback. After the last
+ * reference is gone, and an optional gracePeriodMs elapsed, the value is cleaned up using
+ * options.dispose(key, value) callback.
+ */
+import {IDisposable} from 'grainjs';
+
+export interface IRefCountSub<Value> extends IDisposable {
+  get(): Value;
+  dispose(): void;
+}
+
+export class RefCountMap<Key, Value> implements IDisposable {
+  private _map: Map<Key, RefCountValue<Value>> = new Map();
+  private _createKey: (key: Key) => Value;
+  private _disposeKey: (key: Key, value: Value) => void;
+  private _gracePeriodMs: number;
+
+  /**
+   * Values are created using options.create(key) on first use. They are disposed after last use,
+   * using options.dispose(key, value). If options.gracePeriodMs is greater than zero, values
+   * stick around for this long after last use.
+   */
+  constructor(options: {
+    create: (key: Key) => Value,
+    dispose: (key: Key, value: Value) => void,
+    gracePeriodMs: number,
+  }) {
+    this._createKey = options.create;
+    this._disposeKey = options.dispose;
+    this._gracePeriodMs = options.gracePeriodMs;
+  }
+
+  /**
+   * Use a value, constructing it if needed, or only incrementing the reference count if this key
+   * is already in the map. The returned subscription object has a get() method which returns the
+   * actual value, and a dispose() method, which must be called to release this subscription (i.e.
+   * decrement back the reference count).
+   */
+  public use(key: Key): IRefCountSub<Value> {
+    const rcValue = this._useKey(key);
+    return {
+      get: () => rcValue.value,
+      dispose: () => this._releaseKey(rcValue, key),
+    };
+  }
+
+  /**
+   * Purge a key by immediately removing it from the map. Disposing the remaining IRefCountSub
+   * values will be no-ops.
+   */
+  public purgeKey(key: Key): void {
+    // Note that we must be careful that disposing stale IRefCountSub values is a no-op even when
+    // the same key gets re-added to the map after purgeKey.
+    this._doDisposeKey(key);
+  }
+
+  /**
+   * Disposing clears the map immediately, and calls options.dispose on all values.
+   */
+  public dispose(): void {
+    // Note that a clear() method like this one would not be OK. If the map were to continue being
+    // used after clear(), subscriptions created before clear() would wreak havoc when disposed.
+    for (const [key, r] of this._map) {
+      r.count = 0;
+      this._disposeKey.call(null, key, r.value);
+    }
+    this._map.clear();
+  }
+
+  private _useKey(key: Key): RefCountValue<Value> {
+    const r = this._map.get(key);
+    if (r) {
+      r.count += 1;
+      if (r.disposeTimeout) {
+        clearTimeout(r.disposeTimeout);
+        r.disposeTimeout = undefined;
+      }
+      return r;
+    }
+    const value = this._createKey.call(null, key);
+    const rcValue = new RefCountValue(value);
+    this._map.set(key, rcValue);
+    return rcValue;
+  }
+
+  private _releaseKey(r: RefCountValue<Value>, key: Key): void {
+    if (r.count > 0) {
+      r.count -= 1;
+      if (r.count === 0) {
+        if (this._gracePeriodMs > 0) {
+          if (!r.disposeTimeout) {
+            r.disposeTimeout = setTimeout(() => this._doDisposeKey(key), this._gracePeriodMs);
+          }
+        } else {
+          this._doDisposeKey(key);
+        }
+      }
+    }
+  }
+
+  private _doDisposeKey(key: Key): void {
+    const r = this._map.get(key);
+    if (r) {
+      this._map.delete(key);
+      r.count = 0;
+      this._disposeKey.call(null, key, r.value);
+    }
+  }
+}
+
+/**
+ * This is an implementation detail of the RefCountMap, which represents a single item.
+ */
+class RefCountValue<Value> {
+  public count: number = 1;
+  public disposeTimeout?: ReturnType<typeof setTimeout> = undefined;
+  constructor(public value: Value) {}
+}

app/common/SortFunc.ts

@@ -0,0 +1,88 @@
+/**
+ * SortFunc class interprets the sortSpec (as saved in viewSection.sortColRefs), exposing a
+ * compare(rowId1, rowId2) function that can be used to actually sort rows in a view.
+ *
+ * TODO: When an operation (such as a paste) would cause rows to jump in the sort order, this
+ * class should support freezing of row positions until the user chooses to re-sort. This is not
+ * currently implemented.
+ */
+import {ColumnGetters} from 'app/common/ColumnGetters';
+import {nativeCompare} from 'app/common/gutil';
+
+/**
+ * Compare two cell values, paying attention to types and values. Note that native JS comparison
+ * can't be used for sorting because it isn't transitive across types (e.g. both 1 < "2" and "2" <
+ * "a" are true, but 1 < "a" is false.). In addition, we handle complex values represented in
+ * Grist as arrays.
+ *
+ * Note that we need to handle different types of values regardless of the column type,
+ * because e.g. a numerical column may contain text (alttext) or null values.
+ */
+export function typedCompare(val1: any, val2: any): number {
+  // TODO: We should use Intl.Collator for string comparisons to handle accented strings.
+  let type1, array1;
+  return nativeCompare(type1 = typeof val1, typeof val2) ||
+    // We need to worry about Array comparisons because formulas returing Any may return null or
+    // object values represented as arrays (e.g. ['D', ...] for dates). Comparing those without
+    // distinguishing types would break the sort. Also, arrays need a special comparator.
+    (type1 === 'object' &&
+      (nativeCompare(array1 = val1 instanceof Array, val2 instanceof Array) ||
+       (array1 && _arrayCompare(val1, val2)))) ||
+    nativeCompare(val1, val2);
+}
+
+function _arrayCompare(val1: any[], val2: any[]): number {
+  for (let i = 0; i < val1.length; i++) {
+    if (i >= val2.length) {
+      return 1;
+    }
+    const value = nativeCompare(val1[i], val2[i]);
+    if (value) {
+      return value;
+    }
+  }
+  return val1.length === val2.length ? 0 : -1;
+}
+
+type ColumnGetter = (rowId: number) => any;
+
+/**
+ * getters is an implementation of app.common.ColumnGetters
+ */
+export class SortFunc {
+  // updateSpec() or updateGetters() can populate these fields, used by the compare() method.
+  private _colGetters: ColumnGetter[] = [];  // Array of column getters (mapping rowId to column value)
+  private _ascFlags: number[] = [];           // Array of 1 (ascending) or -1 (descending) flags.
+
+  constructor(private _getters: ColumnGetters) {}
+
+  public updateSpec(sortSpec: number[]): void {
+    // Prepare an array of column getters for each column in sortSpec.
+    this._colGetters = sortSpec.map(colRef => {
+      return this._getters.getColGetter(Math.abs(colRef));
+    }).filter(getter => getter) as ColumnGetter[];
+
+    // Collect "ascending" flags as an array of 1 or -1, one for each column.
+    this._ascFlags = sortSpec.map(colRef => (colRef >= 0 ? 1 : -1));
+
+    const manualSortGetter = this._getters.getManualSortGetter();
+    if (manualSortGetter) {
+      this._colGetters.push(manualSortGetter);
+      this._ascFlags.push(1);
+    }
+  }
+
+  /**
+   * Returns 1 or -1 depending on whether rowId1 should be shown before rowId2.
+   */
+  public compare(rowId1: number, rowId2: number): number {
+    for (let i = 0, len = this._colGetters.length; i < len; i++) {
+      const getter = this._colGetters[i];
+      const value = typedCompare(getter(rowId1), getter(rowId2));
+      if (value) {
+        return value * this._ascFlags[i];
+      }
+    }
+    return nativeCompare(rowId1, rowId2);
+  }
+}

app/common/StringUnion.ts

@@ -0,0 +1,38 @@
+/**
+ * TypeScript will infer a string union type from the literal values passed to
+ * this function. Without `extends string`, it would instead generalize them
+ * to the common string type.
+ *
+ * Example definition:
+ * const Race = StringUnion(
+ *   "orc",
+ *   "human",
+ *   "night elf",
+ *   "undead",
+ * );
+ * type Race = typeof Race.type;
+ *
+ * For more details, see:
+ * https://stackoverflow.com/questions/36836011/checking-validity-of-string
+ *   -literal-union-type-at-runtime?answertab=active#tab-top
+ */
+export const StringUnion = <UnionType extends string>(...values: UnionType[]) => {
+  Object.freeze(values);
+  const valueSet: Set<string> = new Set(values);
+
+  const guard = (value: string): value is UnionType => {
+    return valueSet.has(value);
+  };
+
+  const check = (value: string): UnionType => {
+    if (!guard(value)) {
+      const actual = JSON.stringify(value);
+      const expected = values.map(s => JSON.stringify(s)).join(' | ');
+      throw new TypeError(`Value '${actual}' is not assignable to type '${expected}'.`);
+    }
+    return value;
+  };
+
+  const unionNamespace = {guard, check, values};
+  return Object.freeze(unionNamespace as typeof unionNamespace & {type: UnionType});
+};

app/common/TableData.ts

@@ -0,0 +1,427 @@
+/**
+ * TableData maintains a single table's data.
+ */
+import {getDefaultForType} from 'app/common/gristTypes';
+import fromPairs = require('lodash/fromPairs');
+import {ActionDispatcher} from './ActionDispatcher';
+import {BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction,
+        isSchemaAction, ReplaceTableData, RowRecord, TableDataAction} from './DocActions';
+import {arrayRemove, arraySplice} from './gutil';
+
+export interface ColTypeMap { [colId: string]: string; }
+
+interface ColData {
+  colId: string;
+  type: string;
+  defl: any;
+  values: CellValue[];
+}
+
+/**
+ * TableData class to maintain a single table's data.
+ *
+ * In the browser's memory, table data needs a representation that's reasonably compact. We
+ * represent it as column-wise arrays. (An early hope was to allow use of TypedArrays, but since
+ * types can be mixed, those are not used.)
+ */
+export class TableData extends ActionDispatcher {
+  private _tableId: string;
+  private _isLoaded: boolean = false;
+  private _fetchPromise?: Promise<void>;
+
+  // Storage of the underlying data. Each column is an array, all of the same length. Includes
+  // 'id' column, containing a reference to _rowIdCol.
+  private _columns: Map<string, ColData> = new Map();
+
+  // Array of all ColData objects, omitting 'id'.
+  private _colArray: ColData[] = [];
+
+  // The `id` column is direct reference to the 'id' column, and contains row ids.
+  private _rowIdCol: number[] = [];
+
+  // Maps row id to index in the arrays in _columns. I.e. it's the inverse of _rowIdCol.
+  private _rowMap: Map<number, number> = new Map();
+
+  constructor(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap) {
+    super();
+    this._tableId = tableId;
+
+    // Initialize all columns to empty arrays, while nothing is yet loaded.
+    for (const colId in colTypes) {
+      if (colTypes.hasOwnProperty(colId)) {
+        const type = colTypes[colId];
+        const defl = getDefaultForType(type);
+        const colData: ColData = { colId, type, defl, values: [] };
+        this._columns.set(colId, colData);
+        this._colArray.push(colData);
+      }
+    }
+    this._columns.set('id', {colId: 'id', type: 'Id', defl: 0, values: this._rowIdCol});
+
+    if (tableData) {
+      this.loadData(tableData);
+    }
+    // TODO: We should probably unload big sets of data when no longer needed. This can be left for
+    // when we support loading only parts of a table.
+  }
+
+  /**
+   * Fetch data (as long as a fetch is not in progress), and load it in memory when done.
+   * Returns a promise that's resolved when data finishes loading, and isLoaded becomes true.
+   */
+  public fetchData(fetchFunc: (tableId: string) => Promise<TableDataAction>): Promise<void> {
+    if (!this._fetchPromise) {
+      this._fetchPromise = fetchFunc(this._tableId).then(data => {
+        this._fetchPromise = undefined;
+        this.loadData(data);
+      });
+    }
+    return this._fetchPromise;
+  }
+
+  /**
+   * Populates the data for this table. Returns the array of old rowIds that were loaded before.
+   */
+  public loadData(tableData: TableDataAction|ReplaceTableData): number[] {
+    const rowIds: number[] = tableData[2];
+    const colValues: BulkColValues = tableData[3];
+    const oldRowIds: number[] = this._rowIdCol.slice(0);
+
+    reassignArray(this._rowIdCol, rowIds);
+    for (const colData of this._colArray) {
+      const values = colValues[colData.colId];
+      // If colId is missing from tableData, use an array of default values. Note that reusing
+      // default value like this is only OK because all default values we use are primitive.
+      reassignArray(colData.values, values || this._rowIdCol.map(() => colData.defl));
+    }
+
+    this._rowMap.clear();
+    for (let i = 0; i < rowIds.length; i++) {
+      this._rowMap.set(rowIds[i], i);
+    }
+
+    this._isLoaded = true;
+    return oldRowIds;
+  }
+
+  // Used by QuerySet to load new rows for onDemand tables.
+  public loadPartial(data: TableDataAction): void {
+    // Add the new rows, reusing BulkAddData code.
+    const rowIds: number[] = data[2];
+    this.onBulkAddRecord(data, data[1], rowIds, data[3]);
+
+    // Mark the table as loaded.
+    this._isLoaded = true;
+  }
+
+  // Used by QuerySet to remove unused rows for onDemand tables when a QuerySet is disposed.
+  public unloadPartial(rowIds: number[]): void {
+    // Remove the unneeded rows, reusing BulkRemoveRecord code.
+    this.onBulkRemoveRecord(['BulkRemoveRecord', this.tableId, rowIds], this.tableId, rowIds);
+  }
+
+  /**
+   * Read-only tableId.
+   */
+  public get tableId(): string { return this._tableId; }
+
+  /**
+   * Boolean flag for whether the data for this table is already loaded.
+   */
+  public get isLoaded(): boolean { return this._isLoaded; }
+
+  /**
+   * The number of records loaded in this table.
+   */
+  public numRecords(): number { return this._rowIdCol.length; }
+
+  /**
+   * Returns the specified value from this table.
+   */
+  public getValue(rowId: number, colId: string): CellValue|undefined {
+    const colData = this._columns.get(colId);
+    const index = this._rowMap.get(rowId);
+    return colData && index !== undefined ? colData.values[index] : undefined;
+  }
+
+  /**
+   * Given a column name, returns a function that takes a rowId and returns the value for that
+   * column of that row. The returned function is faster than getValue() calls.
+   */
+  public getRowPropFunc(colId: string): undefined | ((rowId: number|"new") => CellValue|undefined) {
+    const colData = this._columns.get(colId);
+    if (!colData) { return undefined; }
+    const values = colData.values;
+    const rowMap = this._rowMap;
+    return function(rowId: number|"new") { return rowId === "new" ? "new" : values[rowMap.get(rowId)!]; };
+  }
+
+  /**
+   * Returns the list of all rowIds in this table, in unspecified and unstable order. Equivalent
+   * to getColValues('id').
+   */
+  public getRowIds(): ReadonlyArray<number> {
+    return this._rowIdCol;
+  }
+
+  /**
+   * Sort and returns the list of all rowIds in this table.
+   */
+  public getSortedRowIds(): number[] {
+    return this._rowIdCol.slice(0).sort((a, b) => a - b);
+  }
+
+  /**
+   * Returns the list of colIds in this table, including 'id'.
+   */
+  public getColIds(): string[] {
+    return Array.from(this._columns.keys());
+  }
+
+  /**
+   * Returns an unsorted list of all values in the given column. With no intervening actions,
+   * all arrays returned by getColValues() and getRowIds() are parallel to each other, i.e. the
+   * values at the same index correspond to the same record.
+   */
+  public getColValues(colId: string): ReadonlyArray<CellValue>|undefined {
+    const colData = this._columns.get(colId);
+    return colData ? colData.values : undefined;
+  }
+
+  /**
+   * Returns a limited-sized set of distinct values from a column. If count is given, limits how many
+   * distinct values are returned.
+   */
+  public getDistinctValues(colId: string, count: number = Infinity): Set<CellValue>|undefined {
+    const valColumn = this.getColValues(colId);
+    if (!valColumn) { return undefined; }
+    const distinct: Set<CellValue> = new Set();
+    // Add values to the set until it reaches the desired size, or until there are no more values.
+    for (let i = 0; i < valColumn.length && distinct.size < count; i++) {
+      distinct.add(valColumn[i]);
+    }
+    return distinct;
+  }
+
+  /**
+   * Return data in TableDataAction form ['TableData', tableId, [...rowIds], {...}]
+   */
+  public getTableDataAction(): TableDataAction {
+    const rowIds = this.getRowIds();
+    return ['TableData',
+            this.tableId,
+            rowIds as number[],
+            fromPairs(
+              this.getColIds()
+                .filter(colId => colId !== 'id')
+                .map(colId => [colId, this.getColValues(colId)! as CellValue[]]))];
+  }
+
+  /**
+   * Returns the given columns type, if the column exists, or undefined otherwise.
+   */
+  public getColType(colId: string): string|undefined {
+    const colData = this._columns.get(colId);
+    return colData ? colData.type : undefined;
+  }
+
+  /**
+   * Builds and returns a record object for the given rowId.
+   */
+  public getRecord(rowId: number): undefined | RowRecord {
+    const index = this._rowMap.get(rowId);
+    if (index === undefined) { return undefined; }
+    const ret: RowRecord = { id: this._rowIdCol[index] };
+    for (const colData of this._colArray) {
+      ret[colData.colId] = colData.values[index];
+    }
+    return ret;
+  }
+
+  /**
+   * Builds and returns the list of all records on this table, in unspecified and unstable order.
+   */
+  public getRecords(): RowRecord[] {
+    const records: RowRecord[] = this._rowIdCol.map((id) => ({ id }));
+    for (const {colId, values} of this._colArray) {
+      for (let i = 0; i < records.length; i++) {
+        records[i][colId] = values[i];
+      }
+    }
+    return records;
+  }
+
+  /**
+   * Builds and returns the list of records in this table that match the given properties object.
+   * Properties may include 'id' and any table columns. Returned records are not sorted.
+   */
+  public filterRecords(properties: {[key: string]: any}): RowRecord[] {
+    const rowIndices: number[] = [];
+    // Pairs of [valueToMatch, arrayOfColValues]
+    const props = Object.keys(properties).map(p => [properties[p], this._columns.get(p)]);
+    this._rowIdCol.forEach((id, i) => {
+      for (const p of props) {
+        if (p[1].values[i] !== p[0]) { return; }
+      }
+      // Collect the indices of the matching rows.
+      rowIndices.push(i);
+    });
+
+    // Convert the array of indices to an array of RowRecords.
+    const records: RowRecord[] = rowIndices.map(i => ({id: this._rowIdCol[i]}));
+    for (const {colId, values} of this._colArray) {
+      for (let i = 0; i < records.length; i++) {
+        records[i][colId] = values[rowIndices[i]];
+      }
+    }
+    return records;
+  }
+
+  /**
+   * Returns the rowId in the table where colValue is found in the column with the given colId.
+   */
+  public findRow(colId: string, colValue: any): number {
+    const colData = this._columns.get(colId);
+    if (!colData) {
+      return 0;
+    }
+    const index = colData.values.indexOf(colValue);
+    return index < 0 ? 0 : this._rowIdCol[index];
+  }
+
+  /**
+   * Applies a DocAction received from the server; returns true, or false if it was skipped.
+   */
+  public receiveAction(action: DocAction): boolean {
+    if (this._isLoaded || isSchemaAction(action)) {
+      this.dispatchAction(action);
+      return true;
+    }
+    return false;
+  }
+
+  // ---- The following methods implement ActionDispatcher interface ----
+
+  protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
+    const index: number = this._rowIdCol.length;
+    this._rowMap.set(rowId, index);
+    this._rowIdCol[index] = rowId;
+    for (const {colId, defl, values} of this._colArray) {
+      values[index] = colValues.hasOwnProperty(colId) ? colValues[colId] : defl;
+    }
+  }
+
+  protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
+    const index: number = this._rowIdCol.length;
+    for (let i = 0; i < rowIds.length; i++) {
+      this._rowMap.set(rowIds[i], index + i);
+      this._rowIdCol[index + i] = rowIds[i];
+    }
+    for (const {colId, defl, values} of this._colArray) {
+      for (let i = 0; i < rowIds.length; i++) {
+        values[index + i] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl;
+      }
+    }
+  }
+
+  protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {
+    // Note that in this implementation, delete + undo will reorder the storage and the ordering
+    // of rows returned getRowIds() and similar methods.
+    const index = this._rowMap.get(rowId);
+    if (index !== undefined) {
+      const last: number = this._rowIdCol.length - 1;
+      // We keep the column-wise arrays dense by moving the last element into the freed-up spot.
+      for (const {values} of this._columns.values()) {    // This adjusts _rowIdCol too.
+        values[index] = values[last];
+        values.pop();
+      }
+      this._rowMap.set(this._rowIdCol[index], index);
+      this._rowMap.delete(rowId);
+    }
+  }
+
+  protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
+    const index = this._rowMap.get(rowId);
+    if (index !== undefined) {
+      for (const colId in colValues) {
+        if (colValues.hasOwnProperty(colId)) {
+          const colData = this._columns.get(colId);
+          if (colData) {
+            colData.values[index] = colValues[colId];
+          }
+        }
+      }
+    }
+  }
+
+  protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
+    for (let i = 0; i < rowIds.length; i++) {
+      const index = this._rowMap.get(rowIds[i]);
+      if (index !== undefined) {
+        for (const colId in colValues) {
+          if (colValues.hasOwnProperty(colId)) {
+            const colData = this._columns.get(colId);
+            if (colData) {
+              colData.values[index] = colValues[colId][i];
+            }
+          }
+        }
+      }
+    }
+  }
+
+  protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
+    this.loadData(action as ReplaceTableData);
+  }
+
+  protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {
+    if (this._columns.has(colId)) { return; }
+    const type = colInfo.type;
+    const defl = getDefaultForType(type);
+    const colData: ColData = { colId, type, defl, values: this._rowIdCol.map(() => defl) };
+    this._columns.set(colId, colData);
+    this._colArray.push(colData);
+  }
+
+  protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {
+    const colData = this._columns.get(colId);
+    if (!colData) { return; }
+    this._columns.delete(colId);
+    arrayRemove(this._colArray, colData);
+  }
+
+  protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {
+    const colData = this._columns.get(oldColId);
+    if (colData) {
+      colData.colId = newColId;
+      this._columns.set(newColId, colData);
+      this._columns.delete(oldColId);
+    }
+  }
+
+  protected onModifyColumn(action: DocAction, tableId: string, oldColId: string, colInfo: ColInfo): void {
+    const colData = this._columns.get(oldColId);
+    if (colData && colInfo.hasOwnProperty('type')) {
+      colData.type = colInfo.type;
+      colData.defl = getDefaultForType(colInfo.type);
+    }
+  }
+
+  protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void {
+    this._tableId = newTableId;
+  }
+
+  protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void {
+    // A table processing its own addition is a noop
+  }
+
+  protected onRemoveTable(action: DocAction, tableId: string): void {
+    // Stop dispatching actions if we've been deleted. We might also want to clean up in the future.
+    this._isLoaded = false;
+  }
+}
+
+function reassignArray<T>(targetArray: T[], sourceArray: T[]): void {
+  targetArray.length = 0;
+  arraySplice(targetArray, 0, sourceArray);
+}

app/common/TabularDiff.ts

@@ -0,0 +1,31 @@
+/**
+ *
+ * Types for use when summarizing differences between versions of a table, with the
+ * diff itself presented in tabular form.
+ *
+ */
+
+
+/**
+ * Pairs of before/after values of cells. Values, when present, are nested in a trivial
+ * list since they can be literally anything - null, undefined, etc.  Otherwise they
+ * are either null, meaning non-existent, or "?", meaning unknown.  Non-existent values
+ * appear prior to a table/column being created, or after it has been destroyed.
+ * Unknown values appear when they are omitted from summaries of bulk actions, and those
+ * summaries are then merged with others.
+ */
+export type CellDelta = [[any]|"?"|null, [any]|"?"|null];
+
+/** a special column indicating what changes happened on row (addition, update, removal) */
+export type RowChangeType = string;
+
+/** differences for an individual table */
+export interface TabularDiff {
+  header: string[];  /** labels for columns */
+  cells: Array<[RowChangeType, number, CellDelta[]]>;  // "number" is rowId
+}
+
+/** differences for a collection of tables */
+export interface TabularDiffs {
+  [tableId: string]: TabularDiff;
+}

app/common/TestState.ts

@@ -0,0 +1,4 @@
+export interface TestState {
+  clipboard?: string;
+  anchorApplied?: boolean;
+}

app/common/UserAPI.ts

@@ -0,0 +1,667 @@
+import {ApplyUAResult} from 'app/common/ActiveDocAPI';
+import {BaseAPI, IOptions} from 'app/common/BaseAPI';
+import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI';
+import {BrowserSettings} from 'app/common/BrowserSettings';
+import {BulkColValues, TableColValues, UserAction} from 'app/common/DocActions';
+import {DocCreationInfo} from 'app/common/DocListAPI';
+import {Features} from 'app/common/Features';
+import {isClient} from 'app/common/gristUrls';
+import {FullUser} from 'app/common/LoginSessionAPI';
+import * as roles from 'app/common/roles';
+import {addCurrentOrgToPath} from 'app/common/urlUtils';
+
+// Nominal email address of the anonymous user.
+export const ANONYMOUS_USER_EMAIL = 'anon@getgrist.com';
+
+// A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource.
+export const SUPPORT_EMAIL = 'support@getgrist.com';
+
+// A special 'docId' that means to create a new document.
+export const NEW_DOCUMENT_CODE = 'new';
+
+// Properties shared by org, workspace, and doc resources.
+export interface CommonProperties {
+  name: string;
+  createdAt: string;  // ISO date string
+  updatedAt: string;  // ISO date string
+  removedAt?: string; // ISO date string - only can appear on docs and workspaces currently
+  public?: boolean;   // If set, resource is available to the public
+}
+export const commonPropertyKeys = ['createdAt', 'name', 'updatedAt'];
+
+export interface OrganizationProperties extends CommonProperties {
+  domain: string|null;
+}
+export const organizationPropertyKeys = [...commonPropertyKeys, 'domain'];
+
+// Basic information about an organization, excluding the user's access level
+export interface OrganizationWithoutAccessInfo extends OrganizationProperties {
+  id: number;
+  owner: FullUser|null;
+  billingAccount?: BillingAccount;
+  host: string|null;  // if set, org's preferred domain (e.g. www.thing.com)
+}
+
+// Organization information plus the user's access level
+export interface Organization extends OrganizationWithoutAccessInfo {
+  access: roles.Role;
+}
+
+// Basic information about a billing account associated with an org or orgs.
+export interface BillingAccount {
+  id: number;
+  individual: boolean;
+  product: Product;
+  isManager: boolean;
+}
+
+// Information about the product associated with an org or orgs.
+export interface Product {
+  name: string;
+  features: Features;
+}
+
+// The upload types vary based on which fetch implementation is in use.  This is
+// an incomplete list.  For example, node streaming types are supported by node-fetch.
+export type UploadType = string | Blob | Buffer;
+
+/**
+ * Returns a user-friendly org name, which is either org.name, or "@User Name" for personal orgs.
+ */
+export function getOrgName(org: Organization): string {
+  return org.owner ? `@` + org.owner.name : org.name;
+}
+
+export type WorkspaceProperties = CommonProperties;
+export const workspacePropertyKeys = ['createdAt', 'name', 'updatedAt'];
+
+export interface Workspace extends WorkspaceProperties {
+  id: number;
+  docs: Document[];
+  org: Organization;
+  access: roles.Role;
+  owner?: FullUser;  // Set when workspaces are in the "docs" pseudo-organization,
+                     // assembled from multiple personal organizations.
+                     // Not set when workspaces are all from the same organization.
+
+  // Set when the workspace belongs to support@getgrist.com. We expect only one such workspace
+  // ("Examples & Templates"), containing sample documents.
+  isSupportWorkspace?: boolean;
+}
+
+export interface DocumentProperties extends CommonProperties {
+  isPinned: boolean;
+  urlId: string|null;
+}
+export const documentPropertyKeys = [...commonPropertyKeys, 'isPinned', 'urlId'];
+
+export interface Document extends DocumentProperties {
+  id: string;
+  workspace: Workspace;
+  access: roles.Role;
+}
+
+export interface PermissionDelta {
+  maxInheritedRole?: roles.BasicRole|null;
+  users?: {
+    // Maps from email to group name, or null to inherit.
+    [email: string]: roles.NonGuestRole|null
+  };
+}
+
+export interface PermissionData {
+  maxInheritedRole?: roles.BasicRole|null;
+  users: UserAccessData[];
+}
+
+// A structure for modifying managers of a billing account.
+export interface ManagerDelta {
+  users: {
+    // To add a manager, link their email to 'managers'.
+    // To remove a manager, link their email to null.
+    // This format is used to rhyme with the ACL PermissionDelta format.
+    [email: string]: 'managers'|null
+  };
+}
+
+// Information about a user and their access to an unspecified resource of interest.
+export interface UserAccessData {
+  id: number;
+  name: string;
+  email: string;
+  picture?: string|null; // When present, a url to a public image of unspecified dimensions.
+  // Represents the user's direct access to the resource of interest. Lack of access to a resource
+  // is represented by a null value.
+  access: roles.Role|null;
+  // A user's parentAccess represent their effective inheritable access to the direct parent of the resource
+  // of interest. The user's effective access to the resource of interest can be determined based
+  // on the user's parentAccess, the maxInheritedRole setting of the resource and the user's direct
+  // access to the resource. Lack of access to the parent resource is represented by a null value.
+  // If parent has non-inheritable access, this should be null.
+  parentAccess?: roles.BasicRole|null;
+}
+
+export interface ActiveSessionInfo {
+  user: FullUser & {helpScoutSignature?: string};
+  org: Organization|null;
+  orgError?: OrgError;
+}
+
+export interface OrgError {
+  error: string;
+  status: number;
+}
+
+/**
+ * Options to control the source of a document being replaced.  For
+ * example, a document could be initialized from another document
+ * (e.g. a fork) or from a snapshot.
+ */
+export interface DocReplacementOptions {
+  sourceDocId?: string;       // docId to copy from
+  snapshotId?: string;        // s3 VersionId
+}
+
+/**
+ * Information about a single document snapshot/backup.
+ */
+export interface DocSnapshot {
+  lastModified: string;  // when the snapshot was made
+  snapshotId: string;    // the id of the snapshot in the underlying store
+  docId: string;         // an id for accessing the snapshot as a Grist document
+}
+
+/**
+ * A list of document snapshots.
+ */
+export interface DocSnapshots {
+  snapshots: DocSnapshot[];  // snapshots, freshest first.
+}
+
+/**
+ * Information about a single document state.
+ */
+export interface DocState {
+  n: number;  // a sequential identifier
+  h: string;  // a hash identifier
+}
+
+/**
+ * A list of document states.  Most recent is first.
+ */
+export interface DocStates {
+  states: DocState[];
+}
+
+/**
+ * A comparison between two documents, called "left" and "right".
+ * The comparison is based on the action histories in the documents.
+ * If those histories have been truncated, the comparison may report
+ * two documents as being unrelated even if they do in fact have some
+ * shared history.
+ */
+export interface DocStateComparison {
+  left: DocState;         // left / local document
+  right: DocState;        // right / remote document
+  parent: DocState|null;  // most recent common ancestor of left and right
+  // summary of the relationship between the two documents.
+  //        same: documents have the same most recent state
+  //        left: the left document has actions not yet in the right
+  //       right: the right document has actions not yet in the left
+  //        both: both documents have changes (possible divergence)
+  //   unrelated: no common history found
+  summary: 'same' | 'left' | 'right' | 'both' | 'unrelated';
+}
+
+export {UserProfile} from 'app/common/LoginSessionAPI';
+
+export interface UserAPI {
+  getSessionActive(): Promise<ActiveSessionInfo>;
+  setSessionActive(email: string): Promise<void>;
+  getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}>;
+  getOrgs(merged?: boolean): Promise<Organization[]>;
+  getWorkspace(workspaceId: number): Promise<Workspace>;
+  getOrg(orgId: number|string): Promise<Organization>;
+  getOrgWorkspaces(orgId: number|string): Promise<Workspace[]>;
+  getDoc(docId: string): Promise<Document>;
+  newOrg(props: Partial<OrganizationProperties>): Promise<number>;
+  newWorkspace(props: Partial<WorkspaceProperties>, orgId: number|string): Promise<number>;
+  newDoc(props: Partial<DocumentProperties>, workspaceId: number): Promise<string>;
+  newUnsavedDoc(options?: {timezone?: string}): Promise<string>;
+  renameOrg(orgId: number|string, name: string): Promise<void>;
+  renameWorkspace(workspaceId: number, name: string): Promise<void>;
+  renameDoc(docId: string, name: string): Promise<void>;
+  updateDoc(docId: string, props: Partial<DocumentProperties>): Promise<void>;
+  deleteOrg(orgId: number|string): Promise<void>;
+  deleteWorkspace(workspaceId: number): Promise<void>;     // delete workspace permanently
+  softDeleteWorkspace(workspaceId: number): Promise<void>; // soft-delete workspace
+  undeleteWorkspace(workspaceId: number): Promise<void>;   // recover soft-deleted workspace
+  deleteDoc(docId: string): Promise<void>;      // delete doc permanently
+  softDeleteDoc(docId: string): Promise<void>;  // soft-delete doc
+  undeleteDoc(docId: string): Promise<void>;    // recover soft-deleted doc
+  updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise<void>;
+  updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise<void>;
+  updateDocPermissions(docId: string, delta: PermissionDelta): Promise<void>;
+  getOrgAccess(orgId: number|string): Promise<PermissionData>;
+  getWorkspaceAccess(workspaceId: number): Promise<PermissionData>;
+  getDocAccess(docId: string): Promise<PermissionData>;
+  pinDoc(docId: string): Promise<void>;
+  unpinDoc(docId: string): Promise<void>;
+  moveDoc(docId: string, workspaceId: number): Promise<void>;
+  getUserProfile(): Promise<FullUser>;
+  updateUserName(name: string): Promise<void>;
+  getWorker(key: string): Promise<string>;
+  getWorkerAPI(key: string): Promise<DocWorkerAPI>;
+  getBillingAPI(): BillingAPI;
+  getDocAPI(docId: string): DocAPI;
+  fetchApiKey(): Promise<string>;
+  createApiKey(): Promise<string>;
+  deleteApiKey(): Promise<void>;
+  getTable(docId: string, tableName: string): Promise<TableColValues>;
+  applyUserActions(docId: string, actions: UserAction[]): Promise<ApplyUAResult>;
+  importUnsavedDoc(material: UploadType, options?: {
+    filename?: string,
+    timezone?: string,
+    onUploadProgress?: (ev: ProgressEvent) => void,
+  }): Promise<string>;
+  deleteUser(userId: number, name: string): Promise<void>;
+  getBaseUrl(): string;  // Get the prefix for all the endpoints this object wraps.
+  forRemoved(): UserAPI; // Get a version of the API that works on removed resources.
+}
+
+/**
+ * Collect endpoints related to the content of a single document that we've been thinking
+ * of as the (restful) "Doc API".  A few endpoints that could be here are not, for historical
+ * reasons, such as downloads.
+ */
+export interface DocAPI {
+  getRows(tableId: string): Promise<TableColValues>;
+  updateRows(tableId: string, changes: TableColValues): Promise<number[]>;
+  addRows(tableId: string, additions: BulkColValues): Promise<number[]>;
+  replace(source: DocReplacementOptions): Promise<void>;
+  getSnapshots(): Promise<DocSnapshots>;
+  forceReload(): Promise<void>;
+  compareState(remoteDocId: string): Promise<DocStateComparison>;
+}
+
+// Operations that are supported by a doc worker.
+export interface DocWorkerAPI {
+  readonly url: string;
+  importDocToWorkspace(uploadId: number, workspaceId: number, settings?: BrowserSettings): Promise<DocCreationInfo>;
+  upload(material: UploadType, filename?: string): Promise<number>;
+  downloadDoc(docId: string, template?: boolean): Promise<Response>;
+  copyDoc(docId: string, template?: boolean, name?: string): Promise<number>;
+}
+
+export class UserAPIImpl extends BaseAPI implements UserAPI {
+  constructor(private _homeUrl: string, private _options: IOptions = {}) {
+    super(_options);
+  }
+
+  public forRemoved(): UserAPI {
+    const extraParameters = new Map<string, string>([['showRemoved', '1']]);
+    return new UserAPIImpl(this._homeUrl, {...this._options, extraParameters});
+  }
+
+  public async getSessionActive(): Promise<ActiveSessionInfo> {
+    return this.requestJson(`${this._url}/api/session/access/active`, {method: 'GET'});
+  }
+
+  public async setSessionActive(email: string): Promise<void> {
+    const body = JSON.stringify({ email });
+    return this.requestJson(`${this._url}/api/session/access/active`, {method: 'POST', body});
+  }
+
+  public async getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}> {
+    return this.requestJson(`${this._url}/api/session/access/all`, {method: 'GET'});
+  }
+
+  public async getOrgs(merged: boolean = false): Promise<Organization[]> {
+    return this.requestJson(`${this._url}/api/orgs?merged=${merged ? 1 : 0}`, { method: 'GET' });
+  }
+
+  public async getWorkspace(workspaceId: number): Promise<Workspace> {
+    return this.requestJson(`${this._url}/api/workspaces/${workspaceId}`, { method: 'GET' });
+  }
+
+  public async getOrg(orgId: number|string): Promise<Organization> {
+    return this.requestJson(`${this._url}/api/orgs/${orgId}`, { method: 'GET' });
+  }
+
+  public async getOrgWorkspaces(orgId: number|string): Promise<Workspace[]> {
+    return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces?includeSupport=1`,
+      { method: 'GET' });
+  }
+
+  public async getDoc(docId: string): Promise<Document> {
+    return this.requestJson(`${this._url}/api/docs/${docId}`, { method: 'GET' });
+  }
+
+  public async newOrg(props: Partial<OrganizationProperties>): Promise<number> {
+    return this.requestJson(`${this._url}/api/orgs`, {
+      method: 'POST',
+      body: JSON.stringify(props)
+    });
+  }
+
+  public async newWorkspace(props: Partial<WorkspaceProperties>, orgId: number|string): Promise<number> {
+    return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces`, {
+      method: 'POST',
+      body: JSON.stringify(props)
+    });
+  }
+
+  public async newDoc(props: Partial<DocumentProperties>, workspaceId: number): Promise<string> {
+    return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/docs`, {
+      method: 'POST',
+      body: JSON.stringify(props)
+    });
+  }
+
+  public async newUnsavedDoc(options: {timezone?: string} = {}): Promise<string> {
+    return this.requestJson(`${this._url}/api/docs`, {
+      method: 'POST',
+      body: JSON.stringify(options),
+    });
+  }
+
+  public async renameOrg(orgId: number|string, name: string): Promise<void> {
+    await this.request(`${this._url}/api/orgs/${orgId}`, {
+      method: 'PATCH',
+      body: JSON.stringify({ name })
+    });
+  }
+
+  public async renameWorkspace(workspaceId: number, name: string): Promise<void> {
+    await this.request(`${this._url}/api/workspaces/${workspaceId}`, {
+      method: 'PATCH',
+      body: JSON.stringify({ name })
+    });
+  }
+
+  public async renameDoc(docId: string, name: string): Promise<void> {
+    return this.updateDoc(docId, {name});
+  }
+
+  public async updateDoc(docId: string, props: Partial<DocumentProperties>): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}`, {
+      method: 'PATCH',
+      body: JSON.stringify(props)
+    });
+  }
+
+  public async deleteOrg(orgId: number|string): Promise<void> {
+    await this.request(`${this._url}/api/orgs/${orgId}`, { method: 'DELETE' });
+  }
+
+  public async deleteWorkspace(workspaceId: number): Promise<void> {
+    await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: 'DELETE' });
+  }
+
+  public async softDeleteWorkspace(workspaceId: number): Promise<void> {
+    await this.request(`${this._url}/api/workspaces/${workspaceId}/remove`, { method: 'POST' });
+  }
+
+  public async undeleteWorkspace(workspaceId: number): Promise<void> {
+    await this.request(`${this._url}/api/workspaces/${workspaceId}/unremove`, { method: 'POST' });
+  }
+
+  public async deleteDoc(docId: string): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}`, { method: 'DELETE' });
+  }
+
+  public async softDeleteDoc(docId: string): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/remove`, { method: 'POST' });
+  }
+
+  public async undeleteDoc(docId: string): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/unremove`, { method: 'POST' });
+  }
+
+  public async updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise<void> {
+    await this.request(`${this._url}/api/orgs/${orgId}/access`, {
+      method: 'PATCH',
+      body: JSON.stringify({ delta })
+    });
+  }
+
+  public async updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise<void> {
+    await this.request(`${this._url}/api/workspaces/${workspaceId}/access`, {
+      method: 'PATCH',
+      body: JSON.stringify({ delta })
+    });
+  }
+
+  public async updateDocPermissions(docId: string, delta: PermissionDelta): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/access`, {
+      method: 'PATCH',
+      body: JSON.stringify({ delta })
+    });
+  }
+
+  public async getOrgAccess(orgId: number|string): Promise<PermissionData> {
+    return this.requestJson(`${this._url}/api/orgs/${orgId}/access`, { method: 'GET' });
+  }
+
+  public async getWorkspaceAccess(workspaceId: number): Promise<PermissionData> {
+    return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/access`, { method: 'GET' });
+  }
+
+  public async getDocAccess(docId: string): Promise<PermissionData> {
+    return this.requestJson(`${this._url}/api/docs/${docId}/access`, { method: 'GET' });
+  }
+
+  public async pinDoc(docId: string): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/pin`, {
+      method: 'PATCH'
+    });
+  }
+
+  public async unpinDoc(docId: string): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/unpin`, {
+      method: 'PATCH'
+    });
+  }
+
+  public async moveDoc(docId: string, workspaceId: number): Promise<void> {
+    await this.request(`${this._url}/api/docs/${docId}/move`, {
+      method: 'PATCH',
+      body: JSON.stringify({ workspace: workspaceId })
+    });
+  }
+
+  public async getUserProfile(): Promise<FullUser> {
+    return this.requestJson(`${this._url}/api/profile/user`);
+  }
+
+  public async updateUserName(name: string): Promise<void> {
+    await this.request(`${this._url}/api/profile/user/name`, {
+      method: 'POST',
+      body: JSON.stringify({name})
+    });
+  }
+
+  public async getWorker(key: string): Promise<string> {
+    const json = await this.requestJson(`${this._url}/api/worker/${key}`, {
+      method: 'GET',
+      credentials: 'include'
+    });
+    return json.docWorkerUrl;
+  }
+
+  public async getWorkerAPI(key: string): Promise<DocWorkerAPI> {
+    const docUrl = this._urlWithOrg(await this.getWorker(key));
+    return new DocWorkerAPIImpl(docUrl, this._options);
+  }
+
+  public getBillingAPI(): BillingAPI {
+    return new BillingAPIImpl(this._url, this._options);
+  }
+
+  public getDocAPI(docId: string): DocAPI {
+    return new DocAPIImpl(this._url, docId, this._options);
+  }
+
+  public async fetchApiKey(): Promise<string> {
+    const resp = await this.fetch(`${this._url}/api/profile/apiKey`, {
+      credentials: 'include'
+    });
+    return await resp.text();
+  }
+
+  public async createApiKey(): Promise<string> {
+    const res = await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'POST'});
+    return await res.text();
+  }
+
+  public async deleteApiKey(): Promise<void> {
+    await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'DELETE'});
+  }
+
+  // This method is not strictly needed anymore, but is widely used by
+  // tests so supporting as a handy shortcut for getDocAPI(docId).getRows(tableName)
+  public async getTable(docId: string, tableName: string): Promise<TableColValues> {
+    return this.getDocAPI(docId).getRows(tableName);
+  }
+
+  public async applyUserActions(docId: string, actions: UserAction[]): Promise<ApplyUAResult> {
+    return this.requestJson(`${this._url}/api/docs/${docId}/apply`, {
+      method: 'POST',
+      body: JSON.stringify(actions)
+    });
+  }
+
+  public async importUnsavedDoc(material: UploadType, options?: {
+    filename?: string,
+    timezone?: string,
+    onUploadProgress?: (ev: ProgressEvent) => void,
+  }): Promise<string> {
+    options = options || {};
+    const formData = this.newFormData();
+    formData.append('upload', material as any, options.filename);
+    if (options.timezone) { formData.append('timezone', options.timezone); }
+    const resp = await this.requestAxios(`${this._url}/api/docs`, {
+      headers: this._options.headers,
+      method: 'POST',
+      data: formData,
+      onUploadProgress: options.onUploadProgress,
+    });
+    return resp.data;
+  }
+
+  public async deleteUser(userId: number, name: string) {
+    await this.request(`${this._url}/api/users/${userId}`,
+                       {method: 'DELETE',
+                        body: JSON.stringify({name})});
+  }
+
+  public getBaseUrl(): string { return this._url; }
+
+  // Recomputes the URL on every call to pick up changes in the URL when switching orgs.
+  // (Feels inefficient, but probably doesn't matter, and it's simpler than the alternatives.)
+  private get _url(): string {
+    return this._urlWithOrg(this._homeUrl);
+  }
+
+  private _urlWithOrg(base: string): string {
+    return isClient() ? addCurrentOrgToPath(base) : base.replace(/\/$/, '');
+  }
+}
+
+export class DocWorkerAPIImpl extends BaseAPI implements DocWorkerAPI {
+  constructor(readonly url: string, private _options: IOptions = {}) {
+    super(_options);
+  }
+
+  public async importDocToWorkspace(uploadId: number, workspaceId: number, browserSettings?: BrowserSettings)
+    : Promise<DocCreationInfo> {
+    return this.requestJson(`${this.url}/api/workspaces/${workspaceId}/import`, {
+      method: 'POST',
+      body: JSON.stringify({ uploadId, browserSettings })
+    });
+  }
+
+  public async upload(material: UploadType, filename?: string): Promise<number> {
+    const formData = this.newFormData();
+    formData.append('upload', material as any, filename);
+    const json = await this.requestJson(`${this.url}/uploads`, {
+      headers: this._options.headers,
+      method: 'POST',
+      body: formData
+    });
+    return json.uploadId;
+  }
+
+  public async downloadDoc(docId: string, template: boolean = false): Promise<Response> {
+    const extra = template ? '&template=1' : '';
+    const result = await this.request(`${this.url}/download?doc=${docId}${extra}`, {
+      headers: this._options.headers,
+      method: 'GET',
+    });
+    if (!result.ok) { throw new Error(await result.text()); }
+    return result;
+  }
+
+  public async copyDoc(docId: string, template: boolean = false, name?: string): Promise<number> {
+    const url = new URL(`${this.url}/copy?doc=${docId}`);
+    if (template) {
+      url.searchParams.append('template', '1');
+    }
+    if (name) {
+      url.searchParams.append('name', name);
+    }
+    const json = await this.requestJson(url.href, {
+      headers: this._options.headers,
+      method: 'POST',
+    });
+    return json.uploadId;
+  }
+}
+
+export class DocAPIImpl extends BaseAPI implements DocAPI {
+  private _url: string;
+
+  constructor(url: string, readonly docId: string, options: IOptions = {}) {
+    super(options);
+    this._url = `${url}/api/docs/${docId}`;
+  }
+
+  public async getRows(tableId: string): Promise<TableColValues> {
+    return this.requestJson(`${this._url}/tables/${tableId}/data`);
+  }
+
+  public async updateRows(tableId: string, changes: TableColValues): Promise<number[]> {
+    return this.requestJson(`${this._url}/tables/${tableId}/data`, {
+      body: JSON.stringify(changes),
+      method: 'PATCH'
+    });
+  }
+
+  public async addRows(tableId: string, additions: BulkColValues): Promise<number[]> {
+    return this.requestJson(`${this._url}/tables/${tableId}/data`, {
+      body: JSON.stringify(additions),
+      method: 'POST'
+    });
+  }
+
+  public async replace(source: DocReplacementOptions): Promise<void> {
+    return this.requestJson(`${this._url}/replace`, {
+      body: JSON.stringify(source),
+      method: 'POST'
+    });
+  }
+
+  public async getSnapshots(): Promise<DocSnapshots> {
+    return this.requestJson(`${this._url}/snapshots`);
+  }
+
+  public async forceReload(): Promise<void> {
+    await this.request(`${this._url}/force-reload`, {
+      method: 'POST'
+    });
+  }
+
+  public async compareState(remoteDocId: string): Promise<DocStateComparison> {
+    return this.requestJson(`${this._url}/compare/${remoteDocId}`);
+  }
+}

app/common/UserConfig.ts

@@ -0,0 +1,31 @@
+/*
+ * Interface for the user's config found in config.json.
+ */
+export interface UserConfig {
+  enableMetrics?: boolean;
+  docListSortBy?: string;
+  docListSortDir?: number;
+  features?: ISupportedFeatures;
+
+  /*
+   * The host serving the untrusted content: on dev environment could be
+   * "http://getgrist.localtest.me". Port is added at runtime and should not be included.
+   */
+  untrustedContentOrigin?: string;
+}
+
+export interface ISupportedFeatures {
+  signin?: boolean;
+  sharing?: boolean;
+  proxy?: boolean;  // If true, Grist will accept login information via http headers
+  // X-Forwarded-User and X-Forwarded-Email.  Set to true only if
+  // Grist is behind a reverse proxy that is managing those headers,
+  // otherwise they could be spoofed.
+  formulaBar?: boolean;
+
+  // Plugin views, REPL, and Validations all need work, but are exposed here to allow existing
+  // tests to continue running. These only affect client-side code.
+  customViewPlugin?: boolean;
+  replTool?: boolean;
+  validationsTool?: boolean;
+}

app/common/ValueFormatter.ts

@@ -0,0 +1,144 @@
+// tslint:disable:max-classes-per-file
+
+import {CellValue} from 'app/common/DocActions';
+import * as gristTypes from 'app/common/gristTypes';
+import * as gutil from 'app/common/gutil';
+import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
+import * as moment from 'moment-timezone';
+
+// Some text to show on cells whose values are pending.
+export const PENDING_DATA_PLACEHOLDER = "Loading...";
+
+/**
+ * Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
+ * E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
+ */
+export function formatObject(args: [string, ...any[]]): string {
+  const objType = args[0], objArgs = args.slice(1);
+  switch (objType) {
+    case 'L': return JSON.stringify(objArgs);
+    // First arg is seconds since epoch (moment takes ms), second arg is timezone
+    case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
+    case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
+    case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
+    case 'E': return gristTypes.formatError(args);
+    case 'P': return PENDING_DATA_PLACEHOLDER;
+  }
+  return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
+}
+
+/**
+ * Formats a value of unknown type, using formatObject() for encoded objects.
+ */
+export function formatUnknown(value: any): string {
+  return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
+}
+
+export type IsRightTypeFunc = (value: CellValue) => boolean;
+
+export class BaseFormatter {
+  public readonly isRightType: IsRightTypeFunc;
+
+  constructor(public type: string, public opts: object) {
+    this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
+      gristTypes.isRightType('Any')!;
+  }
+
+  /**
+   * Formats a value that matches the type of this formatter. This should be overridden by derived
+   * classes to handle values in formatter-specific ways.
+   */
+  public format(value: any): string {
+    return value;
+  }
+
+  /**
+   * Formats using this.format() if a value is of the right type for this formatter, or using
+   * AnyFormatter otherwise. This method the recommended API. There is no need to override it.
+   */
+  public formatAny(value: any): string {
+    return this.isRightType(value) ? this.format(value) : formatUnknown(value);
+  }
+}
+
+class AnyFormatter extends BaseFormatter {
+  public format(value: any): string {
+    return formatUnknown(value);
+  }
+}
+
+export class NumericFormatter extends BaseFormatter {
+  private _numFormat: Intl.NumberFormat;
+  private _formatter: (val: number) => string;
+
+  constructor(type: string, options: NumberFormatOptions) {
+    super(type, options);
+    this._numFormat = buildNumberFormat(options);
+    this._formatter = (options.numSign === 'parens') ? this._formatParens : this._formatPlain;
+  }
+
+  public format(value: any): string {
+    return value === null ? '' : this._formatter(value);
+  }
+
+  public _formatPlain(value: number): string {
+    return this._numFormat.format(value);
+  }
+
+  public _formatParens(value: number): string {
+    // Surround positive numbers with spaces to align them visually to parenthesized numbers.
+    return (value >= 0) ?
+      ` ${this._numFormat.format(value)} ` :
+      `(${this._numFormat.format(-value)})`;
+  }
+}
+
+class IntFormatter extends NumericFormatter {
+  constructor(type: string, opts: object) {
+    super(type, {decimals: 0, ...opts});
+  }
+}
+
+class DateFormatter extends BaseFormatter {
+  private _dateTimeFormat: string;
+  private _timezone: string;
+
+  constructor(type: string, opts: {dateFormat?: string}, timezone: string = 'UTC') {
+    super(type, opts);
+    this._dateTimeFormat = opts.dateFormat || 'YYYY-MM-DD';
+    this._timezone = timezone;
+  }
+
+  public format(value: any): string {
+    if (value === null) { return ''; }
+    const time = moment.tz(value * 1000, this._timezone);
+    return time.format(this._dateTimeFormat);
+  }
+}
+
+class DateTimeFormatter extends DateFormatter {
+  constructor(type: string, opts: {dateFormat?: string; timeFormat?: string}) {
+    const timezone = gutil.removePrefix(type, "DateTime:") || '';
+    const timeFormat = opts.timeFormat === undefined ? 'h:mma' : opts.timeFormat;
+    const dateFormat = (opts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
+    super(type, {dateFormat}, timezone);
+  }
+}
+
+const formatters: {[name: string]: typeof BaseFormatter} = {
+  Numeric: NumericFormatter,
+  Int: IntFormatter,
+  Bool: BaseFormatter,
+  Date: DateFormatter,
+  DateTime: DateTimeFormatter,
+  // We don't list anything that maps to AnyFormatter, since that's the default.
+};
+
+/**
+ * Takes column type and widget options and returns a constructor with a format function that can
+ * properly convert a value passed to it into the right format for that column.
+ */
+export function createFormatter(type: string, opts: object): BaseFormatter {
+  const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
+  return new ctor(type, opts);
+}

app/common/arrayToString.ts

@@ -0,0 +1,28 @@
+/**
+ * Functions to convert between an array of bytes and a string. The implementations are
+ * different for Node and for the browser.
+ */
+
+declare const TextDecoder: any, TextEncoder: any;
+
+export let arrayToString: (data: Uint8Array) => string;
+export let stringToArray: (data: string) => Uint8Array;
+
+if (typeof TextDecoder !== 'undefined') {
+  // Note that constructing a TextEncoder/Decoder takes time, so it's faster to reuse.
+  const dec = new TextDecoder('utf8');
+  const enc = new TextEncoder('utf8');
+  arrayToString = function(uint8Array: Uint8Array): string {
+    return dec.decode(uint8Array);
+  };
+  stringToArray = function(str: string): Uint8Array {
+    return enc.encode(str);
+  };
+} else {
+  arrayToString = function(uint8Array: Uint8Array): string {
+    return Buffer.from(uint8Array).toString('utf8');
+  };
+  stringToArray = function(str: string): Uint8Array {
+    return new Uint8Array(Buffer.from(str, 'utf8'));
+  };
+}

app/common/declarations.d.ts

@@ -0,0 +1,5 @@
+declare module "app/common/MemBuffer" {
+  const MemBuffer: any;
+  type MemBuffer = any;
+  export = MemBuffer;
+}

app/common/delay.ts

@@ -0,0 +1,7 @@
+/**
+ * Returns a promise that resolves in the given number of milliseconds.
+ * (A replica of bluebird.delay using native promises.)
+ */
+export function delay(msec: number): Promise<void> {
+  return new Promise<void>((resolve) => setTimeout(resolve, msec));
+}

app/common/emails.ts

@@ -0,0 +1,43 @@
+/**
+ *
+ * Utilities related to email normalization.  Currently
+ * trivial, but could potentially need special per-domain
+ * rules in future.
+ *
+ * Email addresses are a bit slippery.  Domain names are
+ * case insensitive, but user names may or may not be,
+ * depending on the mail server handling the domain.
+ * Other special treatment of user names may also be in
+ * place for particular domains (periods, plus sign, etc).
+ *
+ * We treat emails as case-insensitive for the purposes
+ * of determining equality of emails, and indexing users
+ * by email address.
+ *
+ */
+
+/**
+ *
+ * Convert the supplied email address to a normalized form
+ * that we will use for indexing and equality tests.
+ * Many possible email addresses could map to the same
+ * normalized result; as far as we are concerned those
+ * addresses are equivalent.
+ *
+ * The normalization we do is a simple lowercase.  This
+ * means we won't be able to treat both Jane@x.y and
+ * jane@x.y as separate email addresses, even through
+ * they may in fact be separate mailboxes on x.y.
+ *
+ * The normalized email is not something we should show
+ * the user in the UI, but is rather for internal purposes.
+ *
+ * The original non-normalized email is called a
+ * "display email" to distinguish it from a "normalized
+ * email"
+ *
+ */
+export function normalizeEmail(displayEmail: string): string {
+  // We take the lower case, without use of locale.
+  return displayEmail.toLowerCase();
+}

app/common/gristTypes.ts

@@ -0,0 +1,242 @@
+import {CellValue} from 'app/common/DocActions';
+import isString = require('lodash/isString');
+
+// tslint:disable:object-literal-key-quotes
+
+export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
+  'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
+
+// Letter codes for CellValue types encoded as [code, args...] tuples.
+export type GristObjType = 'L' | 'D' | 'd' | 'R' | 'E' | 'P';
+
+export const MANUALSORT = 'manualSort';
+
+// This mapping includes both the default value, and its representation for SQLite.
+const _defaultValues: {[key in GristType]: [CellValue, string]} = {
+  'Any':              [ null,  "NULL"  ],
+  'Attachments':      [ null,  "NULL"  ],
+  'Blob':             [ null,  "NULL"  ],
+  // Bool is only supported by SQLite as 0 and 1 values.
+  'Bool':             [ false, "0" ],
+  'Choice':           [ '',    "''"    ],
+  'Date':             [ null,  "NULL"  ],
+  'DateTime':         [ null,  "NULL"  ],
+  'Id':               [ 0,     "0"     ],
+  'Int':              [ 0,     "0"     ],
+  // Note that "1e999" is a way to store Infinity into SQLite. This is verified by "Defaults"
+  // tests in DocStorage.js. See also http://sqlite.1065341.n5.nabble.com/Infinity-td55327.html.
+  'ManualSortPos':    [ Number.POSITIVE_INFINITY, "1e999" ],
+  'Numeric':          [ 0,     "0"     ],
+  'PositionNumber':   [ Number.POSITIVE_INFINITY, "1e999" ],
+  'Ref':              [ 0,     "0"     ],
+  'RefList':          [ null,  "NULL"  ],
+  'Text':             [ '',    "''"    ],
+};
+
+
+/**
+ * Given a grist column type (e.g Text, Numeric, ...) returns the default value for that type.
+ * If options.sqlFormatted is true, returns the representation of the value for SQLite.
+ */
+export function getDefaultForType(colType: string, options: {sqlFormatted?: boolean} = {}) {
+  const type = extractTypeFromColType(colType);
+  return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
+}
+
+/**
+ * Returns whether a value (as received in a DocAction) represents a custom object.
+ */
+export function isObject(value: CellValue): value is [string, any?] {
+  return Array.isArray(value);
+}
+
+/**
+ * Returns whether a value (as received in a DocAction) represents a raised exception.
+ */
+export function isRaisedException(value: CellValue): boolean {
+  return Array.isArray(value) && value[0] === 'E';
+}
+
+/**
+ * Returns whether a value (as received in a DocAction) represents a list or is null,
+ * which is a valid value for list types in grist.
+ */
+export function isListOrNull(value: CellValue): boolean {
+  return value === null || (Array.isArray(value) && value[0] === 'L');
+}
+
+/**
+ * Returns whether a value (as received in a DocAction) represents an empty list.
+ */
+export function isEmptyList(value: CellValue): boolean {
+  return Array.isArray(value) && value.length === 1 && value[0] === 'L';
+}
+
+/**
+ * Returns whether a value (as received in a DocAction) represents a "Pending" value.
+ */
+export function isPending(value: CellValue): boolean {
+  return Array.isArray(value) && value[0] === 'P';
+}
+
+/**
+ * Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
+ * This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
+ */
+export function formatError(value: [string, ...any[]]): string {
+  const errName = value[1];
+  switch (errName) {
+    case 'ZeroDivisionError': return '#DIV/0!';
+    case 'UnmarshallableError': return value[3] || ('#' + errName);
+    case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
+  }
+  return '#' + errName;
+}
+
+function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
+function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
+function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }
+
+function isNormalValue(value: CellValue) {
+  return !(Array.isArray(value) && (value[0] === 'E' || value[0] === 'P'));
+}
+
+/**
+ * Map of Grist type to an "isRightType" checker function, which determines if a given values type
+ * matches the declared type of the column.
+ */
+const rightType: {[key in GristType]: (value: CellValue) => boolean} = {
+  Any:            isNormalValue,
+  Attachments:    isListOrNull,
+  Text:           isString,
+  Blob:           isString,
+  Int:            isNumberOrNull,
+  Bool:           isBoolean,
+  Date:           isNumberOrNull,
+  DateTime:       isNumberOrNull,
+  Numeric:        isNumberOrNull,
+  Id:             isNumber,
+  PositionNumber: isNumber,
+  ManualSortPos:  isNumber,
+  Ref:            isNumber,
+  RefList:        isListOrNull,
+  Choice:         (v: CellValue, options?: any) => {
+    // TODO widgets options should not be used outside of the client. They are an instance of
+    // modelUtil.jsonObservable, passed in by FieldBuilder.
+    if (v === '') {
+      // Accept empty-string values as valid
+      return true;
+    } else if (options) {
+      const choices = options().choices;
+      return Array.isArray(choices) && choices.includes(v);
+    } else {
+      return false;
+    }
+  }
+};
+
+export function isRightType(type: string): undefined | ((value: CellValue) => boolean) {
+  return rightType[type as GristType];
+}
+
+export function extractTypeFromColType(type: string): string {
+  if (!type) { return type; }
+  const colon = type.indexOf(':');
+  return (colon === -1 ? type : type.slice(0, colon));
+}
+
+/**
+ * Convert pureType to Grist python type name, e.g. 'Ref' to 'Reference'.
+ */
+export function getGristType(pureType: string): string {
+  switch (pureType) {
+    case 'Ref': return 'Reference';
+    case 'RefList': return 'ReferenceList';
+    default: return pureType;
+  }
+}
+
+/**
+ * Converts SQL type strings produced by the Sequelize library into its corresponding
+ * Grist type. The list of types is based on an analysis of SQL type string outputs
+ * produced by the Sequelize library (mostly covered in lib/data-types.js). Some
+ * additional engine/dialect specific types are detailed in dialect directories.
+ *
+ * TODO: A handful of exotic SQL types (mostly from PostgreSQL) will currently throw an
+ * Error, rather than returning a type. Further testing is required to determine
+ * whether Grist can manage those data types.
+ *
+ * @param  {String} sqlType A string produced by Sequelize's describeTable query
+ * @return {String}         The corresponding Grist type string
+ * @throws {Error}          If the sqlType is unrecognized or unsupported
+ */
+export function sequelizeToGristType(sqlType: string): GristType {
+  // Sequelize type strings can include parens (e.g., `CHAR(10)`). This function
+  // ignores those additional details when determining the Grist type.
+  let endMarker = sqlType.length;
+  const parensMarker = sqlType.indexOf('(');
+  endMarker = parensMarker > 0 ? parensMarker : endMarker;
+
+  // Type strings might also include a space after the basic type description.
+  // The type `DOUBLE PRECISION` is one such example, but modifiers or attributes
+  // relevant to the type might also appear after the type itself (e.g., UNSIGNED,
+  // NONZERO). These are ignored when determining the Grist type.
+  const spaceMarker = sqlType.indexOf(' ');
+  endMarker = spaceMarker > 0 && spaceMarker < endMarker ? spaceMarker : endMarker;
+
+  switch (sqlType.substring(0, endMarker)) {
+    case 'INTEGER':
+    case 'BIGINT':
+    case 'SMALLINT':
+    case 'INT':
+      return 'Int';
+    case 'NUMBER':
+    case 'FLOAT':
+    case 'DECIMAL':
+    case 'NUMERIC':
+    case 'REAL':
+    case 'DOUBLE':
+    case 'DOUBLE PRECISION':
+      return 'Numeric';
+    case 'BOOLEAN':
+    case 'TINYINT':
+      return 'Bool';
+    case 'STRING':
+    case 'CHAR':
+    case 'TEXT':
+    case 'UUID':
+    case 'UUIDV1':
+    case 'UUIDV4':
+    case 'VARCHAR':
+    case 'NVARCHAR':
+    case 'TINYTEXT':
+    case 'MEDIUMTEXT':
+    case 'LONGTEXT':
+    case 'ENUM':
+      return 'Text';
+    case 'TIME':
+    case 'DATE':
+    case 'DATEONLY':
+    case 'DATETIME':
+    case 'NOW':
+      return 'Text';
+    case 'BLOB':
+    case 'TINYBLOB':
+    case 'MEDIUMBLOB':
+    case 'LONGBLOB':
+      // TODO: Passing binary data to the Sandbox is throwing Errors. Proper support
+      // for these Blob data types requires some more investigation.
+      throw new Error('SQL type: `' + sqlType + '` is currently unsupported');
+    case 'NONE':
+    case 'HSTORE':
+    case 'JSON':
+    case 'JSONB':
+    case 'VIRTUAL':
+    case 'ARRAY':
+    case 'RANGE':
+    case 'GEOMETRY':
+      throw new Error('SQL type: `' + sqlType + '` is currently untested');
+    default:
+      throw new Error('Unrecognized datatype: `' + sqlType + '`');
+  }
+}

app/common/gristUrls.ts

@@ -1 +1,583 @@
-export type ProductFlavor = 'grist';
+import {BillingPage, BillingSubPage, BillingTask} from 'app/common/BillingAPI';
+import {OpenDocMode} from 'app/common/DocListAPI';
+import {encodeQueryParams} from 'app/common/gutil';
+import {localhostRegex} from 'app/common/LoginState';
+import {Document} from 'app/common/UserAPI';
+import identity = require('lodash/identity');
+import pickBy = require('lodash/pickBy');
+import {StringUnion} from './StringUnion';
+
+export type IDocPage = number | 'new' | 'code';
+
+// What page to show in the user's home area. Defaults to 'workspace' if a workspace is set, and
+// to 'all' otherwise.
+export const HomePage = StringUnion('all', 'workspace', 'trash');
+export type IHomePage = typeof HomePage.type;
+
+export const WelcomePage = StringUnion('user', 'teams');
+export type WelcomePage = typeof WelcomePage.type;
+
+// Default subdomain for home api service if not otherwise specified.
+export const DEFAULT_HOME_SUBDOMAIN = 'api';
+
+// This is the minimum length a urlId may have if it is chosen
+// as a prefix of the docId.
+export const MIN_URLID_PREFIX_LENGTH = 12;
+
+/**
+ * Special ways to open a document, based on what the user intends to do.
+ *   - view: Open document in read-only mode (even if user has edit rights)
+ *   - fork: Open document in fork-ready mode.  This means that while edits are
+ *           permitted, those edits should go to a copy of the document rather than
+ *           the original.
+ */
+
+export const commonUrls = {
+  help: "https://support.getgrist.com",
+  plans: "https://www.getgrist.com/pricing",
+
+  efcrConnect: 'https://efc-r.com/connect',
+  efcrHelp: 'https://www.nioxus.info/eFCR-Help',
+};
+
+/**
+ * Values representable in a URL. The current state is available as urlState().state observable
+ * in client. Updates to this state are expected by functions such as makeUrl() and setLinkUrl().
+ */
+export interface IGristUrlState {
+  org?: string;
+  homePage?: IHomePage;
+  ws?: number;
+  doc?: string;
+  slug?: string;       // if present, this is based on the document title, and is not a stable id
+  mode?: OpenDocMode;
+  fork?: UrlIdParts;
+  docPage?: IDocPage;
+  newui?: boolean;
+  billing?: BillingPage;
+  welcome?: WelcomePage;
+  params?: {
+    billingPlan?: string;
+    billingTask?: BillingTask;
+  };
+  hash?: HashLink;   // if present, this specifies an individual row within a section of a page.
+}
+
+// Subset of GristLoadConfig used by getOrgUrlInfo(), which affects the interpretation of the
+// current URL.
+export interface OrgUrlOptions {
+  // The org associated with the current URL.
+  org?: string;
+
+  // Base domain for constructing new URLs, should start with "." and not include port, e.g.
+  // ".getgrist.com". It should be unset for localhost operation and in single-org mode.
+  baseDomain?: string;
+
+  // In single-org mode, this is the single well-known org.
+  singleOrg?: string;
+}
+
+// Result of getOrgUrlInfo().
+export interface OrgUrlInfo {
+  hostname?: string;      // If hostname should be changed to access the requested org.
+  orgInPath?: string;     // If /o/{orgInPath} should be used to access the requested org.
+}
+
+export function isCustomHost(hostname: string, baseDomain: string) {
+  // .localtest.me is used by plugin tests for arcane reasons.
+  return hostname !== 'localhost' && !hostname.endsWith(baseDomain) && !hostname.endsWith('.localtest.me');
+}
+
+export function getOrgUrlInfo(newOrg: string, currentHostname: string, options: OrgUrlOptions): OrgUrlInfo {
+  if (newOrg === options.singleOrg) {
+    return {};
+  }
+  if (!options.baseDomain || currentHostname === 'localhost') {
+    return {orgInPath: newOrg};
+  }
+  if (newOrg === options.org && isCustomHost(currentHostname, options.baseDomain)) {
+    return {};
+  }
+  return {hostname: newOrg + options.baseDomain};
+}
+
+/**
+ * The actual serialization of a url state into a URL. The URL has the form
+ *    <org-base>/
+ *    <org-base>/ws/<ws>/
+ *    <org-base>/doc/<doc>[/p/<docPage>]
+ *
+ * where <org-base> depends on whether subdomains are in use, e.g.
+ *    <org>.getgrist.com
+ *    localhost:8080/o/<org>
+ */
+export function encodeUrl(gristConfig: Partial<GristLoadConfig>,
+                          state: IGristUrlState, baseLocation: Location | URL): string {
+  const url = new URL(baseLocation.href);
+  const parts = ['/'];
+
+  if (state.org) {
+    // We figure out where to stick the org using the gristConfig and the current host.
+    const {hostname, orgInPath} = getOrgUrlInfo(state.org, baseLocation.hostname, gristConfig);
+    if (hostname) {
+      url.hostname = hostname;
+    }
+    if (orgInPath) {
+      parts.push(`o/${orgInPath}/`);
+    }
+  }
+
+  if (state.ws) { parts.push(`ws/${state.ws}/`); }
+  if (state.doc) {
+    if (state.slug) {
+      parts.push(`${encodeURIComponent(state.doc)}/${encodeURIComponent(state.slug)}`);
+    } else {
+      parts.push(`doc/${encodeURIComponent(state.doc)}`);
+    }
+    if (state.mode && parseOpenDocMode(state.mode)) {
+      parts.push(`/m/${state.mode}`);
+    }
+    if (state.docPage) {
+      parts.push(`/p/${state.docPage}`);
+    }
+  } else {
+    if (state.homePage === 'trash') { parts.push('p/trash'); }
+  }
+
+  if (state.billing) {
+    parts.push(state.billing === 'billing' ? 'billing' : `billing/${state.billing}`);
+  }
+
+  if (state.welcome) {
+    parts.push(`welcome/${state.welcome}`);
+  }
+
+  const queryParams = pickBy(state.params, identity) as {[key: string]: string};
+  if (state.newui !== undefined) {
+    queryParams.newui = state.newui ? '1' : '0';
+  }
+  const hashParts: string[] = [];
+  if (state.hash && state.hash.rowId) {
+    const hash = state.hash;
+    hashParts.push(`a1`);
+    for (const key of ['sectionId', 'rowId', 'colRef'] as Array<keyof HashLink>) {
+      if (hash[key]) { hashParts.push(`${key[0]}${hash[key]}`); }
+    }
+  }
+  const queryStr = encodeQueryParams(queryParams);
+  url.pathname = parts.join('');
+  url.search = queryStr;
+  if (state.hash) {
+    // Project tests use hashes, so only set hash if there is an anchor.
+    url.hash = hashParts.join('.');
+  }
+  return url.href;
+}
+
+/**
+ * Parse a URL location into an IGristUrlState object. See encodeUrl() documentation.
+ */
+export function decodeUrl(gristConfig: Partial<GristLoadConfig>, location: Location | URL): IGristUrlState {
+  const parts = location.pathname.slice(1).split('/');
+  const map = new Map<string, string>();
+  for (let i = 0; i < parts.length; i += 2) {
+    map.set(parts[i], decodeURIComponent(parts[i + 1]));
+  }
+  // When the urlId is a prefix of the docId, documents are identified
+  // as "<urlId>/slug" instead of "doc/<urlId>".  We can detect that because
+  // the minimum length of a urlId prefix is longer than the maximum length
+  // of any of the valid keys in the url.
+  for (const key of map.keys()) {
+    if (key.length >= MIN_URLID_PREFIX_LENGTH) {
+      map.set('doc', key);
+      map.set('slug', map.get(key)!);
+      map.delete(key);
+      break;
+    }
+  }
+
+  const state: IGristUrlState = {};
+  const subdomain = parseSubdomain(location.host);
+  if (gristConfig.org || gristConfig.singleOrg) {
+    state.org = gristConfig.org || gristConfig.singleOrg;
+  } else if (!gristConfig.pathOnly && subdomain.org) {
+    state.org = subdomain.org;
+  }
+  const sp = new URLSearchParams(location.search);
+  if (location.search) { state.params = {}; }
+  if (map.has('o')) { state.org = map.get('o'); }
+  if (map.has('ws')) { state.ws = parseInt(map.get('ws')!, 10); }
+  if (map.has('doc')) {
+    state.doc = map.get('doc');
+    const fork = parseUrlId(map.get('doc')!);
+    if (fork.forkId) { state.fork = fork; }
+    if (map.has('slug')) { state.slug = map.get('slug'); }
+    if (map.has('p')) { state.docPage = parseDocPage(map.get('p')!); }
+  } else {
+    if (map.has('p')) {
+      const p = map.get('p')!;
+      state.homePage = HomePage.guard(p) ? p : undefined;
+    }
+  }
+  if (map.has('m')) { state.mode = parseOpenDocMode(map.get('m')!); }
+  if (sp.has('newui')) { state.newui = useNewUI(sp.get('newui') ? sp.get('newui') === '1' : undefined); }
+  if (map.has('billing')) { state.billing = parseBillingPage(map.get('billing')!); }
+  if (map.has('welcome')) { state.welcome = parseWelcomePage(map.get('welcome')!); }
+  if (sp.has('billingPlan')) { state.params!.billingPlan = sp.get('billingPlan')!; }
+  if (sp.has('billingTask')) {
+    state.params!.billingTask = parseBillingTask(sp.get('billingTask')!);
+  }
+  if (location.hash) {
+    const hash = location.hash;
+    const hashParts = hash.split('.');
+    const hashMap = new Map<string, string>();
+    for (const part of hashParts) {
+      hashMap.set(part.slice(0, 1), part.slice(1));
+    }
+    if (hashMap.has('#') && hashMap.get('#') === 'a1') {
+      const link: HashLink = {};
+      for (const key of ['sectionId', 'rowId', 'colRef'] as Array<keyof HashLink>) {
+        const ch = key.substr(0, 1);
+        if (hashMap.has(ch)) { link[key] = parseInt(hashMap.get(ch)!, 10); }
+      }
+      state.hash = link;
+    }
+  }
+  return state;
+}
+
+export function useNewUI(newui: boolean|undefined) {
+  return newui !== false;
+}
+
+/**
+ * parseDocPage is a noop if p is 'new' or 'code', otherwise parse to integer
+ */
+function parseDocPage(p: string) {
+  if (['new', 'code'].includes(p)) {
+    return p as 'new'|'code';
+  }
+  return parseInt(p, 10);
+}
+
+/**
+ * parseBillingPage ensures that the billing page value is a valid BillingPageType.
+ */
+function parseBillingPage(p: string): BillingPage {
+  return BillingSubPage.guard(p) ? p : 'billing';
+}
+
+/**
+ * parseBillingTask ensures that the value is a valid BillingTask or undefined.
+ */
+function parseBillingTask(t: string): BillingTask|undefined {
+  return BillingTask.guard(t) ? t : undefined;
+}
+
+/**
+ * parseOpenDocMode ensures that the value is a valid OpenDocMode or undefined.
+ */
+function parseOpenDocMode(p: string): OpenDocMode|undefined {
+  return OpenDocMode.guard(p) ? p : undefined;
+}
+
+/**
+ * parse welcome page ensure that the value is a valid WelcomePage, default to 'user' if not.
+ */
+function parseWelcomePage(p: string): WelcomePage {
+  return WelcomePage.guard(p) ? p : 'user';
+}
+
+/**
+ * Parses the URL like "foo.bar.baz" into the pair {org: "foo", base: ".bar.baz"}.
+ * Port is allowed and included into base.
+ *
+ * The "base" part is required to have at least two periods.  The "org" part must pass
+ * the subdomainRegex test.
+ *
+ * If there's no way to parse the URL into such a pair, then an empty object is returned.
+ */
+export function parseSubdomain(host: string|undefined): {org?: string, base?: string} {
+  if (!host) { return {}; }
+  const match = /^([^.]+)(\..+\..+)$/.exec(host.toLowerCase());
+  if (match) {
+    const org = match[1];
+    const base = match[2];
+    if (subdomainRegex.exec(org)) {
+      return {org, base};
+    }
+  }
+  // Host has nowhere to put a subdomain.
+  return {};
+}
+
+/**
+ * Like parseSubdomain, but throws an error if neither of these cases apply:
+ *   - host can be parsed into a valid subdomain and a valid base domain.
+ *   - host is localhost:NNNN
+ * An empty object is only returned when host is localhost:NNNN.
+ */
+export function parseSubdomainStrictly(host: string|undefined): {org?: string, base?: string} {
+  if (!host) { throw new Error('host not known'); }
+  const result = parseSubdomain(host);
+  if (result.org) { return result; }
+  if (!host.match(localhostRegex)) {
+    throw new Error(`host not understood: ${host}`);
+  }
+  // Host is localhost[:NNNN], no org available.
+  return {};
+}
+
+/**
+ * These settings get sent to the client along with the loaded page. At the minimum, the browser
+ * needs to know the URL of the home API server (e.g. api.getgrist.com).
+ */
+export interface GristLoadConfig {
+  // URL of the Home API server for the browser client to use.
+  homeUrl: string|null;
+
+  // When loading /doc/{docId}, we include the id used to assign the document (this is the docId).
+  assignmentId?: string;
+
+  // Org or "subdomain". When present, this overrides org information from the hostname. We rely
+  // on this for custom domains, but set it generally for all pages.
+  org?: string;
+
+  // Base domain for constructing new URLs, should start with "." and not include port, e.g.
+  // ".getgrist.com". It should be unset for localhost operation and in single-org mode.
+  baseDomain?: string;
+
+  // In single-org mode, this is the single well-known org. Suppress any org selection UI.
+  singleOrg?: string;
+
+  // When set, this directs the client to encode org information in path, not in domain.
+  pathOnly?: boolean;
+
+  // Type of error page to show. This is used for pages such as "signed-out" and "not-found",
+  // which don't include the full app.
+  errPage?: string;
+
+  // When errPage is a generic "other-error", this is the message to show.
+  errMessage?: string;
+
+  // URL for client to use for untrusted content.
+  pluginUrl?: string|null;
+
+  // Stripe API key for use on the client.
+  stripeAPIKey?: string;
+
+  // BeaconID for the support widget from HelpScout.
+  helpScoutBeaconId?: string;
+
+  // If set, enable anonymous sharing UI elements.
+  supportAnon?: boolean;
+
+  // Max upload allowed for imports (except .grist files), in bytes; 0 or omitted for unlimited.
+  maxUploadSizeImport?: number;
+
+  // Max upload allowed for attachments, in bytes; 0 or omitted for unlimited.
+  maxUploadSizeAttachment?: number;
+
+  // Pre-fetched call to getDoc for the doc being loaded.
+  getDoc?: {[id: string]: Document};
+
+  // Pre-fetched call to getWorker for the doc being loaded.
+  getWorker?: {[id: string]: string};
+
+  // The timestamp when this gristConfig was generated.
+  timestampMs: number;
+}
+
+// Acceptable org subdomains are alphanumeric (hyphen also allowed) and of
+// non-zero length.
+const subdomainRegex = /^[-a-z0-9]+$/i;
+
+export interface OrgParts {
+  subdomain: string|null;
+  orgFromHost: string|null;
+  orgFromPath: string|null;
+  pathRemainder: string;
+  mismatch: boolean;
+}
+
+/**
+ * Returns true if code is running in client, false if running in server.
+ */
+export function isClient() {
+  return (typeof window !== 'undefined') && window && window.location && window.location.hostname;
+}
+
+/**
+ * Returns a known org "subdomain" if Grist is configured in single-org mode
+ * (GRIST_SINGLE_ORG=<org> on the server) or if the page includes an org in gristConfig.
+ */
+export function getKnownOrg(): string|null {
+  if (isClient()) {
+    const gristConfig: GristLoadConfig = (window as any).gristConfig;
+    return (gristConfig && gristConfig.org) || null;
+  } else {
+    return process.env.GRIST_SINGLE_ORG || null;
+  }
+}
+
+/**
+ * Returns true if org must be encoded in path, not in domain.  Determined from
+ * gristConfig on the client.  On on the server returns true if the host is
+ * supplied and is 'localhost', or if GRIST_ORG_IN_PATH is set to 'true'.
+ */
+export function isOrgInPathOnly(host?: string): boolean {
+  if (isClient()) {
+    const gristConfig: GristLoadConfig = (window as any).gristConfig;
+    return (gristConfig && gristConfig.pathOnly) || false;
+  } else {
+    if (host && host.match(/^localhost(:[0-9]+)?$/)) { return true; }
+    return (process.env.GRIST_ORG_IN_PATH === 'true');
+  }
+}
+
+// Extract an organization name from the host.  Returns null if an organization name
+// could not be recovered.  Organization name may be overridden by server configuration.
+export function getOrgFromHost(reqHost: string): string|null {
+  const singleOrg = getKnownOrg();
+  if (singleOrg) { return singleOrg; }
+  if (isOrgInPathOnly()) { return null; }
+  return parseSubdomain(reqHost).org || null;
+}
+
+/**
+ * Get any information about an organization that is embedded in the host name or the
+ * path.
+ * For example, on nasa.getgrist.com, orgFromHost and subdomain will be set to "nasa".
+ * On localhost:8000/o/nasa, orgFromPath and subdomain will be set to "nasa".
+ * On nasa.getgrist.com/o/nasa, orgFromHost, orgFromPath, and subdomain will all be "nasa".
+ * On spam.getgrist.com/o/nasa, orgFromHost will be "spam", orgFromPath will be "nasa",
+ * subdomain will be null, and mismatch will be true.
+ */
+export function extractOrgParts(reqHost: string|undefined, reqPath: string): OrgParts {
+  let orgFromHost: string|null = getKnownOrg();
+
+  if (!orgFromHost && reqHost) {
+    orgFromHost = getOrgFromHost(reqHost);
+    if (orgFromHost) {
+      // Some subdomains are shared, and do not reflect the name of an organization.
+      // See https://phab.getgrist.com/w/hosting/v1/urls/ for a list.
+      if (/^(api|v1-.*|doc-worker-.*)$/.test(orgFromHost)) {
+        orgFromHost = null;
+      }
+    }
+  }
+
+  const part = parseFirstUrlPart('o', reqPath);
+  if (part.value) {
+    const orgFromPath = part.value.toLowerCase();
+    const mismatch = Boolean(orgFromHost && orgFromPath && (orgFromHost !== orgFromPath));
+    const subdomain = mismatch ? null : orgFromPath;
+    return {orgFromHost, orgFromPath, pathRemainder: part.path, mismatch, subdomain};
+  }
+  return {orgFromHost, orgFromPath: null, pathRemainder: reqPath, mismatch: false, subdomain: orgFromHost};
+}
+
+/**
+ * When a prefix is extracted from the path, the remainder of the path may be empty.
+ * This method makes sure there is at least a "/".
+ */
+export function sanitizePathTail(path: string|undefined) {
+  path = path || '/';
+  return (path.startsWith('/') ? '' : '/') + path;
+}
+
+/*
+ * If path starts with /{tag}/{value}{/rest}, returns value and the remaining path (/rest).
+ * Otherwise, returns value of undefined and the path unchanged.
+ * E.g. parseFirstUrlPart('o', '/o/foo/bar') returns {value: 'foo', path: '/bar'}.
+ */
+export function parseFirstUrlPart(tag: string, path: string): {value?: string, path: string} {
+  const match = path.match(/^\/([^/?#]+)\/([^/?#]+)(.*)$/);
+  if (match && match[1] === tag) {
+    return {value: match[2], path: sanitizePathTail(match[3])};
+  } else {
+    return {path};
+  }
+}
+
+/**
+ * The internal structure of a UrlId.  There is no internal structure. unless the id is
+ * for a fork, in which case the fork has a separate id, and a user id may also be
+ * embedded to track ownership.
+ */
+export interface UrlIdParts {
+  trunkId: string;
+  forkId?: string;
+  forkUserId?: number;
+  snapshotId?: string;
+}
+
+// Parse a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId
+// or trunkId[....]~v=snapshotId
+export function parseUrlId(urlId: string): UrlIdParts {
+  let snapshotId: string|undefined;
+  const parts = urlId.split('~');
+  const bareParts = parts.filter(part => !part.includes('='));
+  for (const part of parts) {
+    if (part.startsWith('v=')) {
+      snapshotId = decodeURIComponent(part.substr(2).replace(/_/g, '%'));
+    }
+  }
+  return {
+    trunkId: bareParts[0],
+    forkId: bareParts[1],
+    forkUserId: (bareParts[2] !== undefined) ? parseInt(bareParts[2], 10) : undefined,
+    snapshotId,
+  };
+}
+
+// Construct a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId
+// or trunkId[....]~v=snapshotId
+export function buildUrlId(parts: UrlIdParts): string {
+  let token = [parts.trunkId, parts.forkId, parts.forkUserId].filter(x => x !== undefined).join('~');
+  if (parts.snapshotId) {
+    // This could be an S3 VersionId, about which AWS makes few promises.
+    // encodeURIComponent leaves untouched the following:
+    //   alphabetic; decimal; any of: - _ . ! ~ * ' ( )
+    // We further encode _.!~*'() to fit within existing limits on what characters
+    // may be in a docId (leaving just the hyphen, which is permitted).  The limits
+    // could be loosened, but without much benefit.
+    const codedSnapshotId = encodeURIComponent(parts.snapshotId)
+      .replace(/[_.!~*'()]/g, ch => `_${ch.charCodeAt(0).toString(16).toUpperCase()}`)
+      .replace(/%/g, '_');
+    token = `${token}~v=${codedSnapshotId}`;
+  }
+  return token;
+}
+
+/**
+ * Values that may be encoded in a hash in a document url.
+ */
+export interface HashLink {
+  sectionId?: number;
+  rowId?: number;
+  colRef?: number;
+}
+
+// Check whether a urlId is a prefix of the docId, and adequately long to be
+// a candidate for use in prettier urls.
+function shouldIncludeSlug(doc: {id: string, urlId: string|null}): boolean {
+  if (!doc.urlId || doc.urlId.length < MIN_URLID_PREFIX_LENGTH) { return false; }
+  return doc.id.startsWith(doc.urlId);
+}
+
+// Convert the name of a document into a slug.  Only alphanumerics are retained,
+// and spaces are replaced with hyphens.
+// TODO: investigate whether there's a better option with unicode than just
+// deleting it, seems unfair to languages using anything other than unaccented
+// Latin characters.
+function nameToSlug(name: string): string {
+  return name.trim().replace(/ /g, '-').replace(/[^-a-zA-Z0-9]/g, '').replace(/---*/g, '-');
+}
+
+// Returns a slug for the given docId/urlId/name, or undefined if a slug should
+// not be used.
+export function getSlugIfNeeded(doc: {id: string, urlId: string|null, name: string}): string|undefined {
+  if (!shouldIncludeSlug(doc)) { return; }
+  return nameToSlug(doc.name);
+}

app/common/gutil.ts

@@ -0,0 +1,782 @@
+import {delay} from 'app/common/delay';
+import {Listener, Observable} from 'grainjs';
+import {Observable as KoObservable} from 'knockout';
+import constant = require('lodash/constant');
+import identity = require('lodash/identity');
+import times = require('lodash/times');
+
+export const UP_TRIANGLE = '\u25B2';
+export const DOWN_TRIANGLE = '\u25BC';
+
+const EMAIL_RE = new RegExp("^\\w[\\w%+/='-]*(\\.[\\w%+/='-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z" +
+  "0-9])?\\.)+[A-Za-z]{2,6}$", "u");
+
+// Returns whether str starts with prefix. (Note that this implementation avoids creating a new
+// string, and only checks a single location.)
+export function startsWith(str: string, prefix: string): boolean {
+  return str.lastIndexOf(prefix, 0) === 0;
+}
+
+// Returns whether str ends with suffix.
+export function endsWith(str: string, suffix: string): boolean {
+  return str.indexOf(suffix, str.length - suffix.length) !== -1;
+}
+
+// If str starts with prefix, removes it and returns what remains. Otherwise, returns null.
+export function removePrefix(str: string, prefix: string): string|null {
+  return startsWith(str, prefix) ? str.slice(prefix.length) : null;
+}
+
+
+// If str ends with suffix, removes it and returns what remains. Otherwise, returns null.
+export function removeSuffix(str: string, suffix: string): string|null {
+  return endsWith(str, suffix) ? str.slice(0, str.length - suffix.length) : null;
+}
+
+export function removeTrailingSlash(str: string): string {
+  const result = removeSuffix(str, '/');
+  return result === null ? str : result;
+}
+
+// Expose <string>.padStart.  The version of node we use has it, but they typings
+// need the es2017 typescript target.  TODO: replace once typings in place.
+export function padStart(str: string, targetLength: number, padString: string) {
+  return (str as any).padStart(targetLength, padString);
+}
+
+// Capitalizes every word in a string.
+export function capitalize(str: string): string {
+  return str.replace(/\b[a-z]/gi, c => c.toUpperCase());
+}
+
+// Returns whether the string n represents a valid number.
+// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric
+export function isNumber(n: string): boolean {
+  // This wasn't right for a long time: isFinite() is key to failing on strings like "5a".
+  return !isNaN(parseFloat(n)) && isFinite(n as any);
+}
+
+/**
+ * Returns a value clamped to the given min-max range.
+ * @param {Number} value - some numeric value.
+ * @param {Number} min - minimum value allowed.
+ * @param {Number} max - maximum value allowed. Must have min <= max.
+ * @returns {Number} - value restricted to the given range.
+ */
+export function clamp(value: number, min: number, max: number): number {
+  return Math.max(min, Math.min(max, value));
+}
+
+/**
+ * Checks if ele is contained within the given bounds.
+ * @param {Number} value
+ * @param {Number} bound1 - does not have to be less than/eqal to bound2
+ * @param {Number} bound2
+ * @returns {Boolean} - True/False
+ */
+export function between(value: number, bound1: number, bound2: number): boolean {
+  const lower = Math.min(bound1, bound2);
+  const upper = Math.max(bound1, bound2);
+  return lower <= value && value <= upper;
+}
+
+/**
+ * Returns the positive modulo of x by n. (Javascript default allows negatives)
+ */
+export function mod(x: number, n: number): number {
+  return ((x % n) + n) % n;
+}
+
+/**
+ * Returns a number that is n rounded down to the next nearest number divisible by m
+ */
+
+export function roundDownToMultiple(n: number, m: number): number {
+  return Math.floor(n / m) * m;
+}
+
+/**
+ * Returns the first argument unless it's undefined, in which case returns the second one.
+ */
+export function undefDefault<T>(x: T|undefined, y: T): T {
+  return (x !== void 0) ? x : y;
+}
+
+/**
+ * Parses json and returns the result, or returns defaultVal if parsing fails.
+ */
+export function safeJsonParse(json: string, defaultVal: any): any {
+  try {
+    return JSON.parse(json);
+  } catch (e) {
+    return defaultVal;
+  }
+}
+
+/**
+ * Just like encodeURIComponent, but does not encode slashes. Slashes don't hurt to be included in
+ * URL parameters, and look much friendlier not encoded.
+ */
+export function encodeQueryParam(str: string|number|undefined): string {
+  return encodeURIComponent(String(str === undefined ? null : str)).replace(/%2F/g, '/');
+}
+
+/**
+ * Encode an object into a querystring ("key=value&key2=value2").
+ * This is similar to JQuery's $.param, but only works on shallow objects.
+ */
+export function encodeQueryParams(obj: {[key: string]: string|number|undefined}): string {
+  return Object.keys(obj).map((k: string) => encodeQueryParam(k) + '=' + encodeQueryParam(obj[k])).join('&');
+}
+
+/**
+ * Return a list of the words in the string, using the given separator string. At most
+ * maxNumSplits splits are done, so the result will have at most maxNumSplits + 1 elements (this
+ * is the main difference from how JS built-in string.split() works, and similar to Python split).
+ * @param {String} str: String to split.
+ * @param {String} sep: Separator to split on.
+ * @param {Number} maxNumSplits: Maximum number of splits to do.
+ * @return {Array[String]} Array of words, of length at most maxNumSplits + 1.
+ */
+export function maxsplit(str: string, sep: string, maxNumSplits: number): string[] {
+  const result: string[] = [];
+  let start = 0, pos;
+  for (let i = 0; i < maxNumSplits; i++) {
+    pos = str.indexOf(sep, start);
+    if (pos === -1) {
+      break;
+    }
+    result.push(str.slice(start, pos));
+    start = pos + sep.length;
+  }
+  result.push(str.slice(start));
+  return result;
+}
+
+
+// Compare arrays of scalars for equality.
+export function arraysEqual(a: any[], b: any[]): boolean {
+  if (a === b) {
+    return true;
+  }
+  if (!a || !b) {
+    return false;
+  }
+  if (a.length !== b.length) {
+    return false;
+  }
+
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) { return false; }
+  }
+  return true;
+}
+
+
+// Gives a set representing the set difference a - b.
+export function setDifference<T>(a: Set<T>, b: Set<T>): Set<T> {
+  const c = new Set<T>();
+  for (const ai of a) {
+    if (!b.has(ai)) { c.add(ai); }
+  }
+  return c;
+}
+
+// Like array.indexOf, but works with array-like objects like HTMLCollection.
+export function indexOf<T>(arrayLike: ArrayLike<T>, item: T): number {
+  return Array.prototype.indexOf.call(arrayLike, item);
+}
+
+
+/**
+ * Removes a value from the given array. Only the first instance is removed.
+ * Returns true on success, false if the value was not found.
+ */
+export function arrayRemove<T>(array: T[], value: T): boolean {
+  const index = array.indexOf(value);
+  if (index === -1) {
+    return false;
+  }
+  array.splice(index, 1);
+  return true;
+}
+
+
+/**
+ * Inserts value into the array before nextValue, or at the end if nextValue is not found.
+ */
+export function arrayInsertBefore<T>(array: T[], value: T, nextValue: T): void {
+  const index = array.indexOf(nextValue);
+  if (index === -1) {
+    array.push(value);
+  } else {
+    array.splice(index, 0, value);
+  }
+}
+
+
+/**
+ * Extends the first array with the second. Like native push, but adds all values in anotherArray.
+ */
+export function arrayExtend<T>(array: T[], anotherArray: T[]): void {
+  for (let i = 0, len = anotherArray.length; i < len; i++) {
+    array.push(anotherArray[i]);
+  }
+}
+
+
+/**
+ * Copies count items from fromArray to toArray, copying in a forward direction (which matters
+ * when the arrays are the same and source and destination indices overlap).
+ *
+ * See test/common/arraySplice.js for alternative implementations with timings, from which this
+ * one is chosen as consistently among the faster ones.
+ */
+export function arrayCopyForward<T>(toArray: T[], toStart: number,
+                                    fromArray: ArrayLike<T>, fromStart: number, count: number): void {
+  const end = toStart + count;
+  for (const xend = end - 7; toStart < xend; fromStart += 8, toStart += 8) {
+    toArray[toStart] = fromArray[fromStart];
+    toArray[toStart + 1] = fromArray[fromStart + 1];
+    toArray[toStart + 2] = fromArray[fromStart + 2];
+    toArray[toStart + 3] = fromArray[fromStart + 3];
+    toArray[toStart + 4] = fromArray[fromStart + 4];
+    toArray[toStart + 5] = fromArray[fromStart + 5];
+    toArray[toStart + 6] = fromArray[fromStart + 6];
+    toArray[toStart + 7] = fromArray[fromStart + 7];
+  }
+  for (; toStart < end; ++fromStart, ++toStart) {
+    toArray[toStart] = fromArray[fromStart];
+  }
+}
+
+
+/**
+ * Copies count items from fromArray to toArray, copying in a backward direction (which matters
+ * when the arrays are the same and source and destination indices overlap).
+ *
+ * See test/common/arraySplice.js for alternative implementations with timings, from which this
+ * one is chosen as consistently among the faster ones.
+ */
+export function arrayCopyBackward<T>(toArray: T[], toStart: number,
+                                     fromArray: ArrayLike<T>, fromStart: number, count: number): void {
+  let i = toStart + count - 1, j = fromStart + count - 1;
+  for (const xStart = toStart + 7; i >= xStart; i -= 8, j -= 8) {
+    toArray[i] = fromArray[j];
+    toArray[i - 1] = fromArray[j - 1];
+    toArray[i - 2] = fromArray[j - 2];
+    toArray[i - 3] = fromArray[j - 3];
+    toArray[i - 4] = fromArray[j - 4];
+    toArray[i - 5] = fromArray[j - 5];
+    toArray[i - 6] = fromArray[j - 6];
+    toArray[i - 7] = fromArray[j - 7];
+  }
+  for ( ; i >= toStart; --i, --j) {
+    toArray[i] = fromArray[j];
+  }
+}
+
+
+/**
+ * Appends a slice of fromArray to the end of toArray.
+ *
+ * See test/common/arraySplice.js for alternative implementations with timings, from which this
+ * one is chosen as consistently among the faster ones.
+ */
+export function arrayAppend<T>(toArray: T[], fromArray: ArrayLike<T>, fromStart: number, count: number): void {
+  if (count === 1) {
+    toArray.push(fromArray[fromStart]);
+  } else {
+    const len = toArray.length;
+    toArray.length = len + count;
+    arrayCopyForward(toArray, len, fromArray, fromStart, count);
+  }
+}
+
+
+/**
+ * Splices array arrToInsert into target starting at the given start index.
+ * This implementation tries to be smart by avoiding allocations, appending to the array
+ * contiguously, then filling in the gap.
+ *
+ * See test/common/arraySplice.js for alternative implementations with timings, from which this
+ * one is chosen as consistently among the faster ones.
+ */
+export function arraySplice<T>(target: T[], start: number, arrToInsert: ArrayLike<T>): T[] {
+  const origLen = target.length;
+  const tailLen = origLen - start;
+  const insLen = arrToInsert.length;
+  target.length = origLen + insLen;
+  if (insLen > tailLen) {
+    arrayCopyForward(target, origLen, arrToInsert, tailLen, insLen - tailLen);
+    arrayCopyForward(target, start + insLen, target, start, tailLen);
+    arrayCopyForward(target, start, arrToInsert, 0, tailLen);
+  } else {
+    arrayCopyForward(target, origLen, target, origLen - insLen, insLen);
+    arrayCopyBackward(target, start + insLen, target, start, tailLen - insLen);
+    arrayCopyForward(target, start, arrToInsert, 0, insLen);
+  }
+  return target;
+}
+
+
+/**
+ * Returns a new array of length count, filled with the given value.
+ */
+export function arrayRepeat<T>(count: number, value: T): T[] {
+  return times(count, constant(value));
+}
+
+// Type for a compare func that returns a positive, negative, or zero value, as used for sorting.
+export type CompareFunc<T> = (a: T, b: T) => number;
+
+/**
+ * Returns the index at which the given element can be inserted to keep the array sorted.
+ * This is equivalent to underscore's sortedIndex and python's bisect_left.
+ * @param {Array} array - sorted array of elements based on the given compareFunc
+ * @param {object} elem - object to be inserted in the given array
+ * @param {function} compareFunc - compares 2 elements. Returns a pos value if the 1st element is
+ *                                 larger, 0 if they're equal, a neg value if the 2nd is larger.
+ */
+export function sortedIndex<T>(array: ArrayLike<T>, elem: T, compareFunc: CompareFunc<T>): number {
+  let lo = 0, mid;
+  let hi = array.length;
+
+  if (array.length === 0) { return 0; }
+  while (lo < hi) {
+    mid = Math.floor((lo + hi) / 2);
+    if (compareFunc(array[mid], elem) < 0) { // mid < elem
+      lo = mid + 1;
+    } else {
+      hi = mid;
+    }
+  }
+  return lo;
+}
+
+/**
+ * Returns true if an array contains duplicate values.
+ * Values are considered equal if their toString() representations are equal.
+ */
+export function hasDuplicates(array: any[]): boolean {
+  const prevVals = Object.create(null);
+  for (const value of array) {
+    if (value in prevVals) {
+      return true;
+    }
+    prevVals[value] = true;
+  }
+  return false;
+}
+
+/**
+ * Counts the number of items in array which satisfy the callback.
+ */
+export function countIf<T>(array: ReadonlyArray<T>, callback: (item: T) => boolean): number {
+  let count = 0;
+  array.forEach(item => {
+    if (callback(item)) { count++; }
+  });
+  return count;
+}
+
+
+/**
+ * For two parallel arrays, calls mapFunc(a[i], b[i]) for each pair of corresponding elements, and
+ * returns an array of the results.
+ */
+export function map2<T, U, V>(array1: ArrayLike<T>, array2: ArrayLike<U>, mapFunc: (a: T, b: U) => V): V[] {
+  const len = array1.length;
+  const result: V[] = new Array(len);
+  for (let i = 0; i < len; i++) {
+    result[i] = mapFunc(array1[i], array2[i]);
+  }
+  return result;
+}
+
+/**
+ * Takes a 2d array returns a new matrix with r rows and c columns
+ * @param [Array] dataMatrix: a 2d array
+ * @param [Number] r: final row length
+ * @param [Number] c: final column length
+ */
+export function growMatrix<T>(dataMatrix: T[][], r: number, c: number): T[][] {
+  const colArr = dataMatrix.map(colVals =>
+    Array.from({length: c}, (_v, k) => colVals[k % colVals.length])
+  );
+  return Array.from({length: r}, (_v, k) => colArr[k % colArr.length]);
+}
+
+/**
+ * Returns a function that compares two elements based on multiple sort keys and the
+ * given compare functions.
+ * Elements are compared using the sort key functions with index 0 having the greatest priority.
+ * Subsequent sort key functions are used as tie breakers.
+ * @param {function Array} sortKeyFuncs - a list of sort key functions.
+ * @param {function Array} compareKeyFuncs - a list of comparison functions parallel to sortKeyFuncs
+ * Each compare function  must satisfy the comparison invariant:
+ *   If compare(a, b) > 0 then a > b,
+ *   If compare(a, b) < 0 then a < b,
+ *   If compare(a, b) == 0 then a == b,
+ * @param {Array of 1/-1's} optAscending - Comparison on sortKeyFuncs[i] is inverted if optAscending[i] == -1
+ */
+export function multiCompareFunc<T, U>(sortKeyFuncs: ReadonlyArray<(a: T) => U>,
+                                       compareFuncs: ArrayLike<CompareFunc<U>>,
+                                       optAscending?: number[]): CompareFunc<T> {
+  if (sortKeyFuncs.length !== compareFuncs.length) {
+    throw new Error('Number of sort key funcs must be the same as the number of compare funcs');
+  }
+  const ascending = optAscending || sortKeyFuncs.map(() => 1);
+  return function(a: T, b: T): number {
+    let compareOutcome, keyA, keyB;
+    for (let i = 0; i < compareFuncs.length; i++) {
+      keyA = sortKeyFuncs[i](a);
+      keyB = sortKeyFuncs[i](b);
+      compareOutcome = compareFuncs[i](keyA, keyB);
+      if (compareOutcome !== 0) { return ascending[i] * compareOutcome; }
+    }
+    return 0;
+  };
+}
+
+
+export function nativeCompare<T>(a: T, b: T): number {
+  return (a < b ? -1 : (a > b ? 1 : 0));
+}
+
+/**
+ * A copy of python`s `setdefault` function.
+ * Sets key in mapInst to value, if key is not already set.
+ * @param {Map} mapInst: Instance of Map.
+ * @param {Object} key: Key into the map.
+ * @param {Object} value: Value to insert, possibly.
+ */
+export function setDefault<K, V>(mapInst: Map<K, V>, key: K, val: V): V {
+  if (!mapInst.has(key)) { mapInst.set(key, val); }
+  return mapInst.get(key)!;
+}
+
+
+/**
+ * Similar to Python's `setdefault`: returns the key `key` from `mapInst`, or if it's not there, sets
+ * it to the result buildValue().
+ */
+export function getSetMapValue<K, V>(mapInst: Map<K, V>, key: K, buildValue: () => V): V {
+  if (!mapInst.has(key)) { mapInst.set(key, buildValue()); }
+  return mapInst.get(key)!;
+}
+
+
+/**
+ * If key is in mapInst, remove it and return its value, else return `undefined`.
+ * @param {Map} mapInst: Instance of Map.
+ * @param {Object} key: Key into the map to remove.
+ */
+export function popFromMap<K, V>(mapInst: Map<K, V>, key: K): V|undefined {
+  const value = mapInst.get(key);
+  mapInst.delete(key);
+  return value;
+}
+
+/**
+ * For each encountered value in `values`, increment the corresponding counter in `valueCounts`.
+ */
+export function addCountsToMap<T>(valueCounts: Map<T, number>, values: Iterable<T>,
+                                  mapFunc: (v: any) => any = identity) {
+  for (const v of values) {
+    const mappedValue = mapFunc(v);
+    valueCounts.set(mappedValue, (valueCounts.get(mappedValue) || 0) + 1);
+  }
+}
+
+/**
+ * Returns whether one Set is a subset of another.
+ */
+export function isSubset(smaller: Set<any>, larger: Set<any>): boolean {
+  for (const value of smaller) {
+    if (!larger.has(value)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+
+/**
+ * Merges the contents of two or more objects together into the first object, recursing into
+ * nested objects and arrays (like jquery.extend(true, ...)).
+ * @param {Object} target - The object to modify. Use {} to create a new merged object.
+ * @param {Object} ... - Additional objects from which to copy properties into target.
+ * @returns {Object} The first argument, target, modified.
+ */
+export function deepExtend(target: any, _varArgObjects: any): any {
+  for (let i = 1; i < arguments.length; i++) {
+    const object = arguments[i];
+    // Extend the base object
+    for (const name in object) {
+      if (!object.hasOwnProperty(name)) { continue; }
+      let src = object[name];
+      if (src === target || src === undefined) {
+        // Prevent one kind of infinite loop, as JQuery's extend does, and skip undefined values.
+        continue;
+      }
+
+      if (src) {
+        // Recurse if we're merging plain objects or arrays
+        const tgt = target[name];
+        if (Array.isArray(src)) {
+          src = deepExtend(tgt && Array.isArray(tgt) ? tgt : [], src);
+        } else if (typeof src === 'object') {
+          src = deepExtend(tgt && typeof tgt === 'object' ? tgt : {}, src);
+        }
+      }
+      target[name] = src;
+    }
+  }
+  // Return the modified object
+  return target;
+}
+
+
+/**
+ * Returns a human-readable string containing a number of bytes, KB, or MB.
+ * @param {Number} bytes. Number of bytes.
+ * @returns {String} A description such as "4.1KB".
+ */
+export function byteString(bytes: number): string {
+  if (bytes < 1024) {
+    return bytes + 'B';
+  } else if (bytes < 1024 * 1024) {
+    return (bytes / 1024).toFixed(1) + 'KB';
+  } else {
+    return (bytes / 1024 / 1024).toFixed(1) + 'MB';
+  }
+}
+
+
+/**
+ * Creates a new object mapping each key in keysArray to the value returned by callback.
+ * @param {Array} keysArray - Array of strings to use as the properties of the returned object.
+ * @param {Function} callback - Function that produces the value for each key. Called in the same
+ *    way as array.map() calls its callbacks.
+ * @param {Object} optThisArg - Value to use as `this` when executing callback.
+ * @returns {Object} - object mapping keys from `keysArray` to values returned by `callback`.
+ */
+export function mapToObject<T>(keysArray: string[], callback: (key: string) => T,
+                               optThisArg: any): {[key: string]: T} {
+  const values: T[] = keysArray.map(callback, optThisArg);
+  const map: {[key: string]: T} = {};
+  for (let i = 0; i < keysArray.length; i++) {
+    map[keysArray[i]] = values[i];
+  }
+  return map;
+}
+
+/**
+ * A List of python identifiers; the result of running keywords.kwlist in Python 2.7.6,
+ * plus additional illegal identifiers None, False, True
+ * Using [] instead of new Array causes a "comprehension error" for some reason
+ */
+const _kwlist = ['and', 'as', 'assert', 'break', 'class', 'continue', 'def',
+             'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global',
+             'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise',
+             'return', 'try', 'while', 'with', 'yield', 'None', 'False', 'True'];
+/**
+ * Given an arbitrary string, makes substitutions to make it a valid SQL/Python identifier.
+ * Corresponds to sandbox/grist/gencode.sanitize_ident
+ */
+export function sanitizeIdent(ident: string, prefix?: string) {
+  prefix = prefix || 'c';
+  // Remove non-alphanumeric non-_ chars
+  ident = ident.replace(/[^a-zA-Z0-9_]+/g, '_');
+  // Remove leading and trailing _
+  ident = ident.replace(/^_+|_+$/g, '');
+  // Place prefix at front if the beginning isn't a number
+  ident = ident.replace(/^(?=[0-9])/g, prefix);
+  // Append prefix until it is not  python keyword
+  while (_kwlist.includes(ident)) {
+    ident = prefix + ident;
+  }
+  return ident;
+}
+
+
+/**
+ * Clone a function, returning a function object that represents a brand new function with the
+ * same code. If the same function is used with different argument types, it would prevent JS V8
+ * engine optimizations (or cause it to deoptimize it). If different clones are called with
+ * different argument types, they can be optimized independently.
+ *
+ * As with all micro-optimizations, only do this when the optimization matters.
+ */
+export function cloneFunc(fn: Function): Function {     // tslint:disable-line:ban-types
+  /* jshint evil:true */  // suppress eval warning.
+  return eval('(' + fn.toString() + ')');   // tslint:disable-line:no-eval
+}
+
+
+/**
+ * Generates a random id using a sequence of uppercase alphanumeric characters
+ * preceeded by an optional prefix.
+ */
+export function genRandomId(len: number, optPrefix?: string): string {
+  const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
+  let ret = optPrefix || '';
+  for (let i = 0; i < len; i++) {
+    ret += chars[Math.floor(Math.random() * chars.length)];
+  }
+  return ret;
+}
+
+/**
+ * Scans through two sorted arrays, calling a function on each item or pair of items
+ * for every present key in order.
+ * @param {Array} arrA - First array to scan. NOTE: Should be sorted by the key value.
+ * @param {Array} arrB - Second array to scan. NOTE: Should be sorted by the key value.
+ * @param {Function} callback - Called with an item from arrA as the first argument and an
+ *  item from arrB as the second. Called for every unique key in order, either with one of the
+ *  arguments null if the key is present only in one array, or both non-null if the key is
+ *  present in both arrays. NOTE: Key values should not be null.
+ * @param {Function} optKeyFunc - Optional function to map each array value to a sort key.
+ *  Defaults to the identity function.
+ */
+export function sortedScan<T, U>(arrA: ArrayLike<T>, arrB: ArrayLike<U>,
+                                 callback: (a: T|null, B: U|null) => void,
+                                 optKeyFunc?: (item: T|U) => any) {
+  const keyFunc = optKeyFunc || identity;
+  let i = 0, j = 0;
+  while (i < arrA.length || j < arrB.length) {
+    const a = arrA[i], b = arrB[j];
+    const keyA = i < arrA.length ? keyFunc(a) : null;
+    const keyB = j < arrB.length ? keyFunc(b) : null;
+    if (keyA !== null && (keyB === null || keyA < keyB)) {
+      callback(a, null);
+      i++;
+    } else if (keyA === null || keyA > keyB) {
+      callback(null, b);
+      j++;
+    } else {
+      callback(a, b);
+      i++;
+      j++;
+    }
+  }
+}
+
+/**
+ * Returns the time in ms to wait until attempting another connection.
+ * @param {Number} attemptNumber - Reconnect attempt number starting at 0.
+ * @param {Array} intervals - Array of reconnect intervals in ms.
+ * @returns {Number}
+ */
+export function getReconnectTimeout(attemptNumber: number, intervals: ArrayLike<number>): number {
+  if (attemptNumber >= intervals.length) {
+    // Add an additional wait time if already at max attempts.
+    const timeout = intervals[intervals.length - 1];
+    return timeout + Math.random() * timeout;
+  } else {
+    return intervals[attemptNumber];
+  }
+}
+
+/**
+ * Returns whether the given email is a valid formatted email string.
+ * @param {String} email - Email to test.
+ * @returns {Boolean}
+ */
+export function isEmail(email: string): boolean {
+  return EMAIL_RE.test(email.toLowerCase());
+}
+
+/*
+ * Takes an observable and returns a promise for when the observable's value matches the given
+ * predicate. It then unsubscribes from the observable, and returns its value.
+ * If a predicate is not given, resolves to the observable values as soon as it's truthy.
+ */
+export function waitObs<T>(observable: KoObservable<T>, predicate: (value: T) => boolean = Boolean): Promise<T> {
+  return new Promise((resolve, _reject) => {
+    const value = observable.peek();
+    if (predicate(value)) { return resolve(value); }
+    const sub = observable.subscribe((val: T) => {
+      if (predicate(val)) {
+        sub.dispose();
+        resolve(val);
+      }
+    });
+  });
+}
+
+/**
+ * Same as waitObs but for grainjs observables.
+ */
+export async function waitGrainObs<T>(observable: Observable<T>): Promise<NonNullable<T>>;
+export async function waitGrainObs<T>(observable: Observable<T>, predicate?: (value: T) => boolean): Promise<T>;
+export async function waitGrainObs<T>(observable: Observable<T>,
+                                      predicate: (value: T) => boolean = Boolean): Promise<T> {
+  let sub: Listener|undefined;
+  const res: T = await new Promise((resolve, _reject) => {
+    const value = observable.get();
+    if (predicate(value)) { return resolve(value); }
+    sub = observable.addListener((val: T) => {
+      if (predicate(val)) {
+        resolve(val);
+      }
+    });
+  });
+  if (sub) { sub.dispose(); }
+  return res;
+}
+
+/**
+ * Class to maintain a chain of promise-returning callbacks. All scheduled callbacks will be
+ * called in order as long as the previous one is successful. If a callback fails is rejected,
+ * already-scheduled callbacks will be skipped, but newly-scheduled ones will be run.
+ */
+export class PromiseChain<T> {
+  private _last: Promise<T|void> = Promise.resolve();
+
+  // Adds a callback to the chain. If the callback runs, the return value is the return value of
+  // the callback. If it's skipped due to a failure earlier in the chain, the return value is the
+  // rejection with the message "Skipped due to an earlier error".
+  public add(nextCB: () => Promise<T>): Promise<T> {
+    const next = this._last.catch(() => { throw new Error("Skipped due to an earlier error"); }).then(nextCB);
+    // If any callback fails, all queued ones will be skipped. Here we reset the chain, so that
+    // callbacks added later do get run.
+    next.catch(() => { this._last = Promise.resolve(); });
+    this._last = next;
+    return next;
+  }
+}
+
+/**
+ * Indicates if a hex color value, e.g. '#000000', is darker than the given value.
+ * Darkness is measured from 0..255, where 0 is the darkest and 255 is the lightest.
+ *
+ * Taken from: https://stackoverflow.com/questions/12043187/how-to-check-if-hex-color-is-too-black
+ */
+export function isColorDark(hexColor: string, isDarkBelow: number = 220): boolean {
+  const c = hexColor.substring(1);  // strip #
+  const rgb = parseInt(c, 16);      // convert rrggbb to decimal
+  // Extract RGB components
+  const r = (rgb >> 16) & 0xff;     // tslint:disable-line:no-bitwise
+  const g = (rgb >>  8) & 0xff;     // tslint:disable-line:no-bitwise
+  const b = (rgb >>  0) & 0xff;     // tslint:disable-line:no-bitwise
+
+  const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b;  // per ITU-R BT.709
+  return luma < isDarkBelow;
+}
+
+
+/**
+ * Returns a promise that resolves to true if promise takes longer than timeoutMsec to resolve. If not
+ * or if promise throws returns false.
+ */
+export async function isLongerThan(promise: Promise<any>, timeoutMsec: number): Promise<boolean> {
+  let isPending = true;
+  const done = () => {isPending = false; };
+  await Promise.race([
+    promise.then(done, done),
+    delay(timeoutMsec)
+  ]);
+  return isPending;
+}

app/common/marshal.ts

@@ -0,0 +1,502 @@
+/**
+ * Module for serializing data in the format of Python 'marshal' module. It's used for
+ * communicating with the Python-based formula engine running in a Pypy sandbox. It supports
+ * version 0 of python marshalling format, which is what the Pypy sandbox supports.
+ *
+ * Usage:
+ *    Marshalling:
+ *      const marshaller = new Marshaller({version: 2});
+ *      marshaller.marshal(value);
+ *      marshaller.marshal(value);
+ *      const buf = marshaller.dump();    // Leaves the marshaller empty.
+ *
+ *    Unmarshalling:
+ *      const unmarshaller = new Unmarshaller();
+ *      unmarshaller.on('value', function(value) { ... });
+ *      unmarshaller.push(buffer);
+ *      unmarshaller.push(buffer);
+ *
+ * In Python, and in the marshalled format, there is a distinction between strings and unicode
+ * objects. In JS, there is a good correspondence to Uint8Array objects and strings, respectively.
+ * Python unicode objects always become JS strings. JS Uint8Arrays always become Python strings.
+ *
+ * JS strings become Python unicode objects, but can be marshalled to Python strings with
+ * 'stringToBuffer' option. Similarly, Python strings become JS Uint8Arrays, but can be
+ * unmarshalled to JS strings if 'bufferToString' option is set.
+ */
+import {BigInt} from 'app/common/BigInt';
+import * as MemBuffer from 'app/common/MemBuffer';
+import {EventEmitter} from 'events';
+import * as util from 'util';
+
+
+export interface MarshalOptions {
+  stringToBuffer?: boolean;
+  version?: number;
+}
+
+export interface UnmarshalOptions {
+  bufferToString?: boolean;
+}
+
+function ord(str: string): number {
+  return str.charCodeAt(0);
+}
+
+/**
+ * Type codes used for python marshalling of values.
+ * See pypy: rpython/translator/sandbox/_marshal.py.
+ */
+const marshalCodes = {
+  NULL     : ord('0'),
+  NONE     : ord('N'),
+  FALSE    : ord('F'),
+  TRUE     : ord('T'),
+  STOPITER : ord('S'),
+  ELLIPSIS : ord('.'),
+  INT      : ord('i'),
+  INT64    : ord('I'),
+  /*
+    BFLOAT, for 'binary float', is an encoding of float that just encodes the bytes of the
+    double in standard IEEE 754 float64 format. It is used by Version 2+ of Python's marshal
+    module. Previously (in versions 0 and 1), the FLOAT encoding is used, which stores floats
+    through their string representations.
+
+    Version 0 (FLOAT) is mandatory for system calls within the sandbox, while Version 2 (BFLOAT)
+    is recommended for Grist's communication because it is more efficient and faster to
+    encode/decode
+   */
+  BFLOAT   : ord('g'),
+  FLOAT    : ord('f'),
+  COMPLEX  : ord('x'),
+  LONG     : ord('l'),
+  STRING   : ord('s'),
+  INTERNED : ord('t'),
+  STRINGREF: ord('R'),
+  TUPLE    : ord('('),
+  LIST     : ord('['),
+  DICT     : ord('{'),
+  CODE     : ord('c'),
+  UNICODE  : ord('u'),
+  UNKNOWN  : ord('?'),
+  SET      : ord('<'),
+  FROZENSET: ord('>'),
+};
+
+type MarshalCode = keyof typeof marshalCodes;
+
+// A little hack to test if the value is a 32-bit integer. Actually, for Python, int might be up
+// to 64 bits (if that's the native size), but this is simpler.
+// See http://stackoverflow.com/questions/3885817/how-to-check-if-a-number-is-float-or-integer.
+function isInteger(n: number): boolean {
+  // Float have +0.0 and -0.0. To represent -0.0 precisely, we have to use a float, not an int
+  // (see also https://stackoverflow.com/questions/7223359/are-0-and-0-the-same).
+  // tslint:disable-next-line:no-bitwise
+  return n === +n && n === (n | 0) && !Object.is(n, -0.0);
+}
+
+// ----------------------------------------------------------------------
+
+/**
+ * To force a value to be serialized using a particular representation (e.g. a number as INT64),
+ * wrap it into marshal.wrap('INT64', value) and serialize that.
+ */
+export function wrap(codeStr: MarshalCode, value: unknown) {
+  return new WrappedObj(marshalCodes[codeStr], value);
+}
+
+export class WrappedObj {
+  constructor(public code: number, public value: unknown) {}
+
+  public inspect() {
+    return util.inspect(this.value);
+  }
+}
+
+// ----------------------------------------------------------------------
+
+/**
+ * @param {Boolean} options.stringToBuffer - If set, JS strings will become Python strings rather
+ *      than unicode objects (as if each JS string is wrapped into MemBuffer.stringToArray(str)).
+ *      This flag becomes a same-named property of Marshaller, which can be set at any time.
+ * @param {Number} options.version - If version >= 2, uses binary representation for floats. The
+ *      default version 0 formats floats as strings.
+ *
+ * TODO: The default should be version 2. (0 was used historically because it was needed for
+ * communication with PyPy-based sandbox.)
+ */
+export class Marshaller {
+  private memBuf: MemBuffer;
+  private readonly floatCode: number;
+  private readonly stringCode: number;
+
+  constructor(options?: MarshalOptions) {
+    this.memBuf = new MemBuffer(undefined);
+    this.floatCode = options && options.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT;
+    this.stringCode = options && options.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE;
+  }
+
+  public dump(): Uint8Array {
+    // asByteArray returns a view on the underlying data, and the constructor creates a new copy.
+    // For some usages, we may want to avoid making the copy.
+    const bytes = new Uint8Array(this.memBuf.asByteArray());
+    this.memBuf.clear();
+    return bytes;
+  }
+
+  public dumpAsBuffer(): Buffer {
+    const bytes = Buffer.from(this.memBuf.asByteArray());
+    this.memBuf.clear();
+    return bytes;
+  }
+
+  public getCode(value: any) {
+    switch (typeof value) {
+      case 'number': return isInteger(value) ? marshalCodes.INT : this.floatCode;
+      case 'string': return this.stringCode;
+      case 'boolean': return value ? marshalCodes.TRUE : marshalCodes.FALSE;
+      case 'undefined': return marshalCodes.NONE;
+      case 'object': {
+        if (value instanceof WrappedObj) {
+          return value.code;
+        } else if (value === null) {
+          return marshalCodes.NONE;
+        } else if (value instanceof Uint8Array) {
+          return marshalCodes.STRING;
+        } else if (Buffer.isBuffer(value)) {
+          return marshalCodes.STRING;
+        } else if (Array.isArray(value)) {
+          return marshalCodes.LIST;
+        }
+        return marshalCodes.DICT;
+      }
+      default: {
+        throw new Error("Marshaller: Unsupported value of type " + (typeof value));
+      }
+    }
+  }
+
+  public marshal(value: any): void {
+    const code = this.getCode(value);
+    if (value instanceof WrappedObj) {
+      value = value.value;
+    }
+    this.memBuf.writeUint8(code);
+    switch (code) {
+      case marshalCodes.NULL:       return;
+      case marshalCodes.NONE:       return;
+      case marshalCodes.FALSE:      return;
+      case marshalCodes.TRUE:       return;
+      case marshalCodes.INT:        return this.memBuf.writeInt32LE(value);
+      case marshalCodes.INT64:      return this._writeInt64(value);
+      case marshalCodes.FLOAT:      return this._writeStringFloat(value);
+      case marshalCodes.BFLOAT:     return this.memBuf.writeFloat64LE(value);
+      case marshalCodes.STRING:
+        return (value instanceof Uint8Array || Buffer.isBuffer(value) ?
+          this._writeByteArray(value) :
+          this._writeUtf8String(value));
+      case marshalCodes.TUPLE:      return this._writeList(value);
+      case marshalCodes.LIST:       return this._writeList(value);
+      case marshalCodes.DICT:       return this._writeDict(value);
+      case marshalCodes.UNICODE:    return this._writeUtf8String(value);
+      // None of the following are supported.
+      case marshalCodes.STOPITER:
+      case marshalCodes.ELLIPSIS:
+      case marshalCodes.COMPLEX:
+      case marshalCodes.LONG:
+      case marshalCodes.INTERNED:
+      case marshalCodes.STRINGREF:
+      case marshalCodes.CODE:
+      case marshalCodes.UNKNOWN:
+      case marshalCodes.SET:
+      case marshalCodes.FROZENSET:  throw new Error("Marshaller: Can't serialize code " + code);
+      default:                      throw new Error("Marshaller: Can't serialize code " + code);
+    }
+  }
+
+  private _writeInt64(value: number) {
+    if (!isInteger(value)) {
+      // TODO We could actually support 53 bits or so.
+      throw new Error("Marshaller: int64 still only supports 32-bit ints for now: " + value);
+    }
+    this.memBuf.writeInt32LE(value);
+    this.memBuf.writeInt32LE(value >= 0 ? 0 : -1);
+  }
+
+  private _writeStringFloat(value: number) {
+    // This could be optimized a bit, but it's only used in V0 marshalling, which is only used in
+    // sandbox system calls, which don't really ever use floats anyway.
+    const bytes = MemBuffer.stringToArray(value.toString());
+    if (bytes.byteLength >= 127) {
+      throw new Error("Marshaller: Trying to write a float that takes " + bytes.byteLength + " bytes");
+    }
+    this.memBuf.writeUint8(bytes.byteLength);
+    this.memBuf.writeByteArray(bytes);
+  }
+
+  private _writeByteArray(value: Uint8Array|Buffer) {
+    // This works for both Uint8Arrays and Node Buffers.
+    this.memBuf.writeInt32LE(value.length);
+    this.memBuf.writeByteArray(value);
+  }
+
+  private _writeUtf8String(value: string) {
+    const offset = this.memBuf.size();
+    // We don't know the length until we write the value.
+    this.memBuf.writeInt32LE(0);
+    this.memBuf.writeString(value);
+    const byteLength = this.memBuf.size() - offset - 4;
+    // Overwrite the 0 length we wrote earlier with the correct byte length.
+    this.memBuf.asDataView.setInt32(this.memBuf.startPos + offset, byteLength, true);
+  }
+
+  private _writeList(array: unknown[]) {
+    this.memBuf.writeInt32LE(array.length);
+    for (const item of array) {
+      this.marshal(item);
+    }
+  }
+
+  private _writeDict(obj: {[key: string]: any}) {
+    const keys = Object.keys(obj);
+    keys.sort();
+    for (const key of keys) {
+      this.marshal(key);
+      this.marshal(obj[key]);
+    }
+    this.memBuf.writeUint8(marshalCodes.NULL);
+  }
+}
+
+// ----------------------------------------------------------------------
+
+const TwoTo32 = 0x100000000;    // 2**32
+const TwoTo15 = 0x8000;         // 2**15
+
+/**
+ * @param {Boolean} options.bufferToString - If set, Python strings will become JS strings rather
+ *      than Buffers (as if each decoded buffer is wrapped into `buf.toString()`).
+ *      This flag becomes a same-named property of Unmarshaller, which can be set at any time.
+ * Note that options.version isn't needed, since this will decode both formats.
+ * TODO: Integers (such as int64 and longs) that are too large for JS are currently represented as
+ * decimal strings. They may need a better representation, or a configurable option.
+ */
+export class Unmarshaller extends EventEmitter {
+  public memBuf: MemBuffer;
+  private consumer: any = null;
+  private _lastCode: number|null = null;
+  private readonly bufferToString: boolean;
+  private emitter: (v: any) => boolean;
+  private stringTable: Array<string|Uint8Array> = [];
+
+  constructor(options?: UnmarshalOptions) {
+    super();
+    this.memBuf = new MemBuffer(undefined);
+    this.bufferToString = Boolean(options && options.bufferToString);
+    this.emitter = this.emit.bind(this, 'value');
+  }
+
+  /**
+   * Adds more data for parsing. Parsed values will be emitted as 'value' events.
+   * @param {Uint8Array|Buffer} byteArray: Uint8Array or Node Buffer with bytes to parse.
+   */
+  public push(byteArray: Uint8Array|Buffer) {
+    this.parse(byteArray, this.emitter);
+  }
+
+  /**
+   * Adds data to parse, and calls valueCB(value) for each value parsed. If valueCB returns the
+   * Boolean false, stops parsing and returns.
+   */
+  public parse(byteArray: Uint8Array|Buffer, valueCB: (val: any) => boolean|void) {
+    this.memBuf.writeByteArray(byteArray);
+    try {
+      while (this.memBuf.size() > 0) {
+        this.consumer = this.memBuf.makeConsumer();
+
+        // Have to reset stringTable for interned strings before each top-level parse call.
+        this.stringTable.length = 0;
+
+        const value = this._parse();
+        this.memBuf.consume(this.consumer);
+        if (valueCB(value) === false) {
+          return;
+        }
+      }
+    } catch (err) {
+      // If the error is `needMoreData`, we silently return. We'll retry by reparsing the message
+      // from scratch after the next push(). If buffers contain complete serialized messages, the
+      // cost should be minor. But this design might get very inefficient if we have big messages
+      // of arrays or dictionaries.
+      if (err.needMoreData) {
+        if (!err.consumedData || err.consumedData > 1024) {
+          // tslint:disable-next-line:no-console
+          console.log("Unmarshaller: Need more data; wasted parsing of %d bytes", err.consumedData);
+        }
+      } else {
+        err.message = "Unmarshaller: " + err.message;
+        throw err;
+      }
+    }
+  }
+
+  private _parse(): unknown {
+    const code = this.memBuf.readUint8(this.consumer);
+    this._lastCode = code;
+    switch (code) {
+      case marshalCodes.NULL:       return null;
+      case marshalCodes.NONE:       return null;
+      case marshalCodes.FALSE:      return false;
+      case marshalCodes.TRUE:       return true;
+      case marshalCodes.INT:        return this._parseInt();
+      case marshalCodes.INT64:      return this._parseInt64();
+      case marshalCodes.FLOAT:      return this._parseStringFloat();
+      case marshalCodes.BFLOAT:     return this._parseBinaryFloat();
+      case marshalCodes.STRING:     return this._parseByteString();
+      case marshalCodes.TUPLE:      return this._parseList();
+      case marshalCodes.LIST:       return this._parseList();
+      case marshalCodes.DICT:       return this._parseDict();
+      case marshalCodes.UNICODE:    return this._parseUnicode();
+      case marshalCodes.INTERNED:   return this._parseInterned();
+      case marshalCodes.STRINGREF:  return this._parseStringRef();
+      case marshalCodes.LONG:       return this._parseLong();
+        // None of the following are supported.
+        // case marshalCodes.STOPITER:
+        // case marshalCodes.ELLIPSIS:
+        // case marshalCodes.COMPLEX:
+        // case marshalCodes.CODE:
+        // case marshalCodes.UNKNOWN:
+        // case marshalCodes.SET:
+        // case marshalCodes.FROZENSET:
+      default:
+        throw new Error(`Unmarshaller: unsupported code "${String.fromCharCode(code)}" (${code})`);
+    }
+  }
+
+  private _parseInt() {
+    return this.memBuf.readInt32LE(this.consumer);
+  }
+
+  private _parseInt64() {
+    const low = this.memBuf.readInt32LE(this.consumer);
+    const hi = this.memBuf.readInt32LE(this.consumer);
+    if ((hi === 0 && low >= 0) || (hi === -1 && low < 0)) {
+      return low;
+    }
+    const unsignedLow = low < 0 ? TwoTo32 + low : low;
+    if (hi >= 0) {
+      return new BigInt(TwoTo32, [unsignedLow, hi], 1).toNative();
+    } else {
+      // This part is tricky. See unittests for check of correctness.
+      return new BigInt(TwoTo32, [TwoTo32 - unsignedLow, -hi - 1], -1).toNative();
+    }
+  }
+
+  private _parseLong() {
+    // The format is a 32-bit size whose sign is the sign of the result, followed by 16-bit digits
+    // in base 2**15.
+    const size = this.memBuf.readInt32LE(this.consumer);
+    const sign = size < 0 ? -1 : 1;
+    const numDigits = size < 0 ? -size : size;
+    const digits = [];
+    for (let i = 0; i < numDigits; i++) {
+      digits.push(this.memBuf.readInt16LE(this.consumer));
+    }
+    return new BigInt(TwoTo15, digits, sign).toNative();
+  }
+
+  private _parseStringFloat() {
+    const len = this.memBuf.readUint8(this.consumer);
+    const buf = this.memBuf.readString(this.consumer, len);
+    return parseFloat(buf);
+  }
+
+  private _parseBinaryFloat() {
+    return this.memBuf.readFloat64LE(this.consumer);
+  }
+
+  private _parseByteString(): string|Uint8Array {
+    const len = this.memBuf.readInt32LE(this.consumer);
+    return (this.bufferToString ?
+      this.memBuf.readString(this.consumer, len) :
+      this.memBuf.readByteArray(this.consumer, len));
+  }
+
+  private _parseInterned() {
+    const s = this._parseByteString();
+    this.stringTable.push(s);
+    return s;
+  }
+
+  private _parseStringRef() {
+    const index = this._parseInt();
+    return this.stringTable[index];
+  }
+
+  private _parseList() {
+    const len = this.memBuf.readInt32LE(this.consumer);
+    const value = [];
+    for (let i = 0; i < len; i++) {
+      value[i] = this._parse();
+    }
+    return value;
+  }
+
+  private _parseDict() {
+    const dict: {[key: string]: any} = {};
+    while (true) {    // eslint-disable-line no-constant-condition
+      let key = this._parse() as string|Uint8Array;
+      if (key === null && this._lastCode === marshalCodes.NULL) {
+        break;
+      }
+      const value = this._parse();
+      if (key !== null) {
+        if (key instanceof Uint8Array) {
+          key = MemBuffer.arrayToString(key);
+        }
+        dict[key as string] = value;
+      }
+    }
+    return dict;
+  }
+
+  private _parseUnicode() {
+    const len = this.memBuf.readInt32LE(this.consumer);
+    return this.memBuf.readString(this.consumer, len);
+  }
+}
+
+/**
+ * Similar to python's marshal.loads(). Parses the given bytes and returns the parsed value. There
+ * must not be any trailing data beyond the single marshalled value.
+ */
+export function loads(byteArray: Uint8Array|Buffer, options?: UnmarshalOptions): any {
+  const unmarshaller = new Unmarshaller(options);
+  let parsedValue;
+  unmarshaller.parse(byteArray, function(value) {
+    parsedValue = value;
+    return false;
+  });
+  if (typeof parsedValue === 'undefined') {
+    throw new Error("loads: input data truncated");
+  } else if (unmarshaller.memBuf.size() > 0) {
+    throw new Error("loads: extra bytes past end of input");
+  }
+  return parsedValue;
+}
+
+/**
+ * Serializes arbitrary data by first marshalling then converting to a base64 string.
+ */
+export function dumpBase64(data: any, options?: MarshalOptions) {
+  const marshaller = new Marshaller(options || {version: 2});
+  marshaller.marshal(data);
+  return marshaller.dumpAsBuffer().toString('base64');
+}
+
+/**
+ * Loads data from a base64 string, as serialized by dumpBase64().
+ */
+export function loadBase64(data: string, options?: UnmarshalOptions) {
+  return loads(Buffer.from(data, 'base64'), options);
+}

app/common/metricConfig.js

@@ -0,0 +1,252 @@
+/**
+ * File for configuring the metric collection bucket duration, data push intervals between client, server,
+ * and Grist Metrics EC2 instance, as well as individual metrics collected in the client and server.
+ */
+
+// Time interval settings (ms)
+exports.BUCKET_SIZE = 60 * 1000;
+
+exports.CLIENT_PUSH_INTERVAL = 120 * 1000;
+exports.SERVER_PUSH_INTERVAL = 120 * 1000;
+exports.MAX_PENDING_BUCKETS = 40;
+exports.CONN_RETRY = 20 * 1000;
+
+// Metrics use the general form:
+//    <category>.<short desc>
+// With prefixes, measurement type, and clientId/serverId added automatically on send.
+
+// 'type' is the measurement tool type, with options 'Switch', 'Counter', 'Gauge', 'Timer', and
+//  'ExecutionTimer'. (See metricTools.js for details)
+// Suffixes are added to the metric names depending on their measurement tool.
+// 'Switch'         => '.instances'
+// 'Gauge'          => '.total'
+// 'Counter'        => '.count'
+// 'Timer'          => '.time'
+// 'ExecutionTimer' => '.execution_time', '.count' (Execution timer automatically records a count)
+
+exports.clientMetrics = [
+  // General
+  {
+    name:   'sidepane.opens',
+    type:   'Counter',
+    desc:   'Number of times the side pane is opened'
+  },
+  {
+    name:   'app.client_active_span',
+    type:   'Timer',
+    desc:   'Total client time spent using grist'
+  },
+  {
+    name:   'app.connected_to_server_span',
+    type:   'Timer',
+    desc:   'Total time spent connected to the server'
+  },
+  {
+    name:   'app.disconnected_from_server_span',
+    type:   'Timer',
+    desc:   'Total time spent disconnected from the server'
+  },
+  // Docs
+  {
+    name:   'docs.num_open_6+_tables',
+    type:   'SamplingGauge',
+    desc:   'Number of open docs with more than 5 tables'
+  },
+  {
+    name:   'docs.num_open_0-5_tables',
+    type:   'SamplingGauge',
+    desc:   'Number of open docs with 0-5 tables'
+  },
+  // Tables
+  {
+    name:   'tables.num_tables',
+    type:   'SamplingGauge',
+    desc:   'Number of open tables'
+  },
+  {
+    name:   'tables.num_summary_tables',
+    type:   'SamplingGauge',
+    desc:   'Number of open sections in the current view'
+  },
+  // Views
+  {
+    name:   'views.code_view_open_span',
+    type:   'Timer',
+    desc:   'Time spent with code viewer open'
+  },
+  // Sections
+  {
+    name:   'sections.grid_open_span',
+    type:   'Timer',
+    desc:   'Time spent with gridview open'
+  },
+  {
+    name:   'sections.detail_open_span',
+    type:   'Timer',
+    desc:   'Time spent with gridview open'
+  },
+  {
+    name:   'sections.num_grid_sections',
+    type:   'SamplingGauge',
+    desc:   'Number of open sections in the current view'
+  },
+  {
+    name:   'sections.num_detail_sections',
+    type:   'SamplingGauge',
+    desc:   'Number of open sections in the current view'
+  },
+  {
+    name:   'sections.num_chart_sections',
+    type:   'SamplingGauge',
+    desc:   'Number of open sections in the current view'
+  },
+  {
+    name:   'sections.multiple_open_span',
+    type:   'Timer',
+    desc:   'Time spent with multiple sections open'
+  },
+  // Performance
+  {
+    name:   'performance.server_action',
+    type:   'ExecutionTimer',
+    desc:   'Time for a server action to complete'
+  },
+  {
+    name:   'performance.doc_load',
+    type:   'ExecutionTimer',
+    desc:   'Time to load a document'
+  },
+  // Columns
+  {
+    name:   'cols.num_formula_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of formula columns in open documents'
+  },
+  {
+    name:   'cols.num_text_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of text columns in open documents'
+  },
+  {
+    name:   'cols.num_int_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of integer columns in open documents'
+  },
+  {
+    name:   'cols.num_numeric_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of numeric columns in open documents'
+  },
+  {
+    name:   'cols.num_date_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of date columns in open documents'
+  },
+  {
+    name:   'cols.num_datetime_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of datetime columns in open documents'
+  },
+  {
+    name:   'cols.num_ref_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of reference columns in open documents'
+  },
+  {
+    name:   'cols.num_attachments_cols',
+    type:   'SamplingGauge',
+    desc:   'Number of attachments columns in open documents'
+  },
+  {
+    name:   'performance.front_end_errors',
+    type:   'Counter',
+    desc:   'Number of frontend errors'
+  }
+  // TODO: Implement the following:
+  // {
+  //   name:   'grist-rt.performance.view_swap',
+  //   type:   'ExecutionTimer',
+  //   desc:   'Time to swap views'
+  // }
+];
+
+exports.serverMetrics = [
+  // General
+  {
+    name:   'app.server_active',
+    type:   'Switch',
+    desc:   'Number of users currently using grist'
+  },
+  {
+    name:   'app.server_active_span',
+    type:   'Timer',
+    desc:   'Total server time spent using grist'
+  },
+  {
+    name:   'app.have_doc_open',
+    type:   'Switch',
+    desc:   'Number of users with at least one doc open'
+  },
+  {
+    name:   'app.doc_open_span',
+    type:   'Timer',
+    desc:   'Total time spent with at least one doc open'
+  },
+  // Docs
+  {
+    name:   'docs.num_open',
+    type:   'Gauge',
+    desc:   'Number of open docs'
+  },
+  {
+    name:   'performance.node_memory_usage',
+    type:   'SamplingGauge',
+    desc:   'Memory utilization in bytes of the node process'
+  }
+  // TODO: Implement the following:
+  // {
+  //   name:   'grist-rt.docs.total_size_open',
+  //   type:   'Gauge',
+  //   desc:   'Cumulative size of open docs'
+  // }
+  // {
+  //   name:   'grist-rt.performance.open_standalone_app',
+  //   type:   'ExecutionTimer',
+  //   desc:   'Time to start standalone app'
+  // }
+  // {
+  //   name:   'grist-rt.performance.sandbox_recalculation',
+  //   type:   'ExecutionTimer',
+  //   desc:   'Time for sandbox recalculation to occur'
+  // }
+  // {
+  //   name:   'grist-rt.performance.open_standalone_app',
+  //   type:   'ExecutionTimer',
+  //   desc:   'Time to start standalone app'
+  // }
+  // {
+  //   name:   'grist-rt.performance.node_cpu_usage',
+  //   type:   'SamplingGauge',
+  //   desc:   'Amount of time node was using the cpu in the interval'
+  // }
+  // {
+  //   name:   'grist-rt.performance.sandbox_cpu_usage',
+  //   type:   'SamplingGauge',
+  //   desc:   'Amount of time the sandbox was using the cpu in the interval'
+  // }
+  // {
+  //   name:   'grist-rt.performance.chrome_cpu_usage',
+  //   type:   'SamplingGauge',
+  //   desc:   'Amount of time chrome was using the cpu in the interval'
+  // }
+  // {
+  //   name:   'grist-rt.performance.sandbox_memory_usage',
+  //   type:   'SamplingGauge',
+  //   desc:   'Memory utilization in bytes of the sandbox process'
+  // }
+  // {
+  //   name:   'grist-rt.performance.chrome_memory_usage',
+  //   type:   'SamplingGauge',
+  //   desc:   'Memory utilization in bytes of the chrome process'
+  // }
+];

app/common/metricTools.js

@@ -0,0 +1,261 @@
+const _ = require('underscore');
+const gutil = require('./gutil');
+const metricConfig = require('./metricConfig');
+
+// TODO: Create a metric test class and write tests for each metric tool.
+
+/**
+ * Base class for tools to gather metrics. Should not be instantiated.
+ */
+function MetricTool(name) {
+  this.name = name;
+}
+
+// Should be implemented by extending classes
+MetricTool.prototype._getSuffix = function() {
+  throw new Error("Not implemented");
+};
+
+// Should be overridden by extending classes depending on desired reset behavior
+MetricTool.prototype.reset = _.noop;
+
+// Returns the name of the metric with its suffix appended to the end.
+// NOTE: Should return names in the same order as getValues.
+MetricTool.prototype.getName = function() {
+  return this.name + '.' + this._getSuffix();
+};
+
+// Should be implemented by extending classes. Returns the value of the tool for a bucket.
+// @param {Number} bucketEndTime - The desired bucket's end time in milliseconds
+MetricTool.prototype.getValue = function(bucketEndTime) {
+  throw new Error("Not implemented");
+};
+
+// Returns a list of all primitive metrics this tool is made up of.
+// Only requires overridding by non-primitive metrics.
+MetricTool.prototype.getPrimitiveMetrics = function() {
+  return [this];
+};
+
+/**
+ * Counts the number of times an event has occurred in the current bucket.
+ */
+function Counter(name) {
+  MetricTool.call(this, name);
+  this.val = 0;
+}
+_.extend(Counter.prototype, MetricTool.prototype);
+
+Counter.prototype.inc = function() {
+  this.val += 1;
+};
+
+Counter.prototype._getSuffix = function() {
+  return 'count';
+};
+
+Counter.prototype.getValue = function(bucketEndTime) {
+  // If the bucket is more recent than the last one where counting occurred, return 0
+  return this.val;
+};
+
+Counter.prototype.reset = function(bucketEndTime) {
+  this.val = 0;
+};
+exports.Counter = Counter;
+
+
+/**
+ * Keeps track of a count that persists across buckets.
+ */
+function Gauge(name) {
+  MetricTool.call(this, name);
+  this.val = null;
+}
+_.extend(Gauge.prototype, MetricTool.prototype);
+
+Gauge.prototype.set = function(num) {
+  this.val = num;
+};
+
+Gauge.prototype.inc = function() {
+  this.val = (this.val ? this.val + 1 : 1);
+};
+
+Gauge.prototype.dec = function() {
+  this.val -= 1;
+};
+
+Gauge.prototype._getSuffix = function() {
+  return 'total';
+};
+
+Gauge.prototype.getValue = function(bucketEndTime) {
+  return this.val;
+};
+exports.Gauge = Gauge;
+
+
+/**
+ * A gauge that pulls samples using a callback function
+ */
+function SamplingGauge(name) {
+  MetricTool.call(this, name);
+  this.callback = _.constant(null);
+}
+_.extend(SamplingGauge.prototype, MetricTool.prototype);
+
+SamplingGauge.prototype.assignCallback = function(callback) {
+  this.callback = callback;
+};
+
+SamplingGauge.prototype._getSuffix = function() {
+  return 'total';
+};
+
+SamplingGauge.prototype.getValue = function(bucketEndTime) {
+  return this.callback();
+};
+exports.SamplingGauge = SamplingGauge;
+
+
+/**
+ * Keeps track of whether or not a certain condition is met. Useful for statistics
+ * which measure the number of users who meet a certain criteria. Persists across buckets.
+ */
+function Switch(name) {
+  MetricTool.call(this, name);
+  this.val = null;
+}
+_.extend(Switch.prototype, Gauge.prototype);
+
+Switch.prototype.set = function(bool) {
+  this.val = bool ? 1 : 0;
+};
+
+Switch.prototype._getSuffix = function() {
+  return 'instances';
+};
+
+Switch.prototype.getValue = function(bucketEndTime) {
+  return this.val;
+};
+exports.Switch = Switch;
+
+
+/**
+ * Keeps track of the amount of time in each bucket that an event is occurring (ms).
+ */
+function Timer(name) {
+  MetricTool.call(this, name);
+  this.val = 0;               // The sum of all runtimes in the last updated bucket
+  this.startTime = 0;         // The time (in ms since the bucket started) when the timer was started
+  this.running = false;
+}
+_.extend(Timer.prototype, MetricTool.prototype);
+
+Timer.prototype.setRunning = function(bool) {
+  return bool ? this.start() : this.stop();
+};
+
+Timer.prototype.start = function() {
+  if (this.running) {
+    return;
+  }
+  // Record start time and set to running
+  this.startTime = Date.now();
+  this.running = true;
+};
+
+Timer.prototype.stop = function() {
+  if (!this.running) {
+    return;
+  }
+  // Add time since start to value and set running to false
+  var stopTime = Date.now();
+  this.val += stopTime - this.startTime;
+  this.running = false;
+};
+
+Timer.prototype._getSuffix = function() {
+  return 'time';
+};
+
+Timer.prototype.getValue = function(bucketEndTime) {
+  // Add the value and the time to the end of the bucket if the timer is running
+  return this.val + (this.running ? Math.max(0, bucketEndTime - this.startTime) : 0);
+};
+
+Timer.prototype.reset = function(bucketEndTime) {
+  this.val = 0;
+  this.startTime = Math.max(bucketEndTime, this.startTime);
+};
+exports.Timer = Timer;
+
+
+/**
+ * Keeps track of the amount of time in an event takes, and the number of times that event occurs (ms).
+ */
+function ExecutionTimer(name) {
+  MetricTool.call(this, name);
+  this.startTime = 0;         // The last time (in ms) the timer was started
+  this.val = 0;
+  this.running = false;
+  // Counter keeps track of the total number of executions in the current bucket.
+  // An execution is in a bucket if it ended in that bucket.
+  this.counter = new Counter(name);
+}
+_.extend(ExecutionTimer.prototype, MetricTool.prototype);
+
+ExecutionTimer.prototype.setRunning = function(bool) {
+  return bool ? this.start() : this.stop();
+};
+
+ExecutionTimer.prototype.start = function() {
+  if (this.running) {
+    return;
+  }
+  this.startTime = Date.now();
+  this.running = true;
+};
+
+ExecutionTimer.prototype.stop = function() {
+  if (!this.running) {
+    return;
+  }
+  var stopTime = Date.now();
+  this.val += stopTime - this.startTime;
+  this.counter.inc();
+  this.running = false;
+};
+
+ExecutionTimer.prototype._getSuffix = function() {
+  return 'execution_time';
+};
+
+ExecutionTimer.prototype.getValue = function(bucketEndTime) {
+  return this.val;
+};
+
+ExecutionTimer.prototype.reset = function(bucketEndTime) {
+  this.val = 0;
+  this.counter.reset();
+};
+
+ExecutionTimer.prototype.getPrimitiveMetrics = function() {
+  return [this, this.counter];
+};
+exports.ExecutionTimer = ExecutionTimer;
+
+
+// Returns the time rounded down to the start of the current bucket's time window (in ms).
+function getBucketStartTime(now) {
+  return gutil.roundDownToMultiple(now, metricConfig.BUCKET_SIZE);
+}
+exports.getBucketStartTime = getBucketStartTime;
+
+// Returns the time until the start of the next bucket (in ms).
+function getDeltaMs(now) {
+  return getBucketStartTime(now) + metricConfig.BUCKET_SIZE - now;
+}
+exports.getDeltaMs = getDeltaMs;

app/common/orgNameUtils.ts

@@ -0,0 +1,56 @@
+const BLACKLISTED_SUBDOMAINS = new Set([
+  // from wiki page as of 2018-12-14
+  'aws',
+  'gristlogin',
+  'issues',
+  'metrics',
+  'phab',
+  'releases',
+  'test',
+  'vpn',
+  'www',
+
+  // A few more reserved just in case.  The minimum length requirement would eliminate
+  // some in any case, but specified here also in case that minimum changes.
+  'w', 'ww', 'wwww', 'wwwww',
+  'docs', 'api', 'static',
+  'ftp', 'imap', 'pop', 'smtp', 'mail', 'git', 'blog', 'wiki', 'support', 'kb', 'help',
+  'admin', 'store', 'dev', 'beta', 'dev',
+
+  // a few random tech brands
+  'google', 'apple', 'microsoft', 'ms', 'facebook', 'fb', 'twitter', 'youtube', 'yt',
+
+  // updates for new special domains
+  'current', 'staging', 'prod', 'login', 'login-dev',
+]);
+
+/**
+ *
+ * Checks whether the subdomain is on the list of forbidden subdomains.
+ * See https://phab.getgrist.com/w/hosting/v1/urls/#organization-subdomains
+ *
+ * Also enforces various sanity checks.
+ *
+ * Throws if the subdomain is invalid.
+ *
+ */
+export function checkSubdomainValidity(subdomain: string): void {
+  // stick with limited alphanumeric subdomains.
+  if (!(/^[a-z][-a-z0-9]*$/.test(subdomain))) {
+    throw new Error('Domain must include letters, numbers, and dashes only.');
+  }
+  // 'docs-*' is reserved for personal orgs.
+  if (subdomain.startsWith('docs-')) { throw new Error('Domain cannot use reserved prefix "docs-".'); }
+  // 'doc-worker-*' is reserved for doc workers.
+  if (subdomain.startsWith('doc-worker-')) { throw new Error('Domain cannot use reserved prefix "doc-worker-".'); }
+  // special subdomains like _domainkey.
+  if (subdomain.startsWith('_')) { throw new Error('Domain cannot use reserved prefix "_".'); }
+  // some domains are currently in use for testing v1.
+  if (subdomain.startsWith('v1-')) { throw new Error('Domain cannot use reserved prefix "v1-".'); }
+  // check limit of 63 characters on dns label.
+  if (subdomain.length > 63) { throw new Error('Domain must contain less than 64 characters.'); }
+  // check the subdomain isn't too short.
+  if (subdomain.length <= 2) { throw new Error('Domain must contain more than 2 characters.'); }
+  // a small blacklist prepared by hand.
+  if (BLACKLISTED_SUBDOMAINS.has(subdomain)) { throw new Error('Invalid domain value.'); }
+}

app/common/parseDate.ts

@@ -0,0 +1,107 @@
+import * as moment from 'moment-timezone';
+
+// Order of formats to try if the date cannot be parsed as the currently set format.
+// Formats are parsed in momentjs strict mode, but separator matching and the MM/DD
+// two digit requirement are ignored. Also, partial completion is permitted, so formats
+// may match even if only beginning elements are provided.
+// TODO: These should be affected by the user's locale/settings.
+// TODO: We may want to consider adding default time formats as well to support more
+//  time formats.
+const PARSER_FORMATS: string[] = [
+  'M D YYYY',
+  'M D YY',
+  'M D',
+  'M',
+  'MMMM D YYYY',
+  'MMMM D',
+  'MMMM Do YYYY',
+  'MMMM Do',
+  'MMMM',
+  'MMM D YYYY',
+  'MMM D',
+  'D MMM YYYY',
+  'D MMM',
+  'MMM',
+  'YYYY M D',
+  'YYYY M',
+  'YYYY',
+  'D M YYYY',
+  'D M YY',
+  'D M',
+  'D'
+];
+
+interface ParseOptions {
+  time?: string;
+  dateFormat?: string;
+  timeFormat?: string;
+  timezone?: string;
+}
+
+/**
+ * parseDate - Attempts to parse a date string using several common formats. Returns the
+ *  timestamp of the parsed date in seconds since epoch, or returns null on failure.
+ * @param {String} date - The date string to parse.
+ * @param {String} options.dateFormat - The preferred momentjs format to use to parse the
+ *  date. This is attempted before the default formats.
+ * @param {String} options.time - The time string to parse.
+ * @param {String} options.timeFormat - The momentjs format to use to parse the time. This
+ *  must be given if options.time is given.
+ * @param {String} options.timezone - The timezone string for the date/time, which affects
+ *  the resulting timestamp.
+ */
+export function parseDate(date: string, options: ParseOptions = {}): number | null {
+  // If no date, return null.
+  if (!date) {
+    return null;
+  }
+  // Not picky about separators, so replace them in the date and format strings to be spaces.
+  const separators = /\W+/g;
+  const dateFormats = PARSER_FORMATS.slice();
+  // If a preferred parse format is given, set that to be the first parser used.
+  if (options.dateFormat) {
+    // Momentjs has an undesirable feature in strict mode where MM and DD
+    // matches require two digit numbers. Change MM, DD to M, D.
+    const format = options.dateFormat.replace(/\bMM\b/g, 'M')
+      .replace(/\bDD\b/g, 'D')
+      .replace(separators, ' ');
+    dateFormats.unshift(_getPartialFormat(date, format));
+  }
+  const cleanDate = date.replace(separators, ' ');
+  const datetime = (options.time ? `${cleanDate} ${options.time}` : cleanDate).trim();
+  for (const f of dateFormats) {
+    // Momentjs has an undesirable feature in strict mode where HH, mm, and ss
+    // matches require two digit numbers. Change HH, mm, and ss to H, m, and s.
+    const timeFormat = options.timeFormat ? options.timeFormat.replace(/\bHH\b/g, 'H')
+      .replace(/\bmm\b/g, 'm')
+      .replace(/\bss\b/g, 's') : null;
+    const fullFormat = options.time && timeFormat ? `${f} ${timeFormat}` : f;
+    const m = moment.tz(datetime, fullFormat, true, options.timezone || 'UTC');
+    if (m.isValid()) {
+      return m.valueOf() / 1000;
+    }
+  }
+  return null;
+}
+
+// Helper function to get the partial format string based on the input. Momentjs has a feature
+// which allows defaulting to the current year, month and/or day if not accounted for in the
+// parser. We remove any parts of the parser not given in the input to take advantage of this
+// feature.
+function _getPartialFormat(input: string, format: string): string {
+  // Define a regular expression to match contiguous separators.
+  const re = /\W+/g;
+  // Clean off any whitespace from the ends, and count the number of separators.
+  const inputMatch = input.trim().match(re);
+  const numInputSeps = inputMatch ? inputMatch.length : 0;
+  // Find the separator matches in the format string.
+  let formatMatch;
+  for (let i = 0; i < numInputSeps + 1; i++) {
+    formatMatch = re.exec(format);
+    if (!formatMatch) {
+      break;
+    }
+  }
+  // Get the format string up until the corresponding input ends.
+  return formatMatch ? format.slice(0, formatMatch.index) : format;
+}

app/common/plugin.ts

@@ -0,0 +1,77 @@
+/**
+ * Plugin's utilities common to server and client.
+ */
+import {BarePlugin, Implementation} from 'app/plugin/PluginManifest';
+
+export type LocalPluginKind = "installed"|"builtIn";
+
+export interface ImplDescription {
+  localPluginId: string;
+  implementation: Implementation;
+}
+
+export interface FileParser {
+  fileExtensions: string[];
+  parseOptions?: ImplDescription;
+  fileParser: ImplDescription;
+}
+
+// Deprecated, use FileParser or ImportSource instead.
+export interface FileImporter {
+  id: string;
+  fileExtensions?: string[];
+  script?: string;
+  scriptFullPath?: string;
+  filePicker?: string;
+  filePickerFullPath?: string;
+}
+
+/**
+ * Manifest parsing error.
+ */
+export interface ManifestParsingError {
+  yamlError?: any;
+  jsonError?: any;
+  cannotReadError?: any;
+  missingEntryErrors?: string;
+}
+
+/**
+ * Whether the importer provides a file picker.
+ */
+export function isPicker(importer: FileImporter): boolean {
+  return importer.filePicker !== undefined;
+}
+
+/**
+ * A Plugin that was found in the system, either installed or builtin.
+ */
+export interface LocalPlugin {
+  /**
+   * the plugin's manifest
+   */
+  manifest: BarePlugin;
+  /**
+   * The path to the plugin's folder.
+   */
+  path: string;
+  /**
+   * A name to uniquely identify a LocalPlugin.
+   */
+  readonly id: string;
+}
+
+export interface DirectoryScanEntry {
+  manifest?: BarePlugin;
+  /**
+   * User-friendly error messages.
+   */
+  errors?: any[];
+  path: string;
+  id: string;
+}
+
+/**
+ * The contributions type.
+ */
+export type Contribution = "importSource" | "fileParser";

app/common/resetOrg.ts

@@ -0,0 +1,44 @@
+import {ManagerDelta, PermissionDelta, UserAPI} from 'app/common/UserAPI';
+
+/**
+ * A utility to reset an organization into the state it would have when first
+ * created - no docs, one workspace called "Home", a single user.  Should be
+ * called by a user who is both an owner of the org and a billing manager.
+ */
+export async function resetOrg(api: UserAPI, org: string|number) {
+  const session = await api.getSessionActive();
+  if (!(session.org && session.org.access === 'owners')) {
+    throw new Error('user must be an owner of the org to be reset');
+  }
+  const billing = api.getBillingAPI();
+  const account = await billing.getBillingAccount();
+  if (!account.managers.some(manager => (manager.id === session.user.id))) {
+    throw new Error('user must be a billing manager');
+  }
+  const wss = await api.getOrgWorkspaces(org);
+  for (const ws of wss) {
+    if (!ws.isSupportWorkspace) {
+      await api.deleteWorkspace(ws.id);
+    }
+  }
+  await api.newWorkspace({name: 'Home'}, org);
+  const permissions: PermissionDelta = { users: {} };
+  for (const user of (await api.getOrgAccess(org)).users) {
+    if (user.id !== session.user.id) {
+      permissions.users![user.email] = null;
+    }
+  }
+  await api.updateOrgPermissions(org, permissions);
+  // For non-individual accounts, update billing managers (individual accounts will
+  // throw an error if we try to do this).
+  if (!account.individual) {
+    const managers: ManagerDelta = { users: {} };
+    for (const user of account.managers) {
+      if (user.id !== session.user.id) {
+        managers.users[user.email] = null;
+      }
+    }
+    await billing.updateBillingManagers(managers);
+  }
+  return api;
+}

app/common/roles.ts

@@ -0,0 +1,85 @@
+export const OWNER  = 'owners';
+export const EDITOR = 'editors';
+export const VIEWER = 'viewers';
+export const GUEST  = 'guests';
+export const MEMBER = 'members';
+
+// Roles ordered from most to least permissive.
+const roleOrder: Array<Role|null> = [OWNER, EDITOR, VIEWER, MEMBER, GUEST, null];
+
+export type BasicRole = 'owners'|'editors'|'viewers';
+export type NonMemberRole = BasicRole|'guests';
+export type NonGuestRole = BasicRole|'members';
+export type Role = NonMemberRole|'members';
+
+// Returns the BasicRole (or null) with the same effective access as the given role.
+export function getEffectiveRole(role: Role|null): BasicRole|null {
+  if (role === GUEST || role === MEMBER) {
+    return VIEWER;
+  } else {
+    return role;
+  }
+}
+
+export function canEditAccess(role: string|null): boolean {
+  return role === OWNER;
+}
+
+// Note that while canEdit has the same return value as canDelete, the functions are
+// kept separate as they may diverge in the future.
+export function canEdit(role: string|null): boolean {
+  return role === OWNER || role === EDITOR;
+}
+
+export function canDelete(role: string|null): boolean {
+  return role === OWNER || role === EDITOR;
+}
+
+export function canView(role: string|null): boolean {
+  return role !== null;
+}
+
+// Returns true if the role string is a valid role or null.
+export function isValidRole(role: string|null): role is Role|null {
+  return (roleOrder as Array<string|null>).includes(role);
+}
+
+// Returns true if the role string is a valid non-Guest, non-Member, non-null role.
+export function isBasicRole(role: string|null): role is BasicRole {
+  return Boolean(role && role !== GUEST && role !== MEMBER && isValidRole(role));
+}
+
+// Returns true if the role string is a valid non-Guest, non-null role.
+export function isNonGuestRole(role: string|null): role is NonGuestRole {
+  return Boolean(role && role !== GUEST && isValidRole(role));
+}
+
+/**
+ * Returns out of any number of group role names the one that offers more permissions. The function
+ * is overloaded so that the output type matches the specificity of the input values.
+ */
+export function getStrongestRole<T extends Role|null>(...args: T[]): T {
+  return getFirstMatchingRole(roleOrder, args);
+}
+
+/**
+ * Returns out of any number of group role names the one that offers fewer permissions. The function
+ * is overloaded so that the output type matches the specificity of the input values.
+ */
+export function getWeakestRole<T extends Role|null>(...args: T[]): T {
+  return getFirstMatchingRole(roleOrder.slice().reverse(), args);
+}
+
+// Returns which of the `anyOf` args comes first in `array`. Helper for getStrongestRole
+// and getWeakestRole.
+function getFirstMatchingRole<T extends Role|null>(array: Array<Role|null>, anyOf: T[]): T {
+  if (anyOf.length === 0) {
+    throw new Error(`getFirstMatchingRole: No roles given`);
+  }
+  for (const role of anyOf) {
+    if (!isValidRole(role)) {
+      throw new Error(`getFirstMatchingRole: Invalid role ${role}`);
+    }
+  }
+  return array.find((item) => anyOf.includes(item as T)) as T;
+}

app/common/schema.ts

@@ -0,0 +1,336 @@
+/*** THIS FILE IS AUTO-GENERATED BY sandbox/gen_js_schema.py ***/
+// tslint:disable:object-literal-key-quotes
+
+export const schema = {
+
+  "_grist_DocInfo": {
+    docId               : "Text",
+    peers               : "Text",
+    basketId            : "Text",
+    schemaVersion       : "Int",
+    timezone            : "Text",
+  },
+
+  "_grist_Tables": {
+    tableId             : "Text",
+    primaryViewId       : "Ref:_grist_Views",
+    summarySourceTable  : "Ref:_grist_Tables",
+    onDemand            : "Bool",
+  },
+
+  "_grist_Tables_column": {
+    parentId            : "Ref:_grist_Tables",
+    parentPos           : "PositionNumber",
+    colId               : "Text",
+    type                : "Text",
+    widgetOptions       : "Text",
+    isFormula           : "Bool",
+    formula             : "Text",
+    label               : "Text",
+    untieColIdFromLabel : "Bool",
+    summarySourceCol    : "Ref:_grist_Tables_column",
+    displayCol          : "Ref:_grist_Tables_column",
+    visibleCol          : "Ref:_grist_Tables_column",
+  },
+
+  "_grist_Imports": {
+    tableRef            : "Ref:_grist_Tables",
+    origFileName        : "Text",
+    parseFormula        : "Text",
+    delimiter           : "Text",
+    doublequote         : "Bool",
+    escapechar          : "Text",
+    quotechar           : "Text",
+    skipinitialspace    : "Bool",
+    encoding            : "Text",
+    hasHeaders          : "Bool",
+  },
+
+  "_grist_External_database": {
+    host                : "Text",
+    port                : "Int",
+    username            : "Text",
+    dialect             : "Text",
+    database            : "Text",
+    storage             : "Text",
+  },
+
+  "_grist_External_table": {
+    tableRef            : "Ref:_grist_Tables",
+    databaseRef         : "Ref:_grist_External_database",
+    tableName           : "Text",
+  },
+
+  "_grist_TableViews": {
+    tableRef            : "Ref:_grist_Tables",
+    viewRef             : "Ref:_grist_Views",
+  },
+
+  "_grist_TabItems": {
+    tableRef            : "Ref:_grist_Tables",
+    viewRef             : "Ref:_grist_Views",
+  },
+
+  "_grist_TabBar": {
+    viewRef             : "Ref:_grist_Views",
+    tabPos              : "PositionNumber",
+  },
+
+  "_grist_Pages": {
+    viewRef             : "Ref:_grist_Views",
+    indentation         : "Int",
+    pagePos             : "PositionNumber",
+  },
+
+  "_grist_Views": {
+    name                : "Text",
+    type                : "Text",
+    layoutSpec          : "Text",
+  },
+
+  "_grist_Views_section": {
+    tableRef            : "Ref:_grist_Tables",
+    parentId            : "Ref:_grist_Views",
+    parentKey           : "Text",
+    title               : "Text",
+    defaultWidth        : "Int",
+    borderWidth         : "Int",
+    theme               : "Text",
+    options             : "Text",
+    chartType           : "Text",
+    layoutSpec          : "Text",
+    filterSpec          : "Text",
+    sortColRefs         : "Text",
+    linkSrcSectionRef   : "Ref:_grist_Views_section",
+    linkSrcColRef       : "Ref:_grist_Tables_column",
+    linkTargetColRef    : "Ref:_grist_Tables_column",
+    embedId             : "Text",
+  },
+
+  "_grist_Views_section_field": {
+    parentId            : "Ref:_grist_Views_section",
+    parentPos           : "PositionNumber",
+    colRef              : "Ref:_grist_Tables_column",
+    width               : "Int",
+    widgetOptions       : "Text",
+    displayCol          : "Ref:_grist_Tables_column",
+    visibleCol          : "Ref:_grist_Tables_column",
+    filter              : "Text",
+  },
+
+  "_grist_Validations": {
+    formula             : "Text",
+    name                : "Text",
+    tableRef            : "Int",
+  },
+
+  "_grist_REPL_Hist": {
+    code                : "Text",
+    outputText          : "Text",
+    errorText           : "Text",
+  },
+
+  "_grist_Attachments": {
+    fileIdent           : "Text",
+    fileName            : "Text",
+    fileType            : "Text",
+    fileSize            : "Int",
+    imageHeight         : "Int",
+    imageWidth          : "Int",
+    timeUploaded        : "DateTime",
+  },
+
+  "_grist_ACLRules": {
+    resource            : "Ref:_grist_ACLResources",
+    permissions         : "Int",
+    principals          : "Text",
+    aclFormula          : "Text",
+    aclColumn           : "Ref:_grist_Tables_column",
+  },
+
+  "_grist_ACLResources": {
+    tableId             : "Text",
+    colIds              : "Text",
+  },
+
+  "_grist_ACLPrincipals": {
+    type                : "Text",
+    userEmail           : "Text",
+    userName            : "Text",
+    groupName           : "Text",
+    instanceId          : "Text",
+  },
+
+  "_grist_ACLMemberships": {
+    parent              : "Ref:_grist_ACLPrincipals",
+    child               : "Ref:_grist_ACLPrincipals",
+  },
+
+};
+
+export interface SchemaTypes {
+
+  "_grist_DocInfo": {
+    docId: string;
+    peers: string;
+    basketId: string;
+    schemaVersion: number;
+    timezone: string;
+  };
+
+  "_grist_Tables": {
+    tableId: string;
+    primaryViewId: number;
+    summarySourceTable: number;
+    onDemand: boolean;
+  };
+
+  "_grist_Tables_column": {
+    parentId: number;
+    parentPos: number;
+    colId: string;
+    type: string;
+    widgetOptions: string;
+    isFormula: boolean;
+    formula: string;
+    label: string;
+    untieColIdFromLabel: boolean;
+    summarySourceCol: number;
+    displayCol: number;
+    visibleCol: number;
+  };
+
+  "_grist_Imports": {
+    tableRef: number;
+    origFileName: string;
+    parseFormula: string;
+    delimiter: string;
+    doublequote: boolean;
+    escapechar: string;
+    quotechar: string;
+    skipinitialspace: boolean;
+    encoding: string;
+    hasHeaders: boolean;
+  };
+
+  "_grist_External_database": {
+    host: string;
+    port: number;
+    username: string;
+    dialect: string;
+    database: string;
+    storage: string;
+  };
+
+  "_grist_External_table": {
+    tableRef: number;
+    databaseRef: number;
+    tableName: string;
+  };
+
+  "_grist_TableViews": {
+    tableRef: number;
+    viewRef: number;
+  };
+
+  "_grist_TabItems": {
+    tableRef: number;
+    viewRef: number;
+  };
+
+  "_grist_TabBar": {
+    viewRef: number;
+    tabPos: number;
+  };
+
+  "_grist_Pages": {
+    viewRef: number;
+    indentation: number;
+    pagePos: number;
+  };
+
+  "_grist_Views": {
+    name: string;
+    type: string;
+    layoutSpec: string;
+  };
+
+  "_grist_Views_section": {
+    tableRef: number;
+    parentId: number;
+    parentKey: string;
+    title: string;
+    defaultWidth: number;
+    borderWidth: number;
+    theme: string;
+    options: string;
+    chartType: string;
+    layoutSpec: string;
+    filterSpec: string;
+    sortColRefs: string;
+    linkSrcSectionRef: number;
+    linkSrcColRef: number;
+    linkTargetColRef: number;
+    embedId: string;
+  };
+
+  "_grist_Views_section_field": {
+    parentId: number;
+    parentPos: number;
+    colRef: number;
+    width: number;
+    widgetOptions: string;
+    displayCol: number;
+    visibleCol: number;
+    filter: string;
+  };
+
+  "_grist_Validations": {
+    formula: string;
+    name: string;
+    tableRef: number;
+  };
+
+  "_grist_REPL_Hist": {
+    code: string;
+    outputText: string;
+    errorText: string;
+  };
+
+  "_grist_Attachments": {
+    fileIdent: string;
+    fileName: string;
+    fileType: string;
+    fileSize: number;
+    imageHeight: number;
+    imageWidth: number;
+    timeUploaded: number;
+  };
+
+  "_grist_ACLRules": {
+    resource: number;
+    permissions: number;
+    principals: string;
+    aclFormula: string;
+    aclColumn: number;
+  };
+
+  "_grist_ACLResources": {
+    tableId: string;
+    colIds: string;
+  };
+
+  "_grist_ACLPrincipals": {
+    type: string;
+    userEmail: string;
+    userName: string;
+    groupName: string;
+    instanceId: string;
+  };
+
+  "_grist_ACLMemberships": {
+    parent: number;
+    child: number;
+  };
+
+}

app/common/sharing.ts

@@ -0,0 +1,43 @@
+import {EncActionBundleFromHub} from 'app/common/EncActionBundle';
+
+export const allToken: string = '#ALL';
+
+/**
+ * Messages received from SQS
+ */
+export interface Message {
+  type: MessageType;
+  content: Invite | EncActionBundleFromHub;
+  docId: string; // The docId to which the message pertains.
+}
+
+export enum MessageType {
+  invite = 1,
+  accept,
+  decline,
+  action
+}
+
+export interface Invite {
+  senderEmail: string;
+  senderName?: string;
+  docId: string;  // Indicates the doc to which the user is being invited to join.
+  docName: string; // Indicates the docName at the time of sending for user doc recognition.
+  isUnread?: boolean;
+  isIgnored?: boolean;
+}
+
+/**
+ * Contains information about someone who may or may not be a Grist user.
+ */
+export interface Peer {
+  email: string;
+  name?: string;
+  instIds?: string[];
+}
+
+export interface EmailResult {
+  email: string;
+  instIds: string[];
+  name?: string;
+}

app/common/tbind.ts

@@ -0,0 +1,15 @@
+/**
+ * A version of Function.bind() that preserves types.
+ */
+
+// tslint:disable:max-line-length
+
+// Bind just the context for a function of up to 4 args.
+export function tbind<T, R, Args extends any[]>(func: (this: T, ...a: Args) => R, context: T): (...a: Args) => R;
+
+// Bind context and first arg for a function of up to 5 args.
+export function tbind<T, R, X, Args extends any[]>(func: (this: T, x: X, ...a: Args) => R, context: T, x: X): (...a: Args) => R;
+
+export function tbind(func: any, context: any, ...boundArgs: any[]): any {
+  return func.bind(context, ...boundArgs);
+}

app/common/timeFormat.ts

@@ -0,0 +1,56 @@
+/**
+ * timeFormat(format, date) formats the passed-in Date object using the format string. The format
+ * string may contain the following:
+ *    'h': hour (00 - 23)
+ *    'm': minute (00 - 59)
+ *    's': second (00 - 59)
+ *    'd': day of the month (01 - 31)
+ *    'n': month (01 - 12)
+ *    'y': 4-digit year
+ *    'M': milliseconds (000 - 999)
+ *    'Y': date as 20140212
+ *    'D': date as 2014-02-12
+ *    'T': time as 00:51:06
+ *    'A': full time and date, as 2014-02-12 00:51:06.123
+ * @param {String} format The format string.
+ * @param {Date} date The date/time object to format.
+ * @returns {String} The formatted date and/or time.
+ */
+
+function pad(num: number, len: number): string {
+  const s = num.toString();
+  return s.length >= len ? s : "00000000".slice(0, len - s.length) + s;
+}
+
+type FormatHelper = (out: string[], date: Date) => void;
+const timeFormatKeys: {[spec: string]: FormatHelper} = {
+  h: (out, date) => out.push(pad(date.getHours(), 2)),
+  m: (out, date) => out.push(pad(date.getMinutes(), 2)),
+  s: (out, date) => out.push(pad(date.getSeconds(), 2)),
+  d: (out, date) => out.push(pad(date.getDate(), 2)),
+  n: (out, date) => out.push(pad(date.getMonth() + 1, 2)),
+  y: (out, date) => out.push("" + date.getFullYear()),
+  M: (out, date) => out.push(pad(date.getMilliseconds(), 3)),
+  Y: (out, date) => timeFormatHelper(out, 'ynd', date),
+  D: (out, date) => timeFormatHelper(out, 'y-n-d', date),
+  T: (out, date) => timeFormatHelper(out, 'h:m:s', date),
+  A: (out, date) => timeFormatHelper(out, 'D T.M', date),
+};
+
+function timeFormatHelper(out: string[], format: string, date: Date) {
+  for (let i = 0, len = format.length; i < len; i++) {
+    const c = format[i];
+    const helper = timeFormatKeys[c];
+    if (helper) {
+      helper(out, date);
+    } else {
+      out.push(c);
+    }
+  }
+}
+
+export function timeFormat(format: string, date: Date): string {
+  const out: string[] = [];
+  timeFormatHelper(out, format, date);
+  return out.join("");
+}

app/common/tpromisified.ts

@@ -0,0 +1,25 @@
+// tslint:disable:max-line-length
+// credits: https://stackoverflow.com/questions/49998665/promisified-function-type
+
+// Generic Function definition
+type AnyFunction = (...args: any[]) => any;
+
+// Extracts the type if wrapped by a Promise
+type Unpacked<T> = T extends Promise<infer U> ? U : T;
+
+type PromisifiedFunction<T extends AnyFunction> =
+  T extends () => infer U ? () => Promise<Unpacked<U>> :
+  T extends (a1: infer A1) => infer U ? (a1: A1) => Promise<Unpacked<U>> :
+  T extends (a1: infer A1, a2: infer A2) => infer U ? (a1: A1, a2: A2) => Promise<Unpacked<U>> :
+  T extends (a1: infer A1, a2: infer A2, a3: infer A3) => infer U ? (a1: A1, a2: A2, a3: A3) => Promise<Unpacked<U>> :
+  T extends (a1: infer A1, a2: infer A2, a3: infer A3, a4: infer A4) => infer U ? (a1: A1, a2: A2, a3: A3, a4: A4) => Promise<Unpacked<U>> :
+  // ...
+  T extends (...args: any[]) => infer U ? (...args: any[]) => Promise<Unpacked<U>> : T;
+
+/**
+ * `Promisified<T>` has the same methods as `T` but they all return promises. This is useful when
+ * creating a stub with `grain-rpc` for an api which is synchronous.
+ */
+export type Promisified<T> = {
+  [K in keyof T]: T[K] extends AnyFunction ? PromisifiedFunction<T[K]> : never
+};

app/common/tsconfig.json

@@ -1,3 +1,6 @@
 {
   "extends": "../../buildtools/tsconfig-base.json",
+  "references": [
+    { "path": "../plugin" }
+  ]
 }

app/common/tsvFormat.ts

@@ -0,0 +1,58 @@
+/**
+ * Given a 2D array of strings, encodes them in tab-separated format.
+ * Certain values are quoted; when quoted, internal quotes get doubled. The behavior attempts to
+ * match Excel's tsv encoding and parsing when using copy-paste.
+ */
+export function tsvEncode(data: any[][]): string {
+  return data.map(row => row.map(value => encode(value)).join("\t")).join("\n");
+}
+
+function encode(rawValue: any): string {
+  // For encoding-decoding symmetry, we should also encode any values that start with '"',
+  // but neither Excel nor Google Sheets do it. They both decode such values to something
+  // different than what produced them (e.g. `"foo""bar"` is encoded into `"foo""bar"`, and
+  // that is decoded into `foo"bar`).
+  const value: string = typeof rawValue === 'string' ? rawValue :
+    (rawValue == null ? "" : String(rawValue));
+  if (value.includes("\t") || value.includes("\n")) {
+    return '"' + value.replace(/"/g, '""') + '"';
+  }
+  return value;
+}
+
+/**
+ * Given a tab-separated string, decodes it and returns a 2D array of strings.
+ * TODO: This does not yet deal with Windows line endings (\r or \r\n).
+ */
+export function tsvDecode(tsvString: string): string[][] {
+  const lines: string[][] = [];
+  let row: string[] = [];
+
+  // This is a complex regexp but it does the job of a lot of parsing code. Here are the parts:
+  //  A: [^\t\n]*         Sequence of character that does not require the field to get quoted.
+  //  B: ([^"]*"")*[^"]*  Sequence of characters containing all double-quotes in pairs (i.e. `""`)
+  //  C: "B"(?!")         Quoted sequence, with all double-quotes inside paired up, and ending in a single quote.
+  //  D: C?A              A value for one field, a relaxation of C|A (to cope with not-quite expected data)
+  //  E: D(\t|\n|$)       Field value with field, line, or file terminator.
+  const fieldRegexp = /(("([^"]*"")*[^"]*"(?!"))?[^\t\n]*)(\t|\n|$)/g;
+  for (;;) {
+    const m = fieldRegexp.exec(tsvString);
+    if (!m) { break; }
+    const sep = m[4];
+    let value = m[1];
+    if (value.startsWith('"')) {
+      // It's a quoted value, so doubled-up quotes should became individual quotes, and individual
+      // quotes should be removed.
+      value = value.replace(/"([^"]*"")*[^"]*"(?!")/, q => q.slice(1, -1).replace(/""/g, '"'));
+    }
+    row.push(value);
+    if (sep !== '\t') {
+      lines.push(row);
+      row = [];
+      if (sep === '') {
+        break;
+      }
+    }
+  }
+  return lines;
+}

app/common/uploads.ts

@@ -0,0 +1,44 @@
+/**
+ * Code and declarations shared by browser and server-side code for handling uploads.
+ *
+ * Browser code has several functions available in app/client/lib/uploads.ts which return an
+ * UploadResult that represents an upload. An upload may contain multiple files.
+ *
+ * An upload is identified by a numeric uploadId which is unique within an UploadSet. An UploadSet
+ * is collection of uploads tied to a browser session (as maintained by app/server/lib/Client).
+ * When the session ends, all uploads are cleaned up.
+ *
+ * The uploadId is useful to identify the upload to the server, which can then consume the actual
+ * files there. It may also be used to clean up the upload once it is no longer needed.
+ *
+ * Files within an upload can be identified by their index in UploadResult.files array. The
+ * origName available for files is not guaranteed to be unique.
+ *
+ * Implementation detail: The upload is usually a temporary directory on the server, but may be a
+ * collection of non-temporary files when files are selected using Electron's native file picker.
+ */
+
+/**
+ * Represents a single upload, containing one or more files. Empty uploads are never created.
+ */
+export interface UploadResult {
+  uploadId: number;
+  files: FileUploadResult[];
+}
+
+/**
+ * Represents a single file within an upload. This is the only information made available to the
+ * browser. (In particular, while the server knows also the actual path of the file on the server,
+ * the browser has no need for it and should not know it.)
+ */
+export interface FileUploadResult {
+  origName: string;     // The filename that the user reports for the file (not guaranteed unique).
+  size: number;         // The size of the file in bytes.
+  ext: string;          // The extension of the file, starting with "."
+}
+
+/**
+ * Path where the server accepts POST requests with uploads.  Don't include a leading / so that
+ * the page's <base> will be respected.
+ */
+export const UPLOAD_URL_PATH = 'uploads';

app/common/urlUtils.ts

@@ -0,0 +1,88 @@
+import {extractOrgParts, GristLoadConfig} from 'app/common/gristUrls';
+
+export function getGristConfig(): GristLoadConfig {
+  return (window as any).gristConfig || {};
+}
+
+/**
+ *
+ * Adds /o/ORG to the supplied path, with ORG extracted from current URL if possible.
+ * If not, path is returned as is, but with any trailing / removed for consistency.
+ *
+ */
+export function addCurrentOrgToPath(path: string, skipIfInDomain: boolean = false) {
+  if (typeof window === 'undefined' || !window) { return path; }
+  return addOrgToPath(path, window.location.href, skipIfInDomain);
+}
+
+/**
+ *
+ * Adds /o/ORG to the supplied path, with ORG extracted from the page URL if possible.
+ * If not, path is returned as is, but with any trailing / removed for consistency.
+ *
+ */
+export function addOrgToPath(path: string, page: string, skipIfInDomain: boolean = false) {
+  if (typeof window === 'undefined' || !window) { return path; }
+  if (path.includes('/o/')) { return path; }
+  const src = new URL(page);
+  const srcParts = extractOrgParts(src.host, src.pathname);
+  if (srcParts.mismatch) {
+    throw new Error('Cannot figure out what organization the URL is for.');
+  }
+  path = path.replace(/\/$/, '');
+  if (!srcParts.subdomain) {
+    return path;
+  }
+  if (skipIfInDomain && srcParts.orgFromHost) {
+    return path;
+  }
+  return `${path}/o/${srcParts.subdomain}`;
+}
+
+/**
+ * Expands an endpoint path to a full url anchored to the given doc worker base url.
+ */
+export function docUrl(docWorkerUrl: string|null|undefined, path?: string) {
+  const base = document.querySelector('base');
+  const baseHref = base && base.href;
+  const baseUrl = new URL(docWorkerUrl || baseHref || window.location.origin);
+  return baseUrl.toString().replace(/\/$/, '') + (path ? `/${path}` : '');
+}
+
+// Get a url on the same webserver as the current page, adding a prefix to encode
+// the current organization if necessary.
+export function getOriginUrl(path: string) {
+  return `${window.location.origin}${addCurrentOrgToPath('/', true)}${path}`;
+}
+
+// Return a string docId if server has provided one (as in hosted Grist), otherwise null
+// (as in classic Grist).
+export function getInitialDocAssignment(): string|null {
+  return getGristConfig().assignmentId || null;
+}
+
+// Return true if we are on a page that can send metrics.
+// TODO: all pages should send suitable metrics.
+export function pageHasMetrics(): boolean {
+  // No metric support on hosted grist.
+  return !getGristConfig().homeUrl;
+}
+
+// Return true if we are on a page that can supply a doc list.
+// TODO: the doclist object isn't relevant to hosted grist and should be factored out.
+export function pageHasDocList(): boolean {
+  // No doc list support on hosted grist.
+  return !getGristConfig().homeUrl;
+}
+
+// Return true if we are on a page that has access to home api.
+export function pageHasHome(): boolean {
+  return Boolean(getGristConfig().homeUrl);
+}
+
+// Construct a url by adding `path` to the home url (adding in the part to the current
+// org if needed), and fetch from it.
+export function fetchFromHome(path: string, opts: RequestInit): Promise<Response> {
+  const baseUrl = addCurrentOrgToPath(getGristConfig().homeUrl!);
+  return window.fetch(`${baseUrl}${path}`, opts);
+}

app/common/version.ts

@@ -0,0 +1,3 @@
+export const version = "1.0.2-dev";
+export const channel = "devtest";
+export const gitcommit = "e33c4e5aeM";

app/gen-server/ApiServer.ts

@@ -0,0 +1,475 @@
+import * as crypto from 'crypto';
+import * as express from 'express';
+import {EntityManager} from 'typeorm';
+
+import {ApiError} from 'app/common/ApiError';
+import {FullUser} from 'app/common/LoginSessionAPI';
+import {OrganizationProperties} from 'app/common/UserAPI';
+import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
+import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession';
+import {expressWrap} from 'app/server/lib/expressWrap';
+import {RequestWithOrg} from 'app/server/lib/extractOrg';
+import * as log from 'app/server/lib/log';
+import {getDocScope, getScope, integerParam, isParameterOn, sendOkReply,
+        sendReply, stringParam} from 'app/server/lib/requestUtils';
+import {Request} from 'express';
+
+import {User} from './entity/User';
+import {HomeDBManager} from './lib/HomeDBManager';
+
+// exposed for testing purposes
+export const Deps = {
+  apiKeyGenerator: () => crypto.randomBytes(20).toString('hex')
+};
+
+// Fetch the org this request was made for, or null if it isn't tied to a particular org.
+// Early middleware should have put the org in the request object for us.
+export function getOrgFromRequest(req: Request): string|null {
+  return (req as RequestWithOrg).org || null;
+}
+
+/**
+ * Compute the signature of the user's email address using HelpScout's secret key, to prove to
+ * HelpScout the user identity for identifying customer information and conversation history.
+ */
+function helpScoutSign(email: string): string|undefined {
+  const secretKey = process.env.HELP_SCOUT_SECRET_KEY;
+  if (!secretKey) { return undefined; }
+  return crypto.createHmac('sha256', secretKey).update(email).digest('hex');
+}
+
+/**
+ * Fetch an identifier for an organization from the "oid" parameter of the request.
+ *   - Integers are accepted, and will be compared with values in orgs.id column
+ *   - Strings are accepted, and will be compared with values in orgs.domain column
+ *     (or, if they match the pattern docs-NNNN, will check orgs.owner_id)
+ *   - The special string "current" is replaced with the current org domain embedded
+ *     in the url
+ *   - If there is no identifier available, a 400 error is thrown.
+ */
+export function getOrgKey(req: Request): string|number {
+  let orgKey: string|null = stringParam(req.params.oid);
+  if (orgKey === 'current') {
+    orgKey = getOrgFromRequest(req);
+  }
+  if (!orgKey) {
+    throw new ApiError("No organization chosen", 400);
+  } else if (/^\d+$/.test(orgKey)) {
+    return parseInt(orgKey, 10);
+  }
+  return orgKey;
+}
+
+// Adds an non-personal org with a new billingAccout, with the given name and domain.
+// Returns a QueryResult with the orgId on success.
+export function addOrg(
+  dbManager: HomeDBManager,
+  userId: number,
+  props: Partial<OrganizationProperties>,
+): Promise<number> {
+  return dbManager.connection.transaction(async manager => {
+    const user = await manager.findOne(User, userId);
+    if (!user) { return handleDeletedUser(); }
+    const query = await dbManager.addOrg(user, props, false, true, manager);
+    if (query.status !== 200) { throw new ApiError(query.errMessage!, query.status); }
+    return query.data!;
+  });
+}
+
+/**
+ * Provides a REST API for the landing page, which returns user's workspaces, organizations and documents.
+ * Temporarily sqlite database is used. Later it will be changed to RDS Aurora or PostgreSQL.
+ */
+export class ApiServer {
+  /**
+   * Add API endpoints to the specified connection. An error handler is added to /api to make sure
+   * all error responses have a body in json format.
+   *
+   * Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside
+   * to apply to these routes, and trustOrigin too for cross-domain requests.
+   */
+  constructor(
+    private _app: express.Application,
+    private _dbManager: HomeDBManager,
+  ) {
+    this._addEndpoints();
+  }
+
+  private _addEndpoints(): void {
+    // GET /api/orgs
+    // Get all organizations user may have some access to.
+    this._app.get('/api/orgs', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      const domain = getOrgFromRequest(req);
+      const merged = Boolean(req.query.merged);
+      const query = merged ?
+        await this._dbManager.getMergedOrgs(userId, userId, domain) :
+        await this._dbManager.getOrgs(userId, domain);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/workspace/:wid
+    // Get workspace by id, returning nested documents that user has access to.
+    this._app.get('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.getWorkspace(getScope(req), wsId);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid
+    // Get organization by id
+    this._app.get('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrg(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid/workspaces
+    // Get all workspaces and nested documents of organization that user has access to.
+    this._app.get('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrgWorkspaces(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/orgs
+    // Body params: name (required), domain
+    // Create a new org.
+    this._app.post('/api/orgs', expressWrap(async (req, res) => {
+      // Don't let anonymous users end up owning organizations, it will be confusing.
+      // Maybe if the user has presented credentials this would be ok - but addOrg
+      // doesn't have access to that information yet, so punting on this.
+      // TODO: figure out who should be allowed to create organizations
+      const userId = getAuthorizedUserId(req);
+      const orgId = await addOrg(this._dbManager, userId, req.body);
+      return sendOkReply(req, res, orgId);
+    }));
+
+    // PATCH /api/orgs/:oid
+    // Body params: name, domain
+    // Update the specified org.
+    this._app.patch('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.updateOrg(getScope(req), org, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // // DELETE /api/orgs/:oid
+    // Delete the specified org and all included workspaces and docs.
+    this._app.delete('/api/orgs/:oid', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.deleteOrg(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/orgs/:oid/workspaces
+    // Body params: name
+    // Create a new workspace owned by the specific organization.
+    this._app.post('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.addWorkspace(getScope(req), org, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/workspaces/:wid
+    // Body params: name
+    // Update the specified workspace.
+    this._app.patch('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.updateWorkspace(getScope(req), wsId, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // // DELETE /api/workspaces/:wid
+    // Delete the specified workspace and all included docs.
+    this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/workspaces/:wid/remove
+    // Soft-delete the specified workspace.  If query parameter "permanent" is set,
+    // delete permanently.
+    this._app.post('/api/workspaces/:wid/remove', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      if (isParameterOn(req.query.permanent)) {
+        const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
+        return sendReply(req, res, query);
+      } else {
+        await this._dbManager.softDeleteWorkspace(getScope(req), wsId);
+        return sendOkReply(req, res);
+      }
+    }));
+
+    // POST /api/workspaces/:wid/unremove
+    // Recover the specified workspace if it was previously soft-deleted and is
+    // still available.
+    this._app.post('/api/workspaces/:wid/unremove', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      await this._dbManager.undeleteWorkspace(getScope(req), wsId);
+      return sendOkReply(req, res);
+    }));
+
+    // POST /api/workspaces/:wid/docs
+    // Create a new doc owned by the specific workspace.
+    this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => {
+      const wsId = integerParam(req.params.wid);
+      const query = await this._dbManager.addDocument(getScope(req), wsId, req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did
+    // Update the specified doc.
+    this._app.patch('/api/docs/:did', expressWrap(async (req, res) => {
+      const query = await this._dbManager.updateDocument(getDocScope(req), req.body);
+      return sendReply(req, res, query);
+    }));
+
+    // POST /api/docs/:did/unremove
+    // Recover the specified doc if it was previously soft-deleted and is
+    // still available.
+    this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => {
+      await this._dbManager.undeleteDocument(getDocScope(req));
+      return sendOkReply(req, res);
+    }));
+
+    // PATCH /api/orgs/:oid/access
+    // Update the specified org acl rules.
+    this._app.patch('/api/orgs/:oid/access', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateOrgPermissions(getScope(req), org, delta);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/workspaces/:wid/access
+    // Update the specified workspace acl rules.
+    this._app.patch('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
+      const workspaceId = integerParam(req.params.wid);
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateWorkspacePermissions(getScope(req), workspaceId, delta);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/docs/:did
+    // Get information about a document.
+    this._app.get('/api/docs/:did', expressWrap(async (req, res) => {
+      const query = await this._dbManager.getDoc(getDocScope(req));
+      return sendOkReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did/access
+    // Update the specified doc acl rules.
+    this._app.patch('/api/docs/:did/access', expressWrap(async (req, res) => {
+      const delta = req.body.delta;
+      const query = await this._dbManager.updateDocPermissions(getDocScope(req), delta);
+      return sendReply(req, res, query);
+    }));
+
+    // PATCH /api/docs/:did/move
+    // Move the doc to the workspace specified in the body.
+    this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => {
+      const workspaceId = req.body.workspace;
+      const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId);
+      return sendReply(req, res, query);
+    }));
+
+    this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => {
+      const query = await this._dbManager.pinDoc(getDocScope(req), true);
+      return sendReply(req, res, query);
+    }));
+
+    this._app.patch('/api/docs/:did/unpin', expressWrap(async (req, res) => {
+      const query = await this._dbManager.pinDoc(getDocScope(req), false);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/orgs/:oid/access
+    // Get user access information regarding an org
+    this._app.get('/api/orgs/:oid/access', expressWrap(async (req, res) => {
+      const org = getOrgKey(req);
+      const query = await this._dbManager.getOrgAccess(getScope(req), org);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/workspaces/:wid/access
+    // Get user access information regarding a workspace
+    this._app.get('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
+      const workspaceId = integerParam(req.params.wid);
+      const query = await this._dbManager.getWorkspaceAccess(getScope(req), workspaceId);
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/docs/:did/access
+    // Get user access information regarding a doc
+    this._app.get('/api/docs/:did/access', expressWrap(async (req, res) => {
+      const query = await this._dbManager.getDocAccess(getDocScope(req));
+      return sendReply(req, res, query);
+    }));
+
+    // GET /api/profile/user
+    // Get user's profile
+    this._app.get('/api/profile/user', expressWrap(async (req, res) => {
+      const fullUser = await this._getFullUser(req);
+      return sendOkReply(req, res, fullUser);
+    }));
+
+    // POST /api/profile/user/name
+    // Body params: string
+    // Update users profile.
+    this._app.post('/api/profile/user/name', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      if (!(req.body && req.body.name)) {
+        throw new ApiError('Name expected in the body', 400);
+      }
+      const name = req.body.name;
+      await this._dbManager.updateUserName(userId, name);
+      res.sendStatus(200);
+    }));
+
+    // GET /api/profile/apikey
+    // Get user's apiKey
+    this._app.get('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getUserId(req);
+      const user = await User.findOne(userId);
+      if (user) {
+        // The null value is of no interest to the user, let's show empty string instead.
+        res.send(user.apiKey || '');
+        return;
+      }
+      handleDeletedUser();
+    }));
+
+    // POST /api/profile/apikey
+    // Update user's apiKey
+    this._app.post('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getAuthorizedUserId(req);
+      const force = req.body ? req.body.force : false;
+      const manager = this._dbManager.connection.manager;
+      let user = await manager.findOne(User, userId);
+      if (!user) { return handleDeletedUser(); }
+      if (!user.apiKey || force) {
+        user = await updateApiKeyWithRetry(manager, user);
+        res.status(200).send(user.apiKey);
+      } else {
+        res.status(400).send({error: "An apikey is already set, use `{force: true}` to override it."});
+      }
+    }));
+
+    // DELETE /api/profile/apiKey
+    // Delete apiKey
+    this._app.delete('/api/profile/apikey', expressWrap(async (req, res) => {
+      const userId = getAuthorizedUserId(req);
+      await this._dbManager.connection.transaction(async manager => {
+        const user = await manager.findOne(User, userId);
+        if (!user) {return handleDeletedUser(); }
+        user.apiKey = null;
+        await manager.save(User, user);
+      });
+      res.sendStatus(200);
+    }));
+
+    // GET /api/session/access/active
+    // Returns active user and active org (if any)
+    this._app.get('/api/session/access/active', expressWrap(async (req, res) => {
+      const fullUser = await this._getFullUser(req);
+      const domain = getOrgFromRequest(req);
+      const org = domain ? (await this._dbManager.getOrg(getScope(req), domain || null)) : null;
+      const orgError = (org && org.errMessage) ? {error: org.errMessage, status: org.status} : undefined;
+      return sendOkReply(req, res, {
+        user: {...fullUser, helpScoutSignature: helpScoutSign(fullUser.email)},
+        org: (org && org.data) || null,
+        orgError
+      });
+    }));
+
+    // POST /api/session/access/active
+    // Body params: email (required)
+    // Sets active user for active org
+    this._app.post('/api/session/access/active', expressWrap(async (req, res) => {
+      const mreq = req as RequestWithLogin;
+      const domain = getOrgFromRequest(mreq);
+      const email = req.body.email;
+      if (!email) { throw new ApiError('email required', 400); }
+      try {
+        // Modify session copy in request. Will be saved to persistent storage before responding
+        // by express-session middleware.
+        linkOrgWithEmail(mreq.session, req.body.email, domain || '');
+        return sendOkReply(req, res, {email});
+      } catch (e) {
+        throw new ApiError('email not available', 403);
+      }
+    }));
+
+    // GET /api/session/access/all
+    // Returns all user profiles (with ids) and all orgs they can access.
+    // Flattens personal orgs into a single org.
+    this._app.get('/api/session/access/all', expressWrap(async (req, res) => {
+      const domain = getOrgFromRequest(req);
+      const users = getUserProfiles(req);
+      const userId = getUserId(req);
+      const orgs = await this._dbManager.getMergedOrgs(userId, users, domain);
+      if (orgs.errMessage) { throw new ApiError(orgs.errMessage, orgs.status); }
+      return sendOkReply(req, res, {
+        users: await this._dbManager.completeProfiles(users),
+        orgs: orgs.data
+      });
+    }));
+
+    // DELETE /users/:uid
+    // Delete the specified user, their personal organization, removing them from all groups.
+    // Not available to the anonymous user.
+    // TODO: should orphan orgs, inaccessible by anyone else, get deleted when last user
+    // leaves?
+    this._app.delete('/api/users/:uid', expressWrap(async (req, res) => {
+      const userIdToDelete = parseInt(req.params.uid, 10);
+      if (!(req.body && req.body.name !== undefined)) {
+        throw new ApiError('to confirm deletion of a user, provide their name', 400);
+      }
+      const query = await this._dbManager.deleteUser(getScope(req), userIdToDelete, req.body.name);
+      return sendReply(req, res, query);
+    }));
+  }
+
+  private async _getFullUser(req: Request): Promise<FullUser> {
+    const mreq = req as RequestWithLogin;
+    const userId = getUserId(mreq);
+    const fullUser = await this._dbManager.getFullUser(userId);
+    const domain = getOrgFromRequest(mreq);
+    const sessionUser = getSessionUser(mreq.session, domain || '');
+    const loginMethod = sessionUser && sessionUser.profile ? sessionUser.profile.loginMethod : undefined;
+    return {...fullUser, loginMethod};
+  }
+}
+
+/**
+ * Throw the error for when a user has been deleted since point of call (very unlikely to happen).
+ */
+function handleDeletedUser(): never {
+  throw new ApiError("user not known", 401);
+}
+
+/**
+ * Helper to update a user's apiKey. Update might fail because of the DB uniqueness constraint on
+ * the apiKey (although it is very unlikely according to `crypto`), we retry until success. Fails
+ * after 5 unsuccessful attempts.
+ */
+async function updateApiKeyWithRetry(manager: EntityManager, user: User): Promise<User> {
+  const currentKey = user.apiKey;
+  for (let i = 0; i < 5; ++i) {
+    user.apiKey = Deps.apiKeyGenerator();
+    try {
+      // if new key is the same as the current, the db update won't fail so we check it here (very
+      // unlikely to happen but but still better to handle)
+      if (user.apiKey === currentKey) {
+        throw new Error('the new key is the same as the current key');
+      }
+      return await manager.save(User, user);
+    } catch (e) {
+      // swallow and retry
+      log.warn(`updateApiKeyWithRetry: failed attempt ${i}/5, %s`, e);
+    }
+  }
+  throw new Error('Could not generate a valid api key.');
+}

app/gen-server/entity/AclRule.ts

@@ -0,0 +1,58 @@
+import {BaseEntity, ChildEntity, Column, Entity, JoinColumn, ManyToOne, OneToOne,
+        PrimaryGeneratedColumn, RelationId, TableInheritance} from "typeorm";
+
+import {Document} from "./Document";
+import {Group} from "./Group";
+import {Organization} from "./Organization";
+import {Workspace} from "./Workspace";
+
+@Entity('acl_rules')
+@TableInheritance({ column: { type: "int", name: "type" } })
+export class AclRule extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public permissions: number;
+
+  @OneToOne(type => Group, group => group.aclRule)
+  @JoinColumn({name: "group_id"})
+  public group: Group;
+}
+
+
+@ChildEntity()
+export class AclRuleWs extends AclRule {
+
+  @ManyToOne(type => Workspace, workspace => workspace.aclRules)
+  @JoinColumn({name: "workspace_id"})
+  public workspace: Workspace;
+
+  @RelationId((aclRule: AclRuleWs) => aclRule.workspace)
+  public workspaceId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleOrg extends AclRule {
+
+  @ManyToOne(type => Organization, organization => organization.aclRules)
+  @JoinColumn({name: "org_id"})
+  public organization: Organization;
+
+  @RelationId((aclRule: AclRuleOrg) => aclRule.organization)
+  public orgId: number;
+}
+
+
+@ChildEntity()
+export class AclRuleDoc extends AclRule {
+
+  @ManyToOne(type => Document, document => document.aclRules)
+  @JoinColumn({name: "doc_id"})
+  public document: Document;
+
+  @RelationId((aclRule: AclRuleDoc) => aclRule.document)
+  public docId: number;
+}

app/gen-server/entity/Alias.ts

@@ -0,0 +1,27 @@
+import {BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne,
+        PrimaryColumn} from 'typeorm';
+import {Document} from './Document';
+import {Organization} from './Organization';
+
+@Entity({name: 'aliases'})
+export class Alias extends BaseEntity {
+  @PrimaryColumn({name: 'org_id'})
+  public orgId: number;
+
+  @PrimaryColumn({name: 'url_id'})
+  public urlId: string;
+
+  @Column({name: 'doc_id'})
+  public docId: string;
+
+  @ManyToOne(type => Document)
+  @JoinColumn({name: 'doc_id'})
+  public doc: Document;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @CreateDateColumn({name: 'created_at'})
+  public createdAt: Date;
+}

app/gen-server/entity/BillingAccount.ts

@@ -0,0 +1,65 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Product} from 'app/gen-server/entity/Product';
+import {nativeValues} from 'app/gen-server/lib/values';
+
+// This type is for billing account status information.  Intended for stuff
+// like "free trial running out in N days".
+interface BillingAccountStatus {
+  stripeStatus?: string;
+  currentPeriodEnd?: Date;
+  message?: string;
+}
+
+/**
+ * This relates organizations to products.  It holds any stripe information
+ * needed to be able to update and pay for the product that applies to the
+ * organization.  It has a list of managers detailing which users have the
+ * right to view and edit these settings.
+ */
+@Entity({name: 'billing_accounts'})
+export class BillingAccount extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Product)
+  @JoinColumn({name: 'product_id'})
+  public product: Product;
+
+  @Column()
+  public individual: boolean;
+
+  // A flag for when all is well with the user's subscription.
+  // Probably shouldn't use this to drive whether service is provided or not.
+  // Strip recommends updating an end-of-service datetime every time payment
+  // is received, adding on a grace period of some days.
+  @Column({name: 'in_good_standing', default: nativeValues.trueValue})
+  public inGoodStanding: boolean;
+
+  @Column({type: nativeValues.jsonEntityType, nullable: true})
+  public status: BillingAccountStatus;
+
+  @Column({name: 'stripe_customer_id', type: String, nullable: true})
+  public stripeCustomerId: string | null;
+
+  @Column({name: 'stripe_subscription_id', type: String, nullable: true})
+  public stripeSubscriptionId: string | null;
+
+  @Column({name: 'stripe_plan_id', type: String, nullable: true})
+  public stripePlanId: string | null;
+
+  @OneToMany(type => BillingAccountManager, manager => manager.billingAccount)
+  public managers: BillingAccountManager[];
+
+  @OneToMany(type => Organization, org => org.billingAccount)
+  public orgs: Organization[];
+
+  // A calculated column that is true if it looks like there is a paid plan.
+  @Column({name: 'paid', type: 'boolean', insert: false, select: false})
+  public paid?: boolean;
+
+  // A calculated column summarizing whether active user is a manager of the billing account.
+  // (No @Column needed since calculation is done in javascript not sql)
+  public isManager?: boolean;
+}

app/gen-server/entity/BillingAccountManager.ts

@@ -0,0 +1,26 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {User} from 'app/gen-server/entity/User';
+
+/**
+ * A list of users with the right to modify a giving billing account.
+ */
+@Entity({name: 'billing_account_managers'})
+export class BillingAccountManager extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User, { onDelete: 'CASCADE' })
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Document.ts

@@ -0,0 +1,69 @@
+import {ApiError} from 'app/common/ApiError';
+import {Role} from 'app/common/roles';
+import {DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
+import {AclRuleDoc} from "./AclRule";
+import {Alias} from "./Alias";
+import {Resource} from "./Resource";
+import {Workspace} from "./Workspace";
+
+// Acceptable ids for use in document urls.
+const urlIdRegex = /^[-a-z0-9]+$/i;
+
+function isValidUrlId(urlId: string) {
+  if (urlId === NEW_DOCUMENT_CODE) { return false; }
+  return urlIdRegex.exec(urlId);
+}
+
+@Entity({name: 'docs'})
+export class Document extends Resource {
+
+  @PrimaryColumn()
+  public id: string;
+
+  @ManyToOne(type => Workspace)
+  @JoinColumn({name: 'workspace_id'})
+  public workspace: Workspace;
+
+  @OneToMany(type => AclRuleDoc, aclRule => aclRule.document)
+  public aclRules: AclRuleDoc[];
+
+  // Indicates whether the doc is pinned to the org it lives in.
+  @Column({name: 'is_pinned', default: false})
+  public isPinned: boolean;
+
+  // Property that may be returned when the doc is fetched to indicate the access the
+  // fetching user has on the doc, i.e. 'owners', 'editors', 'viewers'
+  public access: Role|null;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating documents.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false, update: false})
+  public permissions?: any;
+
+  @Column({name: 'url_id', type: 'text', nullable: true})
+  public urlId: string|null;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  @OneToMany(type => Alias, alias => alias.doc)
+  public aliases: Alias[];
+
+  public checkProperties(props: any): props is Partial<DocumentProperties> {
+    return super.checkProperties(props, documentPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<DocumentProperties>) {
+    super.updateFromProperties(props);
+    if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
+    if (props.urlId !== undefined) {
+      if (props.urlId !== null && !isValidUrlId(props.urlId)) {
+        throw new ApiError('invalid urlId', 400);
+      }
+      this.urlId = props.urlId;
+    }
+  }
+}

app/gen-server/entity/Group.ts

@@ -0,0 +1,33 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToOne, PrimaryGeneratedColumn} from "typeorm";
+
+import {AclRule} from "./AclRule";
+import {User} from "./User";
+
+@Entity({name: 'groups'})
+export class Group extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @ManyToMany(type => User)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'user_id'}
+  })
+  public memberUsers: User[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_groups',
+    joinColumn: {name: 'group_id'},
+    inverseJoinColumn: {name: 'subgroup_id'}
+  })
+  public memberGroups: Group[];
+
+  @OneToOne(type => AclRule, aclRule => aclRule.group)
+  public aclRule: AclRule;
+}

app/gen-server/entity/Login.ts

@@ -0,0 +1,25 @@
+import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn} from "typeorm";
+
+import {User} from "./User";
+
+@Entity({name: 'logins'})
+export class Login extends BaseEntity {
+
+  @PrimaryColumn()
+  public id: number;
+
+  // This is the normalized email address we use for equality and indexing.
+  @Column()
+  public email: string;
+
+  // This is how the user's email address should be displayed.
+  @Column({name: 'display_email'})
+  public displayEmail: string;
+
+  @Column({name: 'user_id'})
+  public userId: number;
+
+  @ManyToOne(type => User)
+  @JoinColumn({name: 'user_id'})
+  public user: User;
+}

app/gen-server/entity/Organization.ts

@@ -0,0 +1,79 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, OneToOne,
+        PrimaryGeneratedColumn, RelationId} from "typeorm";
+import {Role} from "app/common/roles";
+import {OrganizationProperties, organizationPropertyKeys} from "app/common/UserAPI";
+import {AclRuleOrg} from "./AclRule";
+import {BillingAccount} from "./BillingAccount";
+import {Resource} from "./Resource";
+import {User} from "./User";
+import {Workspace} from "./Workspace";
+
+// Information about how an organization may be accessed.
+export interface AccessOption {
+  id: number;     // a user id
+  email: string;  // a user email
+  name: string;   // a user name
+  perms: number;  // permissions the user would have on organization
+}
+
+export interface AccessOptionWithRole extends AccessOption {
+  access: Role;   // summary of permissions
+}
+
+@Entity({name: 'orgs'})
+export class Organization extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column({
+    nullable: true
+  })
+  public domain: string;
+
+  @OneToOne(type => User, user => user.personalOrg)
+  @JoinColumn({name: 'owner_id'})
+  public owner: User;
+
+  @RelationId((org: Organization) => org.owner)
+  public ownerId: number;
+
+  @OneToMany(type => Workspace, workspace => workspace.org)
+  public workspaces: Workspace[];
+
+  @OneToMany(type => AclRuleOrg, aclRule => aclRule.organization)
+  public aclRules: AclRuleOrg[];
+
+  @Column({name: 'billing_account_id'})
+  public billingAccountId: number;
+
+  @ManyToOne(type => BillingAccount)
+  @JoinColumn({name: 'billing_account_id'})
+  public billingAccount: BillingAccount;
+
+  // Property that may be returned when the org is fetched to indicate the access the
+  // fetching user has on the org, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // Property that may be used internally to track multiple ways an org can be accessed
+  public accessOptions?: AccessOptionWithRole[];
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating organizations.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  // For custom domains, this is the preferred host associated with this org/team.
+  @Column({name: 'host', type: 'text', nullable: true})
+  public host: string|null;
+
+  public checkProperties(props: any): props is Partial<OrganizationProperties> {
+    return super.checkProperties(props, organizationPropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<OrganizationProperties>) {
+    super.updateFromProperties(props);
+    if (props.domain) { this.domain = props.domain; }
+  }
+}

app/gen-server/entity/Product.ts

@@ -0,0 +1,176 @@
+import {Features} from 'app/common/Features';
+import {nativeValues} from 'app/gen-server/lib/values';
+import * as assert from 'assert';
+import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm';
+
+/**
+ * A summary of features used in 'starter' plans.
+ */
+export const starterFeatures: Features = {
+  workspaces: true,
+  // no vanity domain
+  maxDocsPerOrg: 10,
+  maxSharesPerDoc: 2,
+  maxWorkspacesPerOrg: 1
+};
+
+/**
+ * A summary of features used in 'team' plans.
+ */
+export const teamFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  maxSharesPerWorkspace: 0,   // all workspace shares need to be org members.
+  maxSharesPerDoc: 2
+};
+
+/**
+ * A summary of features used in unrestricted grandfathered accounts, and also
+ * in some test settings.
+ */
+export const grandfatherFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+};
+
+export const suspendedFeatures: Features = {
+  workspaces: true,
+  vanityDomain: true,
+  readOnlyDocs: true,
+  // clamp down on new docs/workspaces/shares
+  maxDocsPerOrg: 0,
+  maxSharesPerDoc: 0,
+  maxWorkspacesPerOrg: 0,
+};
+
+/**
+ * Basic fields needed for products supported by Grist.
+ */
+export interface IProduct {
+  name: string;
+  features: Features;
+}
+
+/**
+ *
+ * Products are a bundle of enabled features.  Most products in
+ * Grist correspond to products in stripe.  The correspondence is
+ * established by a gristProduct metadata field on stripe plans.
+ *
+ * In addition, there are the following products in Grist that don't
+ * exist in stripe:
+ *   - The product named 'Free'.  This is a product used for organizations
+ *     created prior to the billing system being set up.
+ *   - The product named 'stub'.  This is product assigned to new
+ *     organizations that should not be usable until a paid plan
+ *     is set up for them.
+ *
+ * TODO: change capitalization of name of grandfather product.
+ *
+ */
+const PRODUCTS: IProduct[] = [
+  // This is a product for grandfathered accounts/orgs.
+  {
+    name: 'Free',
+    features: grandfatherFeatures,
+  },
+
+  // This is a product for newly created accounts/orgs.
+  {
+    name: 'stub',
+    features: {},
+  },
+
+  // These are products set up in stripe.
+  {
+    name: 'starter',
+    features: starterFeatures,
+  },
+  {
+    name: 'professional',  // deprecated, can be removed once no longer referred to in stripe.
+    features: teamFeatures,
+  },
+  {
+    name: 'team',
+    features: teamFeatures,
+  },
+
+  // This is a product for a team site that is no longer in good standing, but isn't yet
+  // to be removed / deactivated entirely.
+  {
+    name: 'suspended',
+    features: suspendedFeatures,
+  },
+];
+
+/**
+ * Get names of products for different situations.
+ */
+export function getDefaultProductNames() {
+  return {
+    personal: 'starter',  // Personal site start off on a functional plan.
+    teamInitial: 'stub',  // Team site starts off on a limited plan, requiring subscription.
+    team: 'team',         // Functional team site
+  };
+}
+
+/**
+ * A Grist product.  Corresponds to a set of enabled features and a choice of limits.
+ */
+@Entity({name: 'products'})
+export class Product extends BaseEntity {
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({type: nativeValues.jsonEntityType})
+  public features: Features;
+}
+
+/**
+ * Make sure the products defined for the current stripe setup are
+ * in the database and up to date.  Other products in the database
+ * are untouched.
+ *
+ * If `apply` is set, the products are changed in the db, otherwise
+ * the are left unchanged.  A summary of affected products is returned.
+ */
+export async function synchronizeProducts(connection: Connection, apply: boolean): Promise<string[]> {
+  try {
+    await connection.query('select name, features, stripe_product_id from products limit 1');
+  } catch (e) {
+    // No usable products table, do not try to synchronize.
+    return [];
+  }
+  const changingProducts: string[] = [];
+  await connection.transaction(async transaction => {
+    const desiredProducts = new Map(PRODUCTS.map(p => [p.name, p]));
+    const existingProducts = new Map((await transaction.find(Product))
+                                     .map(p => [p.name, p]));
+    for (const product of desiredProducts.values()) {
+      if (existingProducts.has(product.name)) {
+        const p = existingProducts.get(product.name)!;
+        try {
+          assert.deepStrictEqual(p.features, product.features);
+        } catch (e) {
+          if (apply) {
+            p.features = product.features;
+            await transaction.save(p);
+          }
+          changingProducts.push(p.name);
+        }
+      } else {
+        if (apply) {
+          const p = new Product();
+          p.name = product.name;
+          p.features = product.features;
+          await transaction.save(p);
+        }
+        changingProducts.push(product.name);
+      }
+    }
+  });
+  return changingProducts;
+}

app/gen-server/entity/Resource.ts

@@ -0,0 +1,46 @@
+import {BaseEntity, Column} from "typeorm";
+import {ApiError} from 'app/common/ApiError';
+import {CommonProperties} from "app/common/UserAPI";
+
+export class Resource extends BaseEntity {
+  @Column()
+  public name: string;
+
+  @Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"})
+  public createdAt: Date;
+
+  @Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"})
+  public updatedAt: Date;
+
+  // a computed column which, when present, means the entity should be filtered out
+  // of results.
+  @Column({name: 'filtered_out', type: 'boolean', select: false, insert: false})
+  public filteredOut?: boolean;
+
+  public updateFromProperties(props: Partial<CommonProperties>) {
+    if (props.createdAt) { this.createdAt = _propertyToDate(props.createdAt); }
+    if (props.updatedAt) {
+      this.updatedAt = _propertyToDate(props.updatedAt);
+    } else {
+      this.updatedAt = new Date();
+    }
+    if (props.name) { this.name = props.name; }
+  }
+
+  protected checkProperties(props: any, keys: string[]): props is Partial<CommonProperties> {
+    for (const key of Object.keys(props)) {
+      if (!keys.includes(key)) {
+        throw new ApiError(`unrecognized property ${key}`, 400);
+      }
+    }
+    return true;
+  }
+}
+
+// Ensure iso-string-or-date value is converted to a date.
+function _propertyToDate(d: string|Date): Date {
+  if (typeof(d) === 'string') {
+    return new Date(d);
+  }
+  return d;
+}

app/gen-server/entity/User.ts

@@ -0,0 +1,54 @@
+import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToMany, OneToOne,
+        PrimaryGeneratedColumn} from "typeorm";
+
+import {Group} from "./Group";
+import {Login} from "./Login";
+import {Organization} from "./Organization";
+
+@Entity({name: 'users'})
+export class User extends BaseEntity {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @Column()
+  public name: string;
+
+  @Column({name: 'api_key', type: String, nullable: true})
+  // Found how to make a type nullable in this discussion: https://github.com/typeorm/typeorm/issues/2567
+  // todo: adds constraint for api_key not to equal ''
+  public apiKey: string | null;
+
+  @Column({name: 'picture', type: String, nullable: true})
+  public picture: string | null;
+
+  @Column({name: 'first_login_at', type: Date, nullable: true})
+  public firstLoginAt: Date | null;
+
+  @OneToOne(type => Organization, organization => organization.owner)
+  public personalOrg: Organization;
+
+  @OneToMany(type => Login, login => login.user)
+  public logins: Login[];
+
+  @ManyToMany(type => Group)
+  @JoinTable({
+    name: 'group_users',
+    joinColumn: {name: 'user_id'},
+    inverseJoinColumn: {name: 'group_id'}
+  })
+  public groups: Group[];
+
+  @Column({name: 'is_first_time_user', default: false})
+  public isFirstTimeUser: boolean;
+
+  /**
+   * Get user's email.  Returns undefined if logins has not been joined, or no login
+   * is available
+   */
+  public get loginEmail(): string|undefined {
+    const login = this.logins && this.logins[0];
+    if (!login) { return undefined; }
+    return login.email;
+  }
+}

app/gen-server/entity/Workspace.ts

@@ -0,0 +1,49 @@
+import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from "typeorm";
+import {WorkspaceProperties, workspacePropertyKeys} from "app/common/UserAPI";
+import {nativeValues} from 'app/gen-server/lib/values';
+import {AclRuleWs} from "./AclRule";
+import {Document} from "./Document";
+import {Organization} from "./Organization";
+import {Resource} from "./Resource";
+
+@Entity({name: 'workspaces'})
+export class Workspace extends Resource {
+
+  @PrimaryGeneratedColumn()
+  public id: number;
+
+  @ManyToOne(type => Organization)
+  @JoinColumn({name: 'org_id'})
+  public org: Organization;
+
+  @OneToMany(type => Document, document => document.workspace)
+  public docs: Document[];
+
+  @OneToMany(type => AclRuleWs, aclRule => aclRule.workspace)
+  public aclRules: AclRuleWs[];
+
+  // Property that may be returned when the workspace is fetched to indicate the access the
+  // fetching user has on the workspace, i.e. 'owners', 'editors', 'viewers'
+  public access: string;
+
+  // A computed column that is true if the workspace is a support workspace.
+  @Column({name: 'support', type: 'boolean', insert: false, select: false})
+  public isSupportWorkspace?: boolean;
+
+  // a computed column with permissions.
+  // {insert: false} makes sure typeorm doesn't try to put values into such
+  // a column when creating workspaces.
+  @Column({name: 'permissions', type: 'text', select: false, insert: false})
+  public permissions?: any;
+
+  @Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
+  public removedAt: Date|null;
+
+  public checkProperties(props: any): props is Partial<WorkspaceProperties> {
+    return super.checkProperties(props, workspacePropertyKeys);
+  }
+
+  public updateFromProperties(props: Partial<WorkspaceProperties>) {
+    super.updateFromProperties(props);
+  }
+}

app/gen-server/lib/DocApiForwarder.ts

@@ -0,0 +1,103 @@
+import * as express from "express";
+import fetch, { RequestInit } from 'node-fetch';
+
+import { ApiError } from 'app/common/ApiError';
+import { removeTrailingSlash } from 'app/common/gutil';
+import { HomeDBManager } from "app/gen-server/lib/HomeDBManager";
+import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer';
+import { IDocWorkerMap } from "app/server/lib/DocWorkerMap";
+import { expressWrap } from "app/server/lib/expressWrap";
+import { getAssignmentId } from "app/server/lib/idUtils";
+
+/**
+ * Forwards all /api/docs/:docId/tables requests to the doc worker handling the :docId document. Makes
+ * sure the user has at least view access to the document otherwise rejects the request. For
+ * performance reason we stream the body directly from the request, which requires that no-one reads
+ * the req before, in particular you should register DocApiForwarder before bodyParser.
+ *
+ * Use:
+ *   const home = new ApiServer(false);
+ *   const docApiForwarder = new DocApiForwarder(getDocWorkerMap(), home);
+ *   app.use(docApiForwarder.getMiddleware());
+ *
+ * Note that it expects userId, and jsonErrorHandler middleware to be set up outside
+ * to apply to these routes.
+ */
+export class DocApiForwarder {
+
+  constructor(private _docWorkerMap: IDocWorkerMap, private _dbManager: HomeDBManager) {
+  }
+
+  public addEndpoints(app: express.Application) {
+    // Middleware to forward a request about an existing document that user has access to.
+    // We do not check whether the document has been soft-deleted; that will be checked by
+    // the worker if needed.
+    const withDoc = expressWrap(this._forwardToDocWorker.bind(this, true));
+    // Middleware to forward a request without a pre-existing document (for imports/uploads).
+    const withoutDoc = expressWrap(this._forwardToDocWorker.bind(this, false));
+    app.use('/api/docs/:docId/tables', withDoc);
+    app.use('/api/docs/:docId/force-reload', withDoc);
+    app.use('/api/docs/:docId/remove', withDoc);
+    app.delete('/api/docs/:docId', withDoc);
+    app.use('/api/docs/:docId/download', withDoc);
+    app.use('/api/docs/:docId/apply', withDoc);
+    app.use('/api/docs/:docId/attachments', withDoc);
+    app.use('/api/docs/:docId/snapshots', withDoc);
+    app.use('/api/docs/:docId/replace', withDoc);
+    app.use('/api/docs/:docId/flush', withDoc);
+    app.use('/api/docs/:docId/states', withDoc);
+    app.use('/api/docs/:docId/compare', withDoc);
+    app.use('^/api/docs$', withoutDoc);
+  }
+
+  private async _forwardToDocWorker(withDocId: boolean, req: express.Request, res: express.Response): Promise<void> {
+    let docId: string|null = null;
+    if (withDocId) {
+      const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, req.params.docId);
+      assertAccess('viewers', docAuth, {allowRemoved: true});
+      docId = docAuth.docId;
+    }
+    // Use the docId for worker assignment, rather than req.params.docId, which could be a urlId.
+    const assignmentId = getAssignmentId(this._docWorkerMap, docId === null ? 'import' : docId);
+
+    if (!this._docWorkerMap) {
+      throw new ApiError('no worker map', 404);
+    }
+    const docStatus = await this._docWorkerMap.assignDocWorker(assignmentId);
+
+    // Construct new url by keeping only origin and path prefixes of `docWorker.internalUrl`,
+    // and otherwise reflecting fully the original url (remaining path, and query params).
+    const docWorkerUrl = new URL(docStatus.docWorker.internalUrl);
+    const url = new URL(req.originalUrl, docWorkerUrl.origin);
+    url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
+
+    const headers: {[key: string]: string} = {
+      ...getTransitiveHeaders(req),
+      'Content-Type': req.get('Content-Type') || 'application/json',
+    };
+    for (const key of ['X-Sort', 'X-Limit']) {
+      const hdr = req.get(key);
+      if (hdr) { headers[key] = hdr; }
+    }
+    const options: RequestInit = {
+      method: req.method,
+      headers,
+    };
+    if (['POST', 'PATCH'].includes(req.method)) {
+      // uses `req` as a stream
+      options.body = req;
+    }
+    const docWorkerRes = await fetch(url.href, options);
+    res.status(docWorkerRes.status);
+    for (const key of ['content-type', 'content-disposition', 'cache-control']) {
+      const value = docWorkerRes.headers.get(key);
+      if (value) { res.set(key, value); }
+    }
+    return new Promise<void>((resolve, reject) => {
+      docWorkerRes.body.on('error', reject);
+      res.on('error', reject);
+      res.on('finish', resolve);
+      docWorkerRes.body.pipe(res);
+    });
+  }
+}

app/gen-server/lib/DocWorkerMap.ts

@@ -0,0 +1,440 @@
+import {MapWithTTL} from 'app/common/AsyncCreate';
+import * as version from 'app/common/version';
+import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
+import * as log from 'app/server/lib/log';
+import {checkPermitKey, formatPermitKey, Permit} from 'app/server/lib/Permit';
+import {promisifyAll} from 'bluebird';
+import mapValues = require('lodash/mapValues');
+import {createClient, Multi, RedisClient} from 'redis';
+import * as Redlock from 'redlock';
+import * as uuidv4 from 'uuid/v4';
+
+promisifyAll(RedisClient.prototype);
+promisifyAll(Multi.prototype);
+
+// Max time for which we will hold a lock, by default.  In milliseconds.
+const LOCK_TIMEOUT = 3000;
+
+// How long do checksums stored in redis last.  In milliseconds.
+// Should be long enough to allow S3 to reach consistency with very high probability.
+// Consistency failures shorter than this interval will be detectable, failures longer
+// than this interval will not be detectable.
+const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000;  // 24 hours
+
+// How long do permits stored in redis last, in milliseconds.
+const PERMIT_TTL_MSEC = 1 * 60 * 1000;  // 1 minute
+
+class DummyDocWorkerMap implements IDocWorkerMap {
+  private _worker?: DocWorkerInfo;
+  private _available: boolean = false;
+  private _permits = new MapWithTTL<string, string>(PERMIT_TTL_MSEC);
+  private _elections = new MapWithTTL<string, string>(1);  // default ttl never used
+
+  public async getDocWorker(docId: string) {
+    if (!this._worker) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async assignDocWorker(docId: string) {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    if (!this._worker || !this._available) { throw new Error('no workers'); }
+    if (this._worker.id !== workerId) { throw new Error('worker not known'); }
+    return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
+  }
+
+  public async updateDocStatus(docId: string, checksum: string) {
+    // nothing to do
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    this._worker = info;
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    this._worker = undefined;
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    this._available = available;
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    // nothing to do
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return [];
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    this._permits.set(key, JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit> {
+    const result = this._permits.get(key);
+    return result ? JSON.parse(result) : null;
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    this._permits.delete(key);
+  }
+
+  public async close(): Promise<void> {
+    this._permits.clear();
+    this._elections.clear();
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    if (this._elections.get(name)) { return null; }
+    const key = uuidv4();
+    this._elections.setWithCustomTTL(name, key, durationInMs);
+    return key;
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    if (this._elections.get(name) === electionKey) {
+      this._elections.delete(name);
+    }
+  }
+}
+
+/**
+ * Manage the relationship between document and workers.  Backed by Redis.
+ * Can also assign workers to "groups" for serving particular documents.
+ * Keys used:
+ *   workers - the set of known active workers, identified by workerId
+ *   workers-available - the set of workers available for assignment (a subset of the workers set)
+ *   workers-available-{group} - the set of workers available for a given group
+ *   worker-{workerId} - a hash of contact information for a worker
+ *   worker-{workerId}-docs - a set of docs assigned to a worker, identified by docId
+ *   worker-{workerId}-group - if set, marks the worker as serving a particular group
+ *   doc-${docId} - a hash containing (JSON serialized) DocStatus fields, other than docMD5.
+ *   doc-${docId}-checksum - the docs docMD5, or 'null' if docMD5 is null
+ *   doc-${docId}-group - if set, marks the doc as to be served by workers in a given group
+ *   workers-lock - a lock used when working with the list of workers
+ *   groups - a hash from groupIds (arbitrary strings) to desired number of workers in group
+ *   elections-${deployment} - a hash, from groupId to a (serialized json) list of worker ids
+ *
+ * Assignments of documents to workers can end abruptly at any time.  Clients
+ * should be prepared to retry if a worker is not responding or denies that a document
+ * is assigned to it.
+ *
+ * If the groups key is set, workers assign themselves to groupIds to
+ * fill the counts specified in groups (in order of groupIds), and
+ * once those are exhausted, get assigned to the special group
+ * "default".
+ */
+export class DocWorkerMap implements IDocWorkerMap {
+  private _client: RedisClient;
+  private _redlock: Redlock;
+
+  // Optional deploymentKey argument supplies a key unique to the deployment (this is important
+  // for maintaining groups across redeployments only)
+  constructor(_clients?: RedisClient[], private _deploymentKey?: string, private _options?: {
+    permitMsec?: number
+  }) {
+    this._deploymentKey = this._deploymentKey || version.version;
+    _clients = _clients || [createClient(process.env.REDIS_URL)];
+    this._redlock = new Redlock(_clients);
+    this._client = _clients[0]!;
+  }
+
+  public async addWorker(info: DocWorkerInfo): Promise<void> {
+    log.info(`DocWorkerMap.addWorker ${info.id}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Make a worker-{workerId} key with contact info, then add this worker to available set.
+      await this._client.hmsetAsync(`worker-${info.id}`, info);
+      await this._client.saddAsync('workers', info.id);
+      // Figure out if worker should belong to a group
+      const groups = await this._client.hgetallAsync('groups');
+      if (groups) {
+        const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`) || {};
+        for (const group of Object.keys(groups).sort()) {
+          const count = parseInt(groups[group], 10) || 0;
+          if (count < 1) { continue; }
+          const elected: string[] = JSON.parse(elections[group] || '[]');
+          if (elected.length >= count) { continue; }
+          elected.push(info.id);
+          await this._client.setAsync(`worker-${info.id}-group`, group);
+          await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, JSON.stringify(elected));
+          break;
+        }
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeWorker(workerId: string): Promise<void> {
+    log.info(`DocWorkerMap.removeWorker ${workerId}`);
+    const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
+    try {
+      // Drop out of available set first.
+      await this._client.sremAsync('workers-available', workerId);
+      const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+      // At this point, this worker should no longer be receiving new doc assignments, though
+      // clients may still be directed to the worker.
+
+      // If we were elected for anything, back out.
+      const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`);
+      if (elections) {
+        if (group in elections) {
+          const elected: string[] = JSON.parse(elections[group]);
+          const newElected = elected.filter(worker => worker !== workerId);
+          if (elected.length !== newElected.length) {
+            if (newElected.length > 0) {
+              await this._client.hsetAsync(`elections-${this._deploymentKey}`, group,
+                                           JSON.stringify(newElected));
+            } else {
+              await this._client.hdelAsync(`elections-${this._deploymentKey}`, group);
+              delete elections[group];
+            }
+          }
+          // We're the last one involved in elections - remove the key entirely.
+          if (Object.keys(elected).length === 0) {
+            await this._client.delAsync(`elections-${this._deploymentKey}`);
+          }
+        }
+      }
+
+      // Now, we start removing the assignments.
+      const assignments = await this._client.smembersAsync(`worker-${workerId}-docs`);
+      if (assignments) {
+        const op = this._client.multi();
+        for (const doc of assignments) { op.del(`doc-${doc}`); }
+        await op.execAsync();
+      }
+
+      // Now remove worker-{workerId}* keys.
+      await this._client.delAsync(`worker-${workerId}-docs`);
+      await this._client.delAsync(`worker-${workerId}-group`);
+      await this._client.delAsync(`worker-${workerId}`);
+
+      // Forget about this worker completely.
+      await this._client.sremAsync('workers', workerId);
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
+    log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
+    const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
+    if (available) {
+      await this._client.saddAsync(`workers-available-${group}`, workerId);
+      await this._client.saddAsync('workers-available', workerId);
+    } else {
+      await this._client.sremAsync('workers-available', workerId);
+      await this._client.sremAsync(`workers-available-${group}`, workerId);
+    }
+  }
+
+  public async releaseAssignment(workerId: string, docId: string): Promise<void> {
+    const op = this._client.multi();
+    op.del(`doc-${docId}`);
+    op.srem(`worker-${workerId}-docs`, docId);
+    await op.execAsync();
+  }
+
+  public async getAssignments(workerId: string): Promise<string[]> {
+    return this._client.smembersAsync(`worker-${workerId}-docs`);
+  }
+
+  /**
+   * Defined by IDocWorkerMap.
+   *
+   * Looks up which DocWorker is responsible for this docId.
+   * Responsibility could change at any time after this call, so it
+   * should be treated as a hint, and clients should be prepared to be
+   * refused and need to retry.
+   */
+  public async getDocWorker(docId: string): Promise<DocStatus|null> {
+    // Fetch the various elements that go into making a DocStatus
+    const props = await this._client.multi()
+      .hgetall(`doc-${docId}`)
+      .get(`doc-${docId}-checksum`)
+      .execAsync() as [{[key: string]: any}|null, string|null]|null;
+    if (!props) { return null; }
+
+    // If there is no worker, return null.  An alternative would be to modify
+    // DocStatus so that it is possible for it to not have a worker assignment.
+    if (!props[0]) { return null; }
+
+    // Fields are JSON encoded since redis cannot store them directly.
+    const doc = mapValues(props[0], (val) => JSON.parse(val));
+
+    // Redis cannot store a null value, so we encode it as 'null', which does
+    // not match any possible MD5.
+    doc.docMD5 = props[1] === 'null' ? null : props[1];
+
+    // Ok, we have a valid DocStatus at this point.
+    return doc as DocStatus;
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a DocWorker to this docId if one is not yet assigned.
+   * Note that the assignment could be unmade at any time after this
+   * call if the worker dies, is brought down, or for other potential
+   * reasons in the future such as migration of individual documents
+   * between workers.
+   *
+   * A preferred doc worker can be specified, which will be assigned
+   * if no assignment is already made.
+   *
+   */
+  public async assignDocWorker(docId: string, workerId?: string): Promise<DocStatus> {
+    // Check if a DocWorker is already assigned; if so return result immediately
+    // without locking.
+    let docStatus = await this.getDocWorker(docId);
+    if (docStatus) { return docStatus; }
+
+    // No assignment yet, so let's lock and set an assignment up.
+    const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
+
+    try {
+      // Now that we've locked, recheck that the worker hasn't been reassigned
+      // in the meantime.  Return immediately if it has.
+      docStatus = await this.getDocWorker(docId);
+      if (docStatus) { return docStatus; }
+
+      if (!workerId) {
+        // Check if document has a preferred worker group set.
+        const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
+
+        // Let's start off by assigning documents to available workers randomly.
+        // TODO: use a smarter algorithm.
+        workerId = await this._client.srandmemberAsync(`workers-available-${group}`) || undefined;
+        if (!workerId) {
+          // No workers available in the desired worker group.  Rather than refusing to
+          // open the document, we fall back on assigning a worker from any of the workers
+          // available, regardless of grouping.
+          // This limits the impact of operational misconfiguration (bad redis setup,
+          // or not starting enough workers).  It has the downside of potentially disguising
+          // problems, so we log a warning.
+          log.warn(`DocWorkerMap.assignDocWorker ${docId} found no workers for group ${group}`);
+          workerId = await this._client.srandmemberAsync('workers-available') || undefined;
+        }
+        if (!workerId) { throw new Error('no doc workers available'); }
+      }  else {
+        if (!await this._client.sismemberAsync('workers-available', workerId)) {
+          throw new Error(`worker ${workerId} not known or not available`);
+        }
+      }
+
+      // Look up how to contact the worker.
+      const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
+      if (!docWorker) { throw new Error('no doc worker contact info available'); }
+
+      // We can now construct a DocStatus.
+      const newDocStatus = {docMD5: null, docWorker, isActive: true};
+
+      // We add the assignment to worker-{workerId}-docs and save doc-{docId}.
+      const result = await this._client.multi()
+        .sadd(`worker-${workerId}-docs`, docId)
+        .hmset(`doc-${docId}`, {
+          docWorker: JSON.stringify(docWorker),  // redis can't store nested objects, strings only
+          isActive: JSON.stringify(true)         // redis can't store booleans, strings only
+        })
+        .setex(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, 'null')
+        .execAsync();
+      if (!result) { throw new Error('failed to store new assignment'); }
+      return newDocStatus;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  /**
+   *
+   * Defined by IDocWorkerMap.
+   *
+   * Assigns a specific DocWorker to this docId if one is not yet assigned.
+   *
+   */
+  public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
+    return this.assignDocWorker(docId, workerId);
+  }
+
+  public async updateDocStatus(docId: string, checksum: string): Promise<void> {
+    await this._client.setexAsync(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, checksum);
+  }
+
+  public async setPermit(permit: Permit): Promise<string> {
+    const key = formatPermitKey(uuidv4());
+    const duration = (this._options && this._options.permitMsec) || PERMIT_TTL_MSEC;
+      // seems like only integer seconds are supported?
+    await this._client.setexAsync(key, Math.ceil(duration / 1000.0),
+                                  JSON.stringify(permit));
+    return key;
+  }
+
+  public async getPermit(key: string): Promise<Permit|null> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    const result = await this._client.getAsync(key);
+    return result && JSON.parse(result);
+  }
+
+  public async removePermit(key: string): Promise<void> {
+    if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
+    await this._client.delAsync(key);
+  }
+
+  public async close(): Promise<void> {
+    // nothing to do
+  }
+
+  public async getElection(name: string, durationInMs: number): Promise<string|null> {
+    // Could use "set nx" for election, but redis docs don't encourage that any more,
+    // favoring redlock:
+    //   https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      if (await this._client.getAsync(redisKey) !== null) { return null; }
+      const electionKey = uuidv4();
+      // seems like only integer seconds are supported?
+      await this._client.setexAsync(redisKey, Math.ceil(durationInMs / 1000.0), electionKey);
+      return electionKey;
+    } finally {
+      await lock.unlock();
+    }
+  }
+
+  public async removeElection(name: string, electionKey: string): Promise<void> {
+    const redisKey = `nomination-${name}`;
+    const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
+    try {
+      const current = await this._client.getAsync(redisKey);
+      if (current === electionKey) {
+        await this._client.delAsync(redisKey);
+      } else if (current !== null) {
+        throw new Error('could not remove election');
+      }
+    } finally {
+      await lock.unlock();
+    }
+  }
+}
+
+// If we don't have redis available and use a DummyDocWorker, it should be a singleton.
+let dummyDocWorkerMap: DummyDocWorkerMap|null = null;
+
+export function getDocWorkerMap(): IDocWorkerMap {
+  if (process.env.REDIS_URL) {
+    return new DocWorkerMap();
+  } else {
+    dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap();
+    return dummyDocWorkerMap;
+  }
+}

app/gen-server/lib/Permissions.ts

@@ -0,0 +1,21 @@
+export enum Permissions {
+  NONE          = 0x0,
+  // Note that the view permission bit provides view access ONLY to the resource to which
+  // the aclRule belongs - it does not allow listing that resource's children. A resource's
+  // children may only be listed if those children also have the view permission set.
+  VIEW          = 0x1,
+  UPDATE        = 0x2,
+  ADD           = 0x4,
+  // Note that the remove permission bit provides remove access to a resource AND all of
+  // its child resources/ACLs
+  REMOVE        = 0x8,
+  SCHEMA_EDIT   = 0x10,
+  ACL_EDIT      = 0x20,
+  EDITOR        = VIEW | UPDATE | ADD | REMOVE,   // tslint:disable-line:no-bitwise
+  ADMIN         = EDITOR | SCHEMA_EDIT,           // tslint:disable-line:no-bitwise
+  OWNER         = ADMIN | ACL_EDIT,               // tslint:disable-line:no-bitwise
+
+  // A virtual permission bit signifying that the general public has some access to
+  // the resource via ACLs involving the everyone@ user.
+  PUBLIC        = 0x80
+}

app/gen-server/lib/TypeORMPatches.ts

@@ -0,0 +1,196 @@
+// This contains two TypeORM patches.
+
+// Patch 1:
+// TypeORM Sqlite driver does not support using transactions in async code, if it is possible
+// for two transactions to get called (one of the whole point of transactions).  This
+// patch adds support for that, based on a monkey patch published in:
+//   https://gist.github.com/keenondrums/556f8c61d752eff730841170cd2bc3f1
+// Explanation at https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213
+
+// Patch 2:
+// TypeORM parameters are global, and collisions in setting them are not detected.
+// We add a patch to throw an exception if a parameter value is ever set and then
+// changed during construction of a query.
+
+import * as sqlite3 from '@gristlabs/sqlite3';
+import isEqual = require('lodash/isEqual');
+import {EntityManager, QueryRunner} from 'typeorm';
+import {SqliteDriver} from 'typeorm/driver/sqlite/SqliteDriver';
+import {SqliteQueryRunner} from 'typeorm/driver/sqlite/SqliteQueryRunner';
+import {
+  QueryRunnerProviderAlreadyReleasedError
+} from 'typeorm/error/QueryRunnerProviderAlreadyReleasedError';
+import {QueryBuilder} from 'typeorm/query-builder/QueryBuilder';
+
+
+/**********************
+ * Patch 1
+ **********************/
+
+type Releaser = () => void;
+type Worker<T> = () => Promise<T>|T;
+
+interface MutexInterface {
+  acquire(): Promise<Releaser>;
+  runExclusive<T>(callback: Worker<T>): Promise<T>;
+  isLocked(): boolean;
+}
+
+class Mutex implements MutexInterface {
+  private _queue: Array<(release: Releaser) => void> = [];
+  private _pending = false;
+
+  public isLocked(): boolean {
+    return this._pending;
+  }
+
+  public acquire(): Promise<Releaser> {
+    const ticket = new Promise<Releaser>(resolve => this._queue.push(resolve));
+    if (!this._pending) {
+      this._dispatchNext();
+    }
+    return ticket;
+  }
+
+  public runExclusive<T>(callback: Worker<T>): Promise<T> {
+    return this
+      .acquire()
+      .then(release => {
+        let result: T|Promise<T>;
+
+        try {
+          result = callback();
+        } catch (e) {
+          release();
+          throw(e);
+        }
+
+        return Promise
+          .resolve(result)
+          .then(
+            (x: T) => (release(), x),
+            e => {
+              release();
+              throw e;
+            }
+          );
+      }
+           );
+  }
+
+  private _dispatchNext(): void {
+    if (this._queue.length > 0) {
+      this._pending = true;
+      this._queue.shift()!(this._dispatchNext.bind(this));
+    } else {
+      this._pending = false;
+    }
+  }
+
+}
+
+// A singleton mutex for all sqlite transactions.
+const mutex = new Mutex();
+
+class SqliteQueryRunnerPatched extends SqliteQueryRunner {
+  private _releaseMutex: Releaser | null;
+
+  public async startTransaction(level?: any): Promise<void> {
+    this._releaseMutex = await mutex.acquire();
+    return super.startTransaction(level);
+  }
+
+  public async commitTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.commitTransaction -> mutex releaser unknown');
+    }
+    await super.commitTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async rollbackTransaction(): Promise<void> {
+    if (!this._releaseMutex) {
+      throw new Error('SqliteQueryRunnerPatched.rollbackTransaction -> mutex releaser unknown');
+    }
+    await super.rollbackTransaction();
+    this._releaseMutex();
+    this._releaseMutex = null;
+  }
+
+  public async connect(): Promise<any> {
+    if (!this.isTransactionActive) {
+      const release = await mutex.acquire();
+      release();
+    }
+    return super.connect();
+  }
+}
+
+class SqliteDriverPatched extends SqliteDriver {
+  public createQueryRunner(): QueryRunner {
+    if (!this.queryRunner) {
+      this.queryRunner = new SqliteQueryRunnerPatched(this);
+    }
+    return this.queryRunner;
+  }
+  protected loadDependencies(): void {
+    // Use our own sqlite3 module, which is a fork of the original.
+    this.sqlite = sqlite3;
+  }
+}
+
+// Patch the underlying SqliteDriver, since it's impossible to convince typeorm to use only our
+// patched classes. (Previously we patched DriverFactory and Connection, but those would still
+// create an unpatched SqliteDriver and then overwrite it.)
+SqliteDriver.prototype.createQueryRunner = SqliteDriverPatched.prototype.createQueryRunner;
+(SqliteDriver.prototype as any).loadDependencies = (SqliteDriverPatched.prototype as any).loadDependencies;
+
+export function applyPatch() {
+  // tslint: disable-next-line
+  EntityManager.prototype.transaction = async function <T>(arg1: any,  arg2?: any): Promise<T> {
+    if (this.queryRunner && this.queryRunner.isReleased) {
+      throw new QueryRunnerProviderAlreadyReleasedError();
+    }
+    if (this.queryRunner && this.queryRunner.isTransactionActive) {
+      throw new Error(`Cannot start transaction because its already started`);
+    }
+    const queryRunner = this.connection.createQueryRunner();
+    const runInTransaction = typeof arg1 === "function" ? arg1 : arg2;
+    try {
+      await queryRunner.startTransaction();
+      const result = await runInTransaction(queryRunner.manager);
+      await queryRunner.commitTransaction();
+      return result;
+    } catch (err) {
+      try {
+        // we throw original error even if rollback thrown an error
+        await queryRunner.rollbackTransaction();
+        // tslint: disable-next-line
+      } catch (rollbackError) {
+        // tslint: disable-next-line
+      }
+      throw err;
+    } finally {
+      await queryRunner.release();
+    }
+  };
+}
+
+
+/**********************
+ * Patch 2
+ **********************/
+
+abstract class QueryBuilderPatched<T> extends QueryBuilder<T> {
+  public setParameter(key: string, value: any): this {
+    const prev = this.expressionMap.parameters[key];
+    if (prev !== undefined && !isEqual(prev, value)) {
+      throw new Error(`TypeORM parameter collision for key '${key}' ('${prev}' vs '${value}')`);
+    }
+    this.expressionMap.parameters[key] = value;
+    return this;
+  }
+}
+
+(QueryBuilder.prototype as any).setParameter = (QueryBuilderPatched.prototype as any).setParameter;

app/gen-server/lib/Usage.ts

@@ -0,0 +1,62 @@
+import {Document} from 'app/gen-server/entity/Document';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {User} from 'app/gen-server/entity/User';
+import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import * as log from 'app/server/lib/log';
+
+// Frequency of logging usage information.  Not something we need
+// to track with much granularity.
+const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000;   // log every 1 hour
+
+/**
+ * Occasionally log usage information - number of users, orgs,
+ * docs, etc.
+ */
+export class Usage {
+  private _interval: NodeJS.Timeout;
+
+  public constructor(private _dbManager: HomeDBManager) {
+    this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS);
+    // Log once at beginning, in case we roll over servers faster than
+    // the logging period for an extended length of time,
+    // and to raise the visibility of this logging step so if it gets
+    // slow devs notice.
+    this.apply().catch(log.warn.bind(log));
+  }
+
+  public close() {
+    clearInterval(this._interval);
+  }
+
+  public async apply() {
+    const manager = this._dbManager.connection.manager;
+    // raw count of users
+    const userCount = await manager.count(User);
+    // users who have logged in at least once
+    const userWithLoginCount = await manager.createQueryBuilder()
+      .from(User, 'users')
+      .where('first_login_at is not null')
+      .getCount();
+    // raw count of organizations (excluding personal orgs)
+    const orgCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .where('owner_id is null')
+      .getCount();
+    // organizations with subscriptions that are in a non-terminated state
+    const orgInGoodStandingCount = await manager.createQueryBuilder()
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.billingAccount', 'billing_accounts')
+      .where('owner_id is null')
+      .andWhere('billing_accounts.in_good_standing = true')
+      .getCount();
+    // raw count of documents
+    const docCount = await manager.count(Document);
+    log.rawInfo('activity', {
+      docCount,
+      orgCount,
+      orgInGoodStandingCount,
+      userCount,
+      userWithLoginCount,
+    });
+  }
+}

app/gen-server/lib/scrubUserFromOrg.ts

@@ -0,0 +1,209 @@
+import {EntityManager} from "typeorm";
+import * as roles from 'app/common/roles';
+import {Document} from "app/gen-server/entity/Document";
+import {Group} from "app/gen-server/entity/Group";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Workspace} from "app/gen-server/entity/Workspace";
+import pick = require('lodash/pick');
+
+/**
+ *
+ * Remove the given user from the given org and every resource inside the org.
+ * If the user being removed is an owner of any resources in the org, the caller replaces
+ * them as the owner. This is to prevent complete loss of access to any resource.
+ *
+ * This method transforms ownership without regard to permissions.  We all talked this
+ * over and decided this is what we wanted, but there's no denying it is funky and could
+ * be surprising.
+ * TODO: revisit user scrubbing when we can.
+ *
+ */
+export async function scrubUserFromOrg(
+  orgId: number,
+  removeUserId: number,
+  callerUserId: number,
+  manager: EntityManager
+): Promise<void> {
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+
+  // This will be a list of all mentions of removeUser and callerUser in any resource
+  // within the org.
+  const mentions: Mention[] = [];
+
+  // Base query for all group_users related to these two users and this org.
+  const q = manager.createQueryBuilder()
+    .select('group_users.group_id, group_users.user_id')
+    .from('group_users', 'group_users')
+    .leftJoin(Group, 'groups', 'group_users.group_id = groups.id')
+    .addSelect('groups.name as name')
+    .leftJoin('groups.aclRule', 'acl_rules')
+    .where('(group_users.user_id = :removeUserId or group_users.user_id = :callerUserId)',
+           {removeUserId, callerUserId})
+    .andWhere('orgs.id = :orgId', {orgId});
+
+  // Pick out group_users related specifically to the org resource, in 'mentions' format
+  // (including resource id, a tag for the kind of resource, the group name, the user
+  // id, and the group id).
+  const orgs = q.clone()
+    .addSelect(`'org' as kind, orgs.id`)
+    .innerJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id');
+  mentions.push(...await orgs.getRawMany());
+
+  // Pick out mentions related to any workspace within the org.
+  const wss = q.clone()
+    .innerJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id')
+    .addSelect(`'ws' as kind, workspaces.id`)
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await wss.getRawMany());
+
+  // Pick out mentions related to any doc within the org.
+  const docs = q.clone()
+    .innerJoin(Document, 'docs', 'docs.id = acl_rules.doc_id')
+    .addSelect(`'doc' as kind, docs.id`)
+    .innerJoin('docs.workspace', 'workspaces')
+    .innerJoin('workspaces.org', 'orgs');
+  mentions.push(...await docs.getRawMany());
+
+  // Prepare to add and delete group_users.
+  const toDelete: Mention[] = [];
+  const toAdd: Mention[] = [];
+
+  // Now index the mentions by whether they are for the removeUser or the callerUser,
+  // and the resource they apply to.
+  const removeUserMentions = new Map<MentionKey, Mention>();
+  const callerUserMentions = new Map<MentionKey, Mention>();
+  for (const mention of mentions) {
+    const isGuest = mention.name === roles.GUEST;
+    if (mention.user_id === removeUserId) {
+      // We can safely remove any guest roles for the removeUser without any
+      // further inspection.
+      if (isGuest) { toDelete.push(mention); continue; }
+      removeUserMentions.set(getMentionKey(mention), mention);
+    } else {
+      if (isGuest) { continue; }
+      callerUserMentions.set(getMentionKey(mention), mention);
+    }
+  }
+  // Now iterate across the mentions of removeUser, and see what we need to do
+  // for each of them.
+  for (const [key, removeUserMention] of removeUserMentions) {
+    toDelete.push(removeUserMention);
+    if (removeUserMention.name !== roles.OWNER) {
+      // Nothing fancy needed for cases where the removeUser is not the owner.
+      // Just discard those.
+      continue;
+    }
+    // The removeUser was a direct owner on this resource, but the callerUser was
+    // not.  We set the callerUser as a direct owner on this resource, to preserve
+    // access to it.
+    // TODO: the callerUser might inherit sufficient access, in which case this
+    // step is unnecessary and could be skipped.  I believe it does no harm though.
+    const callerUserMention = callerUserMentions.get(key);
+    if (callerUserMention && callerUserMention.name === roles.OWNER) { continue; }
+    if (callerUserMention) { toDelete.push(callerUserMention); }
+    toAdd.push({...removeUserMention, user_id: callerUserId});
+  }
+  if (toDelete.length > 0) {
+    await manager.createQueryBuilder()
+      .delete()
+      .from('group_users')
+      .whereInIds(toDelete.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+  if (toAdd.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(toAdd.map(m => pick(m, ['user_id', 'group_id'])))
+      .execute();
+  }
+
+  // TODO: At this point, we've removed removeUserId from every mention in group_users.
+  // The user may still be mentioned in billing_account_managers.  If the billing_account
+  // is linked to just this single organization, perhaps it would make sense to remove
+  // the user there, if the callerUser is themselves a billing account manager?
+
+  await addMissingGuestMemberships(callerUserId, orgId, manager);
+}
+
+/**
+ * Adds specified user to any guest groups for the resources of an org where the
+ * user needs to be and is not already.
+ */
+export async function addMissingGuestMemberships(userId: number, orgId: number,
+                                                 manager: EntityManager) {
+  // For workspaces:
+  // User should be in guest group if mentioned in a doc within that workspace.
+  let groupUsers = await manager.createQueryBuilder()
+    .select('workspace_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Workspace, 'workspaces')
+    .where('workspaces.org_id = :orgId', {orgId})
+    .innerJoin('workspaces.docs', 'docs')
+    .innerJoin('docs.aclRules', 'doc_acl_rules')
+    .innerJoin('doc_acl_rules.group', 'doc_groups')
+    .innerJoin('doc_groups.memberUsers', 'doc_group_users')
+    .andWhere('doc_group_users.id = :userId', {userId})
+    .leftJoin('workspaces.aclRules', 'workspace_acl_rules')
+    .leftJoin('workspace_acl_rules.group', 'workspace_groups')
+    .leftJoin('group_users', 'workspace_group_users',
+              'workspace_group_users.group_id = workspace_groups.id and ' +
+              'workspace_group_users.user_id = :userId')
+    .andWhere('workspace_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('workspaces.id, workspace_groups.id, workspace_group_users.user_id')
+    .having('workspace_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For org:
+  // User should be in guest group if mentioned in a workspace within that org.
+  groupUsers = await manager.createQueryBuilder()
+    .select('org_groups.id as group_id, cast(:userId as int) as user_id')
+    .setParameter('userId', userId)
+    .from(Organization, 'orgs')
+    .where('orgs.id = :orgId', {orgId})
+    .innerJoin('orgs.workspaces', 'workspaces')
+    .innerJoin('workspaces.aclRules', 'workspaces_acl_rules')
+    .innerJoin('workspaces_acl_rules.group', 'workspace_groups')
+    .innerJoin('workspace_groups.memberUsers', 'workspace_group_users')
+    .andWhere('workspace_group_users.id = :userId', {userId})
+    .leftJoin('orgs.aclRules', 'org_acl_rules')
+    .leftJoin('org_acl_rules.group', 'org_groups')
+    .leftJoin('group_users', 'org_group_users',
+              'org_group_users.group_id = org_groups.id and ' +
+              'org_group_users.user_id = :userId')
+    .andWhere('org_groups.name = :guestName', {guestName: roles.GUEST})
+    .groupBy('org_groups.id, org_group_users.user_id')
+    .having('org_group_users.user_id is null')
+    .getRawMany();
+  if (groupUsers.length > 0) {
+    await manager.createQueryBuilder()
+      .insert()
+      .into('group_users')
+      .values(groupUsers)
+      .execute();
+  }
+
+  // For doc:
+  // Guest groups are not used.
+}
+
+interface Mention {
+  id: string|number;       // id of resource
+  kind: 'org'|'ws'|'doc';  // type of resource
+  user_id: number;         // id of user in group
+  group_id: number;        // id of group
+  name: string;            // name of group
+}
+
+type MentionKey = string;
+
+function getMentionKey(mention: Mention): MentionKey {
+  return `${mention.kind} ${mention.id}`;
+}

app/gen-server/lib/values.ts

@@ -0,0 +1,36 @@
+/**
+ * This smoothes over some awkward differences between TypeORM treatment of
+ * booleans and json in sqlite and postgres.  Booleans and json work fine
+ * with each db, but have different levels of driver-level support.
+ */
+
+export interface NativeValues {
+  // Json columns are handled natively by the postgres driver, but for
+  // sqlite requires a typeorm wrapper (simple-json).
+  jsonEntityType: 'json' | 'simple-json';
+  jsonType: 'json' | 'varchar';
+  booleanType: 'boolean' | 'integer';
+  dateTimeType: 'timestamp with time zone' | 'datetime';
+  trueValue: boolean | number;
+  falseValue: boolean | number;
+}
+
+const sqliteNativeValues: NativeValues = {
+  jsonEntityType: 'simple-json',
+  jsonType: 'varchar',
+  booleanType: 'integer',
+  dateTimeType: 'datetime',
+  trueValue: 1,
+  falseValue: 0
+};
+
+const postgresNativeValues: NativeValues = {
+  jsonEntityType: 'json',
+  jsonType: 'json',
+  booleanType: 'boolean',
+  dateTimeType: 'timestamp with time zone',
+  trueValue: true,
+  falseValue: false
+};
+
+export const nativeValues = (process.env.TYPEORM_TYPE === 'postgres') ? postgresNativeValues : sqliteNativeValues;

app/gen-server/migration/1536634251710-Initial.ts

@@ -0,0 +1,304 @@
+import {MigrationInterface, QueryRunner, Table} from "typeorm";
+
+
+export class Initial1536634251710 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // TypeORM doesn't currently help with types of created tables:
+    //   https://github.com/typeorm/typeorm/issues/305
+    // so we need to do a little smoothing over postgres and sqlite.
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    const now = "now()";
+
+    await queryRunner.createTable(new Table({
+      name: "users",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "api_key",
+          type: "varchar",
+          isNullable: true,
+          isUnique: true
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "orgs",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "domain",
+          type: "varchar",
+          isNullable: true,
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "owner_id",
+          type: "integer",
+          isNullable: true,
+          isUnique: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["owner_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "users"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "workspaces",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "org_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["org_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "orgs"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "docs",
+      columns: [
+        {
+          name: "id",
+          type: "varchar",
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        },
+        {
+          name: "created_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "updated_at",
+          type: datetime,
+          default: now
+        },
+        {
+          name: "workspace_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["workspace_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "workspaces"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "groups",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "name",
+          type: "varchar",
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "acl_rules",
+      columns: [
+        {
+          name: "id",
+          type: "integer",
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: "permissions",
+          type: "integer"
+        },
+        {
+          name: "type",
+          type: "varchar"
+        },
+        {
+          name: "workspace_id",
+          type: "integer",
+          isNullable: true
+        },
+        {
+          name: "org_id",
+          type: "integer",
+          isNullable: true
+        },
+        {
+          name: "doc_id",
+          type: "varchar",
+          isNullable: true
+        },
+        {
+          name: "group_id",
+          type: "integer",
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["workspace_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "workspaces"
+        },
+        {
+          columnNames: ["org_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "orgs"
+        },
+        {
+          columnNames: ["doc_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "docs"
+        },
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "group_users",
+      columns: [
+        {
+          name: "group_id",
+          type: "integer",
+          isPrimary: true
+        },
+        {
+          name: "user_id",
+          type: "integer",
+          isPrimary: true
+        },
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        },
+        {
+          columnNames: ["user_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "users"
+        }
+      ]
+    }), false);
+
+    await queryRunner.createTable(new Table({
+      name: "group_groups",
+      columns: [
+        {
+          name: "group_id",
+          type: "integer",
+          isPrimary: true
+        },
+        {
+          name: "subgroup_id",
+          type: "integer",
+          isPrimary: true
+        },
+      ],
+      foreignKeys: [
+        {
+          columnNames: ["group_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        },
+        {
+          columnNames: ["subgroup_id"],
+          referencedColumnNames: ["id"],
+          referencedTableName: "groups"
+        }
+      ]
+    }), false);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.query(`DROP TABLE "group_groups"`);
+    await queryRunner.query(`DROP TABLE "group_users"`);
+    await queryRunner.query(`DROP TABLE "acl_rules"`);
+    await queryRunner.query(`DROP TABLE "groups"`);
+    await queryRunner.query(`DROP TABLE "docs"`);
+    await queryRunner.query(`DROP TABLE "workspaces"`);
+    await queryRunner.query(`DROP TABLE "orgs"`);
+    await queryRunner.query(`DROP TABLE "users"`);
+  }
+}

app/gen-server/migration/1539031763952-Login.ts

@@ -0,0 +1,39 @@
+import {MigrationInterface, QueryRunner, Table} from "typeorm";
+
+export class Login1539031763952 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.createTable(new Table({
+        name: 'logins',
+        columns: [
+          {
+            name: "id",
+            type: "integer",
+            isGenerated: true,
+            generationStrategy: 'increment',
+            isPrimary: true
+          },
+          {
+            name: 'user_id',
+            type: 'integer'
+          },
+          {
+            name: 'email',
+            type: 'varchar',
+            isUnique: true
+          }
+        ],
+        foreignKeys: [
+          {
+            columnNames: ["user_id"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "users"
+          }
+        ]
+      }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.query('DROP TABLE logins');
+    }
+}

app/gen-server/migration/1549313797109-PinDocs.ts

@@ -0,0 +1,17 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class PinDocs1549313797109 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    await queryRunner.addColumn('docs', new TableColumn({
+      name: 'is_pinned',
+      type: 'boolean',
+      default: sqlite ? 0 : false
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('docs', 'is_pinned');
+  }
+}

app/gen-server/migration/1549381727494-UserPicture.ts

@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class UserPicture1549381727494 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.addColumn("users", new TableColumn({
+        name: "picture",
+        type: "varchar",
+        isNullable: true,
+      }));
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+      await queryRunner.dropColumn("users", "picture");
+    }
+}

app/gen-server/migration/1551805156919-LoginDisplayEmail.ts

@@ -0,0 +1,16 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class LoginDisplayEmail1551805156919 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.addColumn('logins', new TableColumn({
+      name: 'display_email',
+      type: 'varchar',
+      isNullable: true,
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('logins', 'display_email');
+  }
+}

app/gen-server/migration/1552416614755-LoginDisplayEmailNonNull.ts

@@ -0,0 +1,32 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class LoginDisplayEmailNonNull1552416614755 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.query('update logins set display_email = email where display_email is null');
+    // if our db will already heavily loaded, it might be better to add a check constraint
+    // rather than modifying the column properties.  But for our case, this will be fast.
+
+    // To work correctly with RDS version of postgres, it is important to clone
+    // and change typeorm's settings for the column, rather than the settings specified
+    // in previous migrations.  Otherwise typeorm will fall back on a brutal method of
+    // drop-and-recreate that doesn't work for non-null in any case.
+    //
+    // The pg command is very simple, just alter table logins alter column display_email set not null
+    // but sqlite migration is tedious since table needs to be rebuilt, so still just
+    // marginally worthwhile letting typeorm deal with it.
+    const logins = (await queryRunner.getTable('logins'))!;
+    const displayEmail = logins.findColumnByName('display_email')!;
+    const displayEmailNonNull = displayEmail.clone();
+    displayEmailNonNull.isNullable = false;
+    await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('logins'))!;
+    const displayEmail = logins.findColumnByName('display_email')!;
+    const displayEmailNonNull = displayEmail.clone();
+    displayEmailNonNull.isNullable = true;
+    await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
+  }
+}

app/gen-server/migration/1553016106336-Indexes.ts

@@ -0,0 +1,66 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class Indexes1553016106336 implements MigrationInterface {
+
+   public async up(queryRunner: QueryRunner): Promise<any> {
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__org_id",
+       columnNames: ["org_id"]
+     }));
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__workspace_id",
+       columnNames: ["workspace_id"]
+     }));
+     await queryRunner.createIndex("acl_rules", new TableIndex({
+       name: "acl_rules__doc_id",
+       columnNames: ["doc_id"]
+     }));
+
+     await queryRunner.createIndex("group_groups", new TableIndex({
+       name: "group_groups__group_id",
+       columnNames: ["group_id"]
+     }));
+     await queryRunner.createIndex("group_groups", new TableIndex({
+       name: "group_groups__subgroup_id",
+       columnNames: ["subgroup_id"]
+     }));
+
+     await queryRunner.createIndex("group_users", new TableIndex({
+       name: "group_users__group_id",
+       columnNames: ["group_id"]
+     }));
+     await queryRunner.createIndex("group_users", new TableIndex({
+       name: "group_users__user_id",
+       columnNames: ["user_id"]
+     }));
+
+     await queryRunner.createIndex("workspaces", new TableIndex({
+       name: "workspaces__org_id",
+       columnNames: ["org_id"]
+     }));
+
+     await queryRunner.createIndex("docs", new TableIndex({
+       name: "docs__workspace_id",
+       columnNames: ["workspace_id"]
+     }));
+
+     await queryRunner.createIndex("logins", new TableIndex({
+       name: "logins__user_id",
+       columnNames: ["user_id"]
+     }));
+   }
+
+   public async down(queryRunner: QueryRunner): Promise<any> {
+     await queryRunner.dropIndex("acl_rules", "acl_rules__org_id");
+     await queryRunner.dropIndex("acl_rules", "acl_rules__workspace_id");
+     await queryRunner.dropIndex("acl_rules", "acl_rules__doc_id");
+     await queryRunner.dropIndex("group_groups", "group_groups__group_id");
+     await queryRunner.dropIndex("group_groups", "group_groups__subgroup_id");
+     await queryRunner.dropIndex("group_users", "group_users__group_id");
+     await queryRunner.dropIndex("group_users", "group_users__user_id");
+     await queryRunner.dropIndex("workspaces", "workspaces__org_id");
+     await queryRunner.dropIndex("docs", "docs__workspace_id");
+     await queryRunner.dropIndex("logins", "logins__user_id");
+   }
+
+}

app/gen-server/migration/1556726945436-Billing.ts

@@ -0,0 +1,225 @@
+import {MigrationInterface, QueryRunner, Table, TableColumn, TableForeignKey} from 'typeorm';
+import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
+import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
+import {Organization} from 'app/gen-server/entity/Organization';
+import {Product} from 'app/gen-server/entity/Product';
+import {nativeValues} from 'app/gen-server/lib/values';
+
+export class Billing1556726945436 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // Create table for products.
+    await queryRunner.createTable(new Table({
+      name: 'products',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'name',
+          type: 'varchar'
+        },
+        {
+          name: 'stripe_product_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'features',
+          type: nativeValues.jsonType
+        }
+      ]
+    }));
+
+    // Create a basic free product that existing orgs can use.
+    const product = new Product();
+    product.name = 'Free';
+    product.features = {};
+    await queryRunner.manager.save(product);
+
+    // Create billing accounts and billing account managers.
+    await queryRunner.createTable(new Table({
+      name: 'billing_accounts',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'product_id',
+          type: 'integer'
+        },
+        {
+          name: 'individual',
+          type: nativeValues.booleanType
+        },
+        {
+          name: 'in_good_standing',
+          type: nativeValues.booleanType,
+          default: nativeValues.trueValue
+        },
+        {
+          name: 'status',
+          type: nativeValues.jsonType,
+          isNullable: true
+        },
+        {
+          name: 'stripe_customer_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'stripe_subscription_id',
+          type: 'varchar',
+          isUnique: true,
+          isNullable: true
+        },
+        {
+          name: 'stripe_plan_id',
+          type: 'varchar',
+          isNullable: true
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['product_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'products'
+        }
+      ]
+    }));
+
+    await queryRunner.createTable(new Table({
+      name: 'billing_account_managers',
+      columns: [
+        {
+          name: 'id',
+          type: 'integer',
+          isGenerated: true,
+          generationStrategy: 'increment',
+          isPrimary: true
+        },
+        {
+          name: 'billing_account_id',
+          type: 'integer'
+        },
+        {
+          name: 'user_id',
+          type: 'integer'
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['billing_account_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'billing_accounts',
+          onDelete: 'CASCADE'  // delete manager if referenced billing_account goes away
+        },
+        {
+          columnNames: ['user_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'users',
+          onDelete: 'CASCADE'  // delete manager if referenced user goes away
+        }
+      ]
+    }));
+
+    // Add a reference to billing accounts from orgs.
+    await queryRunner.addColumn('orgs', new TableColumn({
+        name: 'billing_account_id',
+        type: 'integer',
+        isNullable: true
+    }));
+    await queryRunner.createForeignKey('orgs', new TableForeignKey({
+      columnNames: ['billing_account_id'],
+      referencedColumnNames: ['id'],
+      referencedTableName: 'billing_accounts'
+    }));
+
+    // Let's add billing accounts to all existing orgs.
+    // Personal orgs are put on an individual billing account.
+    // Other orgs are put on a team billing account, with the
+    // list of payment managers seeded by owners of that account.
+    const query =
+      queryRunner.manager.createQueryBuilder()
+      .select('orgs.id')
+      .from(Organization, 'orgs')
+      .leftJoin('orgs.owner', 'owners')
+      .addSelect('orgs.owner.id')
+      .leftJoinAndSelect('orgs.aclRules', 'acl_rules')
+      .leftJoinAndSelect('acl_rules.group', 'groups')
+      .leftJoin('groups.memberUsers', 'users')
+      .addSelect('users.id')
+      .where('permissions & 8 = 8');  // seed managers with owners+editors, omitting guests+viewers
+                                      // (permission 8 is "Remove")
+    const orgs = await query.getMany();
+    for (const org of orgs) {
+      const individual = Boolean(org.owner);
+      const billingAccountInsert = await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(BillingAccount)
+        .values([{product, individual}])
+        .execute();
+      const billingAccountId = billingAccountInsert.identifiers[0].id;
+      if (individual) {
+        await queryRunner.manager.createQueryBuilder()
+          .insert()
+          .into(BillingAccountManager)
+          .values([{billingAccountId, userId: org.owner.id}])
+          .execute();
+      } else {
+        for (const rule of org.aclRules) {
+          for (const user of rule.group.memberUsers) {
+            await queryRunner.manager.createQueryBuilder()
+              .insert()
+              .into(BillingAccountManager)
+              .values([{billingAccountId, userId: user.id}])
+              .execute();
+          }
+        }
+      }
+      await queryRunner.manager.createQueryBuilder()
+        .update(Organization)
+        .set({billingAccountId})
+        .where('id = :id', {id: org.id})
+        .execute();
+    }
+
+    // TODO: in a future migration, orgs.billing_account_id could be constrained
+    // to be non-null.  All code deployments linked to a database that will be
+    // migrated must have code that sets orgs.billing_account_id by that time,
+    // otherwise they would fail to create orgs (and remember creating a user
+    // involves creating an org).
+    /*
+    // Now that all orgs have a billing account (and this migration is running within
+    // a transaction), we can constrain orgs.billing_account_id to be non-null.
+    const orgTable = (await queryRunner.getTable('orgs'))!;
+    const billingAccountId = orgTable.findColumnByName('billing_account_id')!;
+    const billingAccountIdNonNull = billingAccountId.clone();
+    billingAccountIdNonNull.isNullable = false;
+    await queryRunner.changeColumn('orgs', billingAccountId, billingAccountIdNonNull);
+    */
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    // this is a bit ugly, but is the documented way to remove a foreign key
+    const table = await queryRunner.getTable('orgs');
+    const foreignKey = table!.foreignKeys.find(fk => fk.columnNames.indexOf('billing_account_id') !== -1);
+    await queryRunner.dropForeignKey('orgs', foreignKey!);
+
+    await queryRunner.dropColumn('orgs', 'billing_account_id');
+    await queryRunner.dropTable('billing_account_managers');
+    await queryRunner.dropTable('billing_accounts');
+    await queryRunner.dropTable('products');
+  }
+
+}

app/gen-server/migration/1557157922339-OrgDomainUnique.ts

@@ -0,0 +1,27 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class OrgDomainUnique1557157922339 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('orgs'))!;
+    const domain = logins.findColumnByName('domain')!;
+    const domainUnique = domain.clone();
+    domainUnique.isUnique = true;
+    await queryRunner.changeColumn('orgs', domain, domainUnique);
+
+    // On postgres, all of the above amounts to:
+    //   ALTER TABLE "orgs" ADD CONSTRAINT "..." UNIQUE ("domain")
+    // On sqlite, the table gets regenerated.
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    const logins = (await queryRunner.getTable('orgs'))!;
+    const domain = logins.findColumnByName('domain')!;
+    const domainNonUnique = domain.clone();
+    domainNonUnique.isUnique = false;
+    await queryRunner.changeColumn('orgs', domain, domainNonUnique);
+
+    // On postgres, all of the above amount to:
+    //   ALTER TABLE "orgs" DROP CONSTRAINT "..."
+  }
+}

app/gen-server/migration/1561589211752-Aliases.ts

@@ -0,0 +1,67 @@
+import {MigrationInterface, QueryRunner, Table, TableColumn, TableIndex} from 'typeorm';
+import {datetime, now} from 'app/gen-server/sqlUtils';
+
+export class Aliases1561589211752 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const dbType = queryRunner.connection.driver.options.type;
+
+    // Make a table for document aliases.
+    await queryRunner.createTable(new Table({
+      name: 'aliases',
+      columns: [
+        {
+          name: 'url_id',
+          type: 'varchar',
+          isPrimary: true
+        },
+        {
+          name: 'org_id',
+          type: 'integer',
+          isPrimary: true
+        },
+        {
+          name: 'doc_id',
+          type: 'varchar',
+          isNullable: true   // nullable in case in future we make aliases for other resources
+        },
+        {
+          name: "created_at",
+          type: datetime(dbType),
+          default: now(dbType)
+        }
+      ],
+      foreignKeys: [
+        {
+          columnNames: ['doc_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'docs',
+          onDelete: 'CASCADE'  // delete alias if doc goes away
+        },
+        {
+          columnNames: ['org_id'],
+          referencedColumnNames: ['id'],
+          referencedTableName: 'orgs'
+          // no CASCADE set - let deletions be triggered via docs
+        }
+      ]
+    }));
+
+    // Add preferred alias to docs.  Not quite a foreign key (we'd need org as well)
+    await queryRunner.addColumn('docs', new TableColumn({
+        name: 'url_id',
+        type: 'varchar',
+        isNullable: true
+    }));
+    await queryRunner.createIndex("docs", new TableIndex({
+      name: "docs__url_id",
+      columnNames: ["url_id"]
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropIndex('docs', 'docs__url_id');
+    await queryRunner.dropColumn('docs', 'url_id');
+    await queryRunner.dropTable('aliases');
+  }
+}

app/gen-server/migration/1568238234987-TeamMembers.ts

@@ -0,0 +1,56 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+import * as roles from "app/common/roles";
+import {AclRuleOrg} from "app/gen-server/entity/AclRule";
+import {Group} from "app/gen-server/entity/Group";
+import {Organization} from "app/gen-server/entity/Organization";
+import {Permissions} from "app/gen-server/lib/Permissions";
+
+export class TeamMembers1568238234987 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // Get all orgs and add a team member ACL (with group) to each.
+    const orgs = await queryRunner.manager.createQueryBuilder()
+      .select("orgs.id")
+      .from(Organization, "orgs")
+      .getMany();
+    for (const org of orgs) {
+      const groupInsert = await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(Group)
+        .values([{name: roles.MEMBER}])
+        .execute();
+      const groupId = groupInsert.identifiers[0].id;
+      await queryRunner.manager.createQueryBuilder()
+        .insert()
+        .into(AclRuleOrg)
+        .values([{
+          permissions: Permissions.VIEW,
+          organization: {id: org.id},
+          group: groupId
+        }])
+        .execute();
+    }
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    // Remove all team member groups and corresponding ACLs.
+    const groups = await queryRunner.manager.createQueryBuilder()
+      .select("groups")
+      .from(Group, "groups")
+      .where('name = :name', {name: roles.MEMBER})
+      .getMany();
+    for (const group of groups) {
+      await queryRunner.manager.createQueryBuilder()
+        .delete()
+        .from(AclRuleOrg)
+        .where("group_id = :id", {id: group.id})
+        .execute();
+    }
+    await queryRunner.manager.createQueryBuilder()
+      .delete()
+      .from(Group)
+      .where("name = :name", {name: roles.MEMBER})
+      .execute();
+  }
+
+}

app/gen-server/migration/1569593726320-FirstLogin.ts

@@ -0,0 +1,18 @@
+import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
+
+export class FirstLogin1569593726320 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    await queryRunner.addColumn('users', new TableColumn({
+      name: 'first_login_at',
+      type: datetime,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('users', 'first_login_at');
+  }
+}