(core) move home server into core

Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552
pull/2/head
Paul Fitzpatrick 4 years ago
parent c756f663ee
commit 5ef889addd

@ -0,0 +1,74 @@
/**
* Basic definitions of types needed for ActionBundles.
* See also EncActionBundle for how these are packaged for encryption.
*/
import {DocAction, UserAction} from 'app/common/DocActions';
// Metadata about the action.
export interface ActionInfo {
time: number; // Milliseconds since epoch.
user: string;
inst: string;
desc?: string;
otherId: number;
linkId: number;
}
// Envelope contains information about recipients. In EncActionBundle, it's augmented with
// information about the symmetric key that encrypts this envelope's contents.
export interface Envelope {
recipients: string[]; // sorted array of recipient instanceIds
}
// EnvContent packages arbitrary content with the index of the envelope to which it belongs.
export type EnvContent<Content> = [number, Content];
// ActionBundle contains actions arranged into envelopes, i.e. split up by sets of recipients.
// Note that different Envelopes contain different sets of recipients (which may overlap however).
// ActionBundle is what gets encrypted/decrypted and then sent between hub and instance.
export interface ActionBundle {
actionNum: number;
actionHash: string|null; // a checksum of bundle, (not including actionHash and other parts).
parentActionHash: string|null; // a checksum of the parent action bundle, if there is one.
envelopes: Envelope[];
info: EnvContent<ActionInfo>; // Should be in the envelope addressed to all peers.
stored: Array<EnvContent<DocAction>>;
calc: Array<EnvContent<DocAction>>;
}
export function getEnvContent<Content>(items: Array<EnvContent<Content>>): Content[] {
return items.map((item) => item[1]);
}
// ======================================================================
// Types for ActionBundles used locally inside an instance.
// Local action received from the browser, that is not yet applied. It is usually one UserAction,
// but when multiple actions are sent by the browser in one call, they will form one bundle.
export interface UserActionBundle {
info: ActionInfo;
userActions: UserAction[];
}
// ActionBundle as received from the sandbox. It does not have some action metadata, but does have
// undo information and a retValue for each input UserAction. Note that it is satisfied by the
// ActionBundle structure defined in sandbox/grist/action_obj.py.
export interface SandboxActionBundle {
envelopes: Envelope[];
stored: Array<EnvContent<DocAction>>;
calc: Array<EnvContent<DocAction>>;
undo: Array<EnvContent<DocAction>>; // Inverse actions for all 'stored' actions.
retValues: any[]; // Contains retValue for each of userActions.
}
// Local action that's been applied. It now has an actionNum, and includes doc actions packaged
// into envelopes, as well as undo, and userActions, which allow rebasing.
export interface LocalActionBundle extends ActionBundle {
userActions: UserAction[];
// Inverse actions for all 'stored' actions. These aren't shared and not split by envelope.
// Applying 'undo' is governed by EDIT rather than READ permissions, so we always apply all undo
// actions. (It is the result of applying 'undo' that may be addressed to different recipients).
undo: DocAction[];
}

@ -0,0 +1,71 @@
import mapValues = require('lodash/mapValues');
import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction} from "./DocActions";
// TODO this replaces modelUtil's ActionDispatcher and bulkActionExpand. Those should be removed.
/**
* Helper class which provides a `dispatchAction` method that dispatches DocActions received from
* the server to methods `this.on{ActionType}`, e.g. `this.onUpdateRecord`.
*
* Implementation methods `on*` are called with the action as the first argument, and with
* the action arguments as additional method arguments, for convenience.
*
* Methods for bulk actions may be implemented directly, or will iterate through each record in
* the action, and call the single-record methods for each one.
*/
export abstract class ActionDispatcher {
public dispatchAction(action: DocAction): void {
// In node 6 testing, this switch is 5+ times faster than looking up "on"+action[0].
const a: any[] = action;
switch (action[0]) {
case "AddRecord": return this.onAddRecord (action, a[1], a[2], a[3]);
case "UpdateRecord": return this.onUpdateRecord (action, a[1], a[2], a[3]);
case "RemoveRecord": return this.onRemoveRecord (action, a[1], a[2]);
case "BulkAddRecord": return this.onBulkAddRecord (action, a[1], a[2], a[3]);
case "BulkUpdateRecord": return this.onBulkUpdateRecord(action, a[1], a[2], a[3]);
case "BulkRemoveRecord": return this.onBulkRemoveRecord(action, a[1], a[2]);
case "ReplaceTableData": return this.onReplaceTableData(action, a[1], a[2], a[3]);
case "AddColumn": return this.onAddColumn (action, a[1], a[2], a[3]);
case "RemoveColumn": return this.onRemoveColumn (action, a[1], a[2]);
case "RenameColumn": return this.onRenameColumn (action, a[1], a[2], a[3]);
case "ModifyColumn": return this.onModifyColumn (action, a[1], a[2], a[3]);
case "AddTable": return this.onAddTable (action, a[1], a[2]);
case "RemoveTable": return this.onRemoveTable (action, a[1]);
case "RenameTable": return this.onRenameTable (action, a[1], a[2]);
default: throw new Error(`Received unknown action ${action[0]}`);
}
}
protected abstract onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void;
protected abstract onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void;
protected abstract onRemoveRecord(action: DocAction, tableId: string, rowId: number): void;
// If not overridden, these will make multiple calls to single-record action methods.
protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
for (let i = 0; i < rowIds.length; i++) {
this.onAddRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i]));
}
}
protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
for (let i = 0; i < rowIds.length; i++) {
this.onUpdateRecord(action, tableId, rowIds[i], mapValues(colValues, (values) => values[i]));
}
}
protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) {
for (const r of rowIds) {
this.onRemoveRecord(action, tableId, r);
}
}
protected abstract onReplaceTableData(
action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void;
protected abstract onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void;
protected abstract onRemoveColumn(action: DocAction, tableId: string, colId: string): void;
protected abstract onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void;
protected abstract onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void;
protected abstract onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void;
protected abstract onRemoveTable(action: DocAction, tableId: string): void;
protected abstract onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void;
}

@ -0,0 +1,19 @@
import {ActionSummary} from 'app/common/ActionSummary';
/** This is the action representation the client works with. */
export interface ActionGroup {
actionNum: number;
actionHash: string;
desc?: string;
actionSummary: ActionSummary;
fromSelf: boolean;
linkId: number;
otherId: number;
time: number;
user: string;
rowIdHint: number; // If non-zero, this is a rowId that would be a good place to put
// the cursor after an undo.
primaryAction: string; // The name of the first user action in the ActionGroup.
isUndo: boolean; // True if the first user action is ApplyUndoActions.
internal: boolean; // True if it is inappropriate to log/undo the action.
}

@ -0,0 +1,49 @@
import { Rpc } from "grain-rpc";
/**
* ActionRouter allows to choose what actions to send over rpc. Action are posted as message `{type:
* "docAction", action }` over rpc.
*/
export class ActionRouter {
private _subscribedTables: Set<string> = new Set();
constructor(private _rpc: Rpc) {}
/**
* Subscribe to send all actions related to a table. Keeps sending actions if table is renamed.
*/
public subscribeTable(tableId: string): Promise<void> {
this._subscribedTables.add(tableId);
return Promise.resolve();
}
/**
* Stop sending all message related to a table.
*/
public unsubscribeTable(tableId: string): Promise<void> {
this._subscribedTables.delete(tableId);
return Promise.resolve();
}
/**
* Process a action updates subscription set in case of table rename and table remove, and post
* action if it matches a subscriptions.
*/
public process(action: any[]): Promise<void> {
const tableId = action[1];
if (!this._subscribedTables.has(tableId)) {
return Promise.resolve();
}
switch (action[0]) {
case "RemoveTable":
this._subscribedTables.delete(tableId);
break;
case "RenameTable":
this._subscribedTables.delete(tableId);
this._subscribedTables.add(action[2]);
break;
}
return this._rpc.postMessage({type: "docAction", action});
}
}

@ -0,0 +1,235 @@
import {CellDelta, TabularDiff, TabularDiffs} from 'app/common/TabularDiff';
import toPairs = require('lodash/toPairs');
/**
* An ActionSummary represents the overall effect of changes that took place
* during a period of history.
* - Only net changes are represented. Intermediate changes within the period are
* not represented. Changes that are done and undone within the period are not
* represented.
* - Net addition, removal, and renaming of tables is represented. The names
* of tables, for ActionSummary purposes are their tableIds, the database-safe
* version of their names.
* - Net addition, removal, and renaming of columns is represented. As for tables,
* the names of columns for ActionSummary purposes are their colIds.
* - Net additions and removals of rows are partially represented. The rowIds of added
* and removed rows are represented fully. The *values* of cells in the rows that
* were added or removed are stored in some cases. There is a threshold on the
* number of rows whose values will be cached for each DocAction scanned.
* - Net updates of rows are partially represented. The rowIds of updated rows are
* represented fully, but the *values* of updated cells partially, as for additions/
* removals.
* - Cell value changes affecting _grist_* tables are always represented in full,
* even if they are bulk changes.
*
* The representation of table name changes and column name changes is the same,
* simply a list of name pairs [before, after]. We represent the addition of a
* a table (or column) as the special name pair [null, initialName], and the
* removal of a table (or column) as the special name pair [finalName, null].
*
* An ActionSummary contains two fields:
* - tableRenames: a list of table name changes (incuding addition/removal).
* - tableDeltas: a dictionary of changes within a table.
*
* The key of the tableDeltas dictionary is the name of a table at the end of the
* period of history covered by the ActionSummary.
* - For example, if we add a table called N, we use the key N for it.
* - If we rename a table from N1 to N2, we use the key N2 for it.
* - If we add a table called N1, then rename it to N2, we use the key N2 for it.
* If the table was removed during that period, we use its name at the beginning
* of the period, preceded by "-".
* - If we remove a table called N, we use the key -N for it.
* - If we add a table called N, then remove it, there is no net change to represent.
* - If we remove a table called N, then add a new table called N, we use the key -N
* for the first, and the key N for the second.
*
* The changes within a table are represented as a TableDelta, which has the following
* fields:
* - columnRenames: a list of column name changes (incuding addition/removal).
* - columnDeltas: a dictionary of changes within a column.
* - updateRows, removeRows, addRows: lists of affected rows.
*
* The columnRenames/columnDeltas pair work just like tableRenames/tableDeltas, just
* on the scope of columns within a table rather than tables within a document.
*
* The changes within a column are represented as a ColumnDelta, which is a dictionary
* keyed by rowIds. It contains CellDelta values. CellDelta values represent before
* and after values of a particular cell.
* - a CellDelta of [null, [value]] represents a cell that was non-existent coming into
* existence with the given value.
* - a CellDelta of [[value], null] represents an existing cell with the given value that
* is removed.
* - a CellDelta of [[value1], [value2]] represents a change in value of a cell between
* two known values.
* - a CellDelta of ['?', [value2]] represents a change in value of a cell from an
* unknown value to a known value. Unknown values happen when we know a cell was
* implicated in a bulk change but its value didn't happen to be stored.
* - a CellDelta of [[value1], '?'] represents a change in value of a cell from an
* known value to an unknown value.
* The CellDelta itself does not tell you whether the rowId has the same identity before
* and after -- for example it may have been removed and then added. That information
* is available by consulting the removeRows and addRows fields.
*
*/
/**
* A collection of changes related to a set of tables.
*/
export interface ActionSummary {
tableRenames: LabelDelta[]; /** a list of table renames/additions/removals */
tableDeltas: {[tableId: string]: TableDelta}; /** changes within an individual table */
}
/**
* A collection of changes related to rows and columns of a single table.
*/
export interface TableDelta {
updateRows: number[]; /** rowIds of rows that exist before+after and were changed during */
removeRows: number[]; /** rowIds of rows that existed before but were removed during */
addRows: number[]; /** rowIds of rows that were added during, and exist after */
/** Partial record of cell-level changes - large bulk changes not included. */
columnDeltas: {[colId: string]: ColumnDelta};
columnRenames: LabelDelta[]; /** a list of column renames/additions/removals */
}
/**
* Pairs of before/after names of tables and columns. Null represents non-existence,
* so the addition and removal of tables/columns can be represented.
*/
export type LabelDelta = [string|null, string|null];
/**
* A collection of changes related to cells in a specific column.
*/
export interface ColumnDelta {
[rowId: number]: CellDelta;
}
/** Create an ActionSummary for a period with no action */
export function createEmptyActionSummary(): ActionSummary {
return { tableRenames: [], tableDeltas: {} };
}
/** Create a TableDelta for a period with no action */
export function createEmptyTableDelta(): TableDelta {
return {
updateRows: [],
removeRows: [],
addRows: [],
columnDeltas: {},
columnRenames: []
};
}
/**
* Distill a summary further, into tabular form, for ease of rendering.
*/
export function asTabularDiffs(summary: ActionSummary): TabularDiffs {
const allChanges: TabularDiffs = {};
for (const [tableId, td] of toPairs(summary.tableDeltas)) {
const tableChanges: TabularDiff = allChanges[tableId] = {
header: [],
cells: [],
};
// swap order to row-dominant for visualization purposes
const perRow: {[row: number]: {[name: string]: any}} = {};
const activeCols = new Set<string>();
// iterate through the column-dominant representation grist prefers internally
for (const [col, perCol] of toPairs(td.columnDeltas)) {
activeCols.add(col);
// iterate through the rows for that column, writing out the row-dominant
// results we want for visualization.
for (const row of Object.keys(perCol)) {
if (!perRow[row as any]) { perRow[row as any] = {}; }
perRow[row as any][col] = perCol[row as any];
}
}
// TODO: recover order of columns; recover row numbers (as opposed to rowIds)
const activeColsWithoutManualSort = [...activeCols].filter(c => c !== 'manualSort');
tableChanges.header = activeColsWithoutManualSort;
const addedRows = new Set(td.addRows);
const removedRows = new Set(td.removeRows);
const updatedRows = new Set(td.updateRows);
const rowIds = Object.keys(perRow).map(row => parseInt(row, 10));
const presentRows = new Set(rowIds);
const droppedRows = [...addedRows, ...removedRows, ...updatedRows]
.filter(x => !presentRows.has(x))
.sort((a, b) => a - b);
// Now that we have pulled together rows of changes, we will add a summary cell
// to each row to show whether they were caused by row updates, additions or removals.
// We also at this point make sure the cells of the row are output in a consistent
// order with a header.
for (const rowId of rowIds) {
if (droppedRows.length > 0) {
// Bulk additions/removals/updates may result in just some rows being saved.
// We signal this visually with a "..." row. The order of where this should
// go isn't well defined at this point (there's a row number TODO above).
if (rowId > droppedRows[0]) {
tableChanges.cells.push(['...', droppedRows[0],
activeColsWithoutManualSort.map(x => [null, null] as [null, null])]);
while (rowId > droppedRows[0]) {
droppedRows.shift();
}
}
}
// For each rowId, we need to issue either 1 or 2 rows. We issue 2 rows
// if the rowId is both added and removed - in this scenario, the rows
// before and after are unrelated. In all other cases, the before and
// after values refer to the same row.
const versions: Array<[string, (diff: CellDelta) => CellDelta]> = [];
if (addedRows.has(rowId) && removedRows.has(rowId)) {
versions.push(['-', (diff) => [diff[0], null]]);
versions.push(['+', (diff) => [null, diff[1]]]);
} else {
let code: string = '...';
if (updatedRows.has(rowId)) { code = '→'; }
if (addedRows.has(rowId)) { code = '+'; }
if (removedRows.has(rowId)) { code = '-'; }
versions.push([code, (diff) => diff]);
}
for (const [code, transform] of versions) {
const acc: CellDelta[] = [];
const perCol = perRow[rowId];
activeColsWithoutManualSort.forEach(col => {
const diff = perCol ? perCol[col] : null;
if (!diff) {
acc.push([null, null]);
} else {
acc.push(transform(diff));
}
});
tableChanges.cells.push([code, rowId, acc]);
}
}
}
return allChanges;
}
/**
* Return a suitable key for a removed table/column. We cannot use their id directly
* since it could clash with an added table/column of the same name.
*/
export function defunctTableName(id: string): string {
return `-${id}`;
}
export function rootTableName(id: string): string {
return id.replace('-', '');
}
/**
* Returns a list of all tables changed by the summarized action. Changes include
* schema or data changes. Tables are identified by their post-action name.
* Deleted tables are identified by their pre-action name, with "-" prepended.
*/
export function getAffectedTables(summary: ActionSummary): string[] {
return [
// Tables added, renamed, or removed in this action.
...summary.tableRenames.map(pair => pair[1] || defunctTableName(pair[0] || "")),
// Tables modified in this action.
...Object.keys(summary.tableDeltas)
];
}

@ -0,0 +1,226 @@
import {ActionGroup} from 'app/common/ActionGroup';
import {CellValue, TableDataAction, UserAction} from 'app/common/DocActions';
import {Peer} from 'app/common/sharing';
import {UploadResult} from 'app/common/uploads';
import {ParseOptions} from 'app/plugin/FileParserAPI';
import {IMessage} from 'grain-rpc';
export interface ApplyUAOptions {
desc?: string; // Overrides the description of the action.
otherId?: number; // For undo/redo; the actionNum of the original action to which it applies.
linkId?: number; // For bundled actions, actionNum of the previous action in the bundle.
}
export interface ApplyUAResult {
actionNum: number; // number of the action that got recorded.
retValues: any[]; // array of return values, one for each of the passed-in user actions.
isModification: boolean; // true if document was modified.
}
export interface DataSourceTransformed {
// Identifies the upload, which may include multiple files.
uploadId: number;
// For each file in the upload, the transform rules for that file.
transforms: TransformRuleMap[];
}
export interface TransformRuleMap {
[origTableName: string]: TransformRule;
}
export interface TransformRule {
destTableId: string|null;
destCols: TransformColumn[];
sourceCols: string[];
}
export interface TransformColumn {
label: string;
colId: string|null;
type: string;
formula: string;
}
export interface ImportResult {
options: ParseOptions;
tables: ImportTableResult[];
}
export interface ImportTableResult {
hiddenTableId: string;
uploadFileIndex: number; // Index into upload.files array, for the file reponsible for this table.
origTableName: string;
transformSectionRef: number;
destTableId: string|null;
}
/**
* Represents a query for Grist data. The tableId is required. An empty set of filters indicates
* the full table. Examples:
* {tableId: "Projects", filters: {}}
* {tableId: "Employees", filters: {Status: ["Active"], Dept: ["Sales", "HR"]}}
*/
export interface Query {
tableId: string;
filters: {
[colId: string]: any[];
};
// Queries to server for onDemand tables will set a limit to avoid bringing down the browser.
limit?: number;
}
/**
* Response from useQuerySet(). A query returns data AND creates a subscription to receive
* DocActions that affect this data. The querySubId field identifies this subscription, and must
* be used in a disposeQuerySet() call to unsubscribe.
*/
export interface QueryResult {
querySubId: number; // ID of the subscription, to use with disposeQuerySet.
tableData: TableDataAction;
}
/**
* Result of a fork operation, with newly minted ids.
* For a document with docId XXXXX and urlId UUUUU, the fork will have a
* docId of XXXXX~FORKID[~USERID] and a urlId of UUUUU~FORKID[~USERID].
*/
export interface ForkResult {
docId: string;
urlId: string;
}
export interface ActiveDocAPI {
/**
* Closes a document, and unsubscribes from its userAction events.
*/
closeDoc(): Promise<void>;
/**
* Fetches a particular table from the data engine to return to the client.
*/
fetchTable(tableId: string): Promise<TableDataAction>;
/**
* Fetches the generated Python code for this document. (TODO rename this misnomer.)
*/
fetchTableSchema(): Promise<string>;
/**
* Makes a query (documented elsewhere) and subscribes to it, so that the client receives
* docActions that affect this query's results. The subscription remains functional even when
* tables or columns get renamed.
*/
useQuerySet(query: Query): Promise<QueryResult>;
/**
* Removes the subscription to a Query, identified by QueryResult.querySubId, so that the
* client stops receiving docActions relevant only to that query.
*/
disposeQuerySet(querySubId: number): Promise<void>;
/**
* Applies an array of user actions to the document.
*/
applyUserActions(actions: UserAction[], options?: ApplyUAOptions): Promise<ApplyUAResult>;
/**
* A variant of applyUserActions where actions are passed in by ids (actionNum, actionHash)
* rather than by value.
*/
applyUserActionsById(actionNums: number[], actionHashes: string[],
undo: boolean, options?: ApplyUAOptions): Promise<ApplyUAResult>;
/**
* Imports files, removes previously created temporary hidden tables and creates the new ones.
*/
importFiles(dataSource: DataSourceTransformed,
parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
/**
* Finishes import files, creates the new tables, and cleans up temporary hidden tables and uploads.
*/
finishImportFiles(dataSource: DataSourceTransformed,
parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult>;
/**
* Cancels import files, cleans up temporary hidden tables and uploads.
*/
cancelImportFiles(dataSource: DataSourceTransformed, prevTableIds: string[]): Promise<void>;
/**
* Saves attachments from a given upload and creates an entry for them in the database. It
* returns the list of rowIds for the rows created in the _grist_Attachments table.
*/
addAttachments(uploadId: number): Promise<number[]>;
/**
* Returns up to n columns in the document, or a specific table, which contain the given values.
* Columns are returned ordered from best to worst based on an estimate for number of matches.
*/
findColFromValues(values: any[], n: number, optTableId?: string): Promise<number[]>;
/**
* Returns cell value with an error message (traceback) for one invalid formula cell.
*/
getFormulaError(tableId: string, colId: string, rowId: number): Promise<CellValue>;
/**
* Fetch content at a url.
*/
fetchURL(url: string): Promise<UploadResult>;
/**
* Find and return a list of auto-complete suggestions that start with `txt`, when editing a
* formula in table `tableId`.
*/
autocomplete(txt: string, tableId: string): Promise<string[]>;
/**
* Shares the doc and invites peers.
*/
shareDoc(peers: Peer[]): Promise<void>;
/**
* Removes the current instance from the doc.
*/
removeInstanceFromDoc(): Promise<void>;
/**
* Get recent actions in ActionGroup format with summaries included.
*/
getActionSummaries(): Promise<ActionGroup[]>;
/**
* Initiates user actions bandling for undo.
*/
startBundleUserActions(): Promise<void>;
/**
* Stopes user actions bandling for undo.
*/
stopBundleUserActions(): Promise<void>;
/**
* Forward a grain-rpc message to a given plugin.
*/
forwardPluginRpc(pluginId: string, msg: IMessage): Promise<any>;
/**
* Reload documents plugins.
*/
reloadPlugins(): Promise<void>;
/**
* Immediately close the document and data engine, to be reloaded from scratch, and cause all
* browser clients to reopen it.
*/
reloadDoc(): Promise<void>;
/**
* Prepare a fork of the document, and return the id(s) of the fork.
* TODO: remove string option here, it is present to ease transition.
*/
fork(): Promise<string | ForkResult>;
}

@ -0,0 +1,40 @@
/**
* A tip for fixing an error.
*/
export interface ApiTip {
action: 'add-members' | 'upgrade' |'ask-for-help';
message: string;
}
/**
* Documentation of a limit relevant to an API error.
*/
export interface ApiLimit {
quantity: 'collaborators' | 'docs' | 'workspaces'; // what are we counting
subquantity?: string; // a nuance to what we are counting
maximum: number; // maximum allowed
value: number; // current value of quantity for user
projectedValue: number; // value of quantity expected if request had been allowed
}
/**
* Structured details about an API error.
*/
export interface ApiErrorDetails {
limit?: ApiLimit;
// If set, this is the more user-friendly message to show to the user than error.message.
userError?: string;
// If set, contains suggestions for fixing a problem.
tips?: ApiTip[];
}
/**
* An error with an http status code.
*/
export class ApiError extends Error {
constructor(message: string, public status: number, public details?: ApiErrorDetails) {
super(message);
}
}

@ -0,0 +1,147 @@
/**
* Implements a pattern for creating objects requiring asynchronous construction. The given
* asynchronous createFunc() is called on the .get() call, and the result is cached on success.
* On failure, the result is cleared, so that subsequent calls attempt the creation again.
*
* Usage:
* this._obj = AsyncCreate<MyObject>(asyncCreateFunc);
* obj = await this._obj.get(); // calls asyncCreateFunc
* obj = await this._obj.get(); // uses cached object if asyncCreateFunc succeeded, else calls it again.
*
* Note that multiple calls while createFunc() is running will return the same promise, and will
* succeed or fail together.
*/
export class AsyncCreate<T> {
private _value?: Promise<T> = undefined;
constructor(private _createFunc: () => Promise<T>) {}
/**
* Returns createFunc() result, returning the cached promise if createFunc() succeeded, or if
* another call to it is currently pending.
*/
public get(): Promise<T> {
return this._value || (this._value = this._clearOnError(this._createFunc.call(null)));
}
/** Clears the cached promise, forcing createFunc to be called again on next get(). */
public clear(): void {
this._value = undefined;
}
/** Returns a boolean indicating whether the object is created. */
public isSet(): boolean {
return Boolean(this._value);
}
/** Returns the value if it's set and successful, or undefined otherwise. */
public async getIfValid(): Promise<T|undefined> {
return this._value ? this._value.catch(() => undefined) : undefined;
}
// Helper which clears this AsyncCreate if the given promise is rejected.
private _clearOnError(p: Promise<T>): Promise<T> {
p.catch(() => this.clear());
return p;
}
}
/**
* Supports a usage similar to AsyncCreate in a Map. Returns map.get(key) if it is set to a
* resolved or pending promise. Otherwise, calls creator(key) to create and return a new promise,
* and sets the key to it. If the new promise is rejected, the key will be removed from the map,
* so that subsequent calls would call creator() again.
*
* As with AsyncCreate, while the promise for a key is pending, multiple calls to that key will
* return the same promise, and will succeed or fail together.
*/
export function mapGetOrSet<K, V>(map: Map<K, Promise<V>>, key: K, creator: (key: K) => Promise<V>): Promise<V> {
return map.get(key) || mapSetOrClear(map, key, creator(key));
}
/**
* Supports a usage similar to AsyncCreate in a Map. Sets the given key in a map to the given
* promise, and removes it later if the promise is rejected. Returns the same promise.
*/
export function mapSetOrClear<K, V>(map: Map<K, Promise<V>>, key: K, pvalue: Promise<V>): Promise<V> {
pvalue.catch(() => map.delete(key));
map.set(key, pvalue);
return pvalue;
}
/**
* A Map implementation that allows for expiration of old values.
*/
export class MapWithTTL<K, V> extends Map<K, V> {
private _timeouts = new Map<K, NodeJS.Timer>();
/**
* Create a map with keys that will be automatically deleted _ttlMs
* milliseconds after they have been last set. Precision of timing
* may vary.
*/
constructor(private _ttlMs: number) {
super();
}
/**
* Set a key, with expiration.
*/
public set(key: K, value: V): this {
return this.setWithCustomTTL(key, value, this._ttlMs);
}
/**
* Set a key, with custom expiration.
*/
public setWithCustomTTL(key: K, value: V, ttlMs: number): this {
const curr = this._timeouts.get(key);
if (curr) { clearTimeout(curr); }
super.set(key, value);
this._timeouts.set(key, setTimeout(this.delete.bind(this, key), ttlMs));
return this;
}
/**
* Remove a key.
*/
public delete(key: K): boolean {
const result = super.delete(key);
const timeout = this._timeouts.get(key);
if (timeout) {
clearTimeout(timeout);
this._timeouts.delete(key);
}
return result;
}
/**
* Forcibly expire everything.
*/
public clear(): void {
for (const timeout of this._timeouts.values()) {
clearTimeout(timeout);
}
this._timeouts.clear();
super.clear();
}
}
/**
* Sometimes it is desirable to cache either fulfilled or rejected
* outcomes. This method wraps a promise so that it never throws.
* The result has an unfreeze method which, when called, is either
* fulfilled or rejected.
*/
export async function freezeError<T>(promise: Promise<T>): Promise<ErrorOrValue<T>> {
try {
const value = await promise;
return { unfreeze: async () => value };
} catch (error) {
return { unfreeze: async () => { throw error; } };
}
}
export interface ErrorOrValue<T> {
unfreeze(): Promise<T>;
}

@ -0,0 +1,83 @@
/**
* This module is a helper for asynchronous work. It allows resources acquired asynchronously to
* be conveniently and reliably released.
*
* Usage:
* (1) Implement a function `myFunc(flow: AsyncFlow)`. The `flow` argument provides some helpers:
*
* // Create a disposable, making it owned by the flow. It will be disposed when the flow
* // ends, whether successfully, on error, or by being cancelled.
* const foo = Foo.create(flow, ...);
*
* // As with Disposables in general, schedule a callback to be called when the flow ends.
* flow.onDispose(...);
*
* // Release foo from the flow's ownership, and give its ownership to another object. This way
* // `other` will be responsible for disposing foo, and not flow.
* other.autoDispose(flow.release(foo))
*
* // Abort the flow (by throwing CancelledError) if cancellation is requested. This should
* // be called after async work, in case the flow shouldn't be continued.
* checkIfCancelled();
*
* (2) Call `runner = FlowRunner.create(owner, myFunc)`. The flow will start. Once myFunc's
* promise resolves (including on failure), the objects owned by the flow will be disposed.
*
* The runner exposes the promise for when the flow ends as `runner.resultPromise`.
*
* If the runner itself is disposed, the flow will be cancelled, and disposed once it notices
* the cancellation.
*
* To replace one FlowRunner with another, put it in a grainjs Holder.
*/
import {Disposable, IDisposable} from 'grainjs';
type DisposeListener = ReturnType<Disposable["onDispose"]>;
export class CancelledError extends Error {}
export class FlowRunner extends Disposable {
public resultPromise: Promise<void>;
constructor(func: (flow: AsyncFlow) => Promise<void>) {
super();
const flow = AsyncFlow.create(null);
async function runFlow() {
try {
return await func(flow);
} finally {
flow.dispose();
}
}
this.resultPromise = runFlow();
this.onDispose(flow.cancel, flow);
}
}
export class AsyncFlow extends Disposable {
private _handles = new Map<IDisposable, DisposeListener>();
private _isCancelled = false;
public autoDispose<T extends IDisposable>(obj: T): T {
const lis = this.onDispose(obj.dispose, obj);
this._handles.set(obj, lis);
return obj;
}
public release<T extends IDisposable>(obj: T): T {
const h = this._handles.get(obj);
if (h) { h.dispose(); }
this._handles.delete(obj);
return obj;
}
public checkIfCancelled() {
if (this._isCancelled) {
throw new CancelledError('cancelled');
}
}
public cancel() {
this._isCancelled = true;
}
}

@ -0,0 +1,124 @@
import {ApiError, ApiErrorDetails} from 'app/common/ApiError';
import axios, {AxiosRequestConfig, AxiosResponse} from 'axios';
import {tbind} from './tbind';
export type ILogger = Pick<Console, 'log'|'debug'|'info'|'warn'|'error'>;
export interface IOptions {
headers?: Record<string, string>;
fetch?: typeof fetch;
newFormData?: () => FormData; // constructor for FormData depends on platform.
logger?: ILogger;
extraParameters?: Map<string, string>; // if set, add query parameters to requests.
}
/**
* Base setup class for creating a REST API client interface.
*/
export class BaseAPI {
// Count of pending requests. It is relied on by tests.
public static numPendingRequests(): number { return this._numPendingRequests; }
// Wrap a promise to add to the count of pending requests until the promise is resolved.
public static async countPendingRequest<T>(promise: Promise<T>): Promise<T> {
try {
BaseAPI._numPendingRequests++;
return await promise;
} finally {
BaseAPI._numPendingRequests--;
}
}
// Define a decorator for methods in BaseAPI or derived classes.
public static countRequest(target: unknown, propertyKey: string, descriptor: PropertyDescriptor) {
const originalMethod = descriptor.value;
descriptor.value = async function(...args: any[]) {
return BaseAPI.countPendingRequest(originalMethod.apply(this, args));
};
}
private static _numPendingRequests: number = 0;
protected fetch: typeof fetch;
protected newFormData: () => FormData;
private _headers: Record<string, string>;
private _logger: ILogger;
private _extraParameters?: Map<string, string>;
constructor(options: IOptions = {}) {
this.fetch = options.fetch || tbind(window.fetch, window);
this.newFormData = options.newFormData || (() => new FormData());
this._logger = options.logger || console;
this._headers = {
'Content-Type': 'application/json',
...options.headers
};
this._extraParameters = options.extraParameters;
}
// Make a modified request, exposed for test convenience.
public async testRequest(url: string, init: RequestInit = {}): Promise<Response> {
return this.request(url, init);
}
// Similar to request, but uses the axios library, and supports progress indicator.
@BaseAPI.countRequest
protected async requestAxios(url: string, config: AxiosRequestConfig): Promise<AxiosResponse> {
// If using with FormData in node, axios needs the headers prepared by FormData.
let headers = config.headers;
if (config.data && typeof config.data.getHeaders === 'function') {
headers = {...config.data.getHeaders(), ...headers};
}
const resp = await axios.request({
url,
withCredentials: true,
validateStatus: (status) => true, // This is more like fetch
...config,
headers,
});
if (resp.status !== 200) {
throwApiError(url, resp, resp.data);
}
return resp;
}
@BaseAPI.countRequest
protected async request(input: string, init: RequestInit = {}): Promise<Response> {
init = Object.assign({ headers: this._headers, credentials: 'include' }, init);
if (this._extraParameters) {
const url = new URL(input);
for (const [key, val] of this._extraParameters.entries()) {
url.searchParams.set(key, val);
input = url.href;
}
}
const resp = await this.fetch(input, init);
this._logger.log("Fetched", input);
if (resp.status !== 200) {
const body = await resp.json().catch(() => ({}));
throwApiError(input, resp, body);
}
return resp;
}
/**
* Make a request, and read the response as JSON. This allows counting the request as pending
* until it has been read, which is relied on by tests.
*/
@BaseAPI.countRequest
protected async requestJson(input: string, init: RequestInit = {}): Promise<any> {
return (await this.request(input, init)).json();
}
}
function throwApiError(url: string, resp: Response | AxiosResponse, body: any) {
// If the response includes details, include them into the ApiError we construct. Include
// also the error message from the server as details.userError. It's used by the Notifier.
if (!body) { body = {}; }
const details: ApiErrorDetails = body.details && typeof body.details === 'object' ? body.details : {};
if (body.error) {
details.userError = body.error;
}
throw new ApiError(`Request to ${url} failed with status ${resp.status}: ` +
`${resp.statusText} (${body.error || 'unknown cause'})`, resp.status, details);
}

@ -0,0 +1,11 @@
export interface BasketClientAPI {
/**
* Returns an array of all tableIds in this basket.
*/
getBasketTables(): Promise<string[]>;
/**
* Adds, updates or deletes a table's data to/from Grist Basket.
*/
embedTable(tableId: string, action: "add"|"update"|"delete"): Promise<void>;
}

@ -0,0 +1,72 @@
/**
* A minimal library to represent arbitrarily large integers. Unlike the many third party
* libraries, which are big, this only implements a representation and conversion to string (such
* as base 10 or base 16), so it's tiny in comparison.
*
* Big integers
* base: number - the base for the digits
* digits: number[] - digits, from least significant to most significant, in [0, base) range.
* sign: number - 1 or -1
*/
export class BigInt {
constructor(
private _base: number, // Base for the digits
private _digits: number[], // Digits from least to most significant, in [0, base) range.
private _sign: number, // +1 or -1
) {}
public copy() { return new BigInt(this._base, this._digits, this._sign); }
/** Convert to Number if there is no loss of precision, or string (base 10) otherwise. */
public toNative(): number|string {
const num = this.toNumber();
return Number.isSafeInteger(num) ? num : this.toString(10);
}
/** Convert to Number as best we can. This will lose precision beying 53 bits. */
public toNumber(): number {
let res = 0;
let baseFactor = 1;
for (const digit of this._digits) {
res += digit * baseFactor;
baseFactor *= this._base;
}
return res * (this._sign < 0 ? -1 : 1);
}
/** Like Number.toString(). Radix (or base) is an integer between 2 and 36, defaulting to 10. */
public toString(radix: number = 10): string {
const copy = this.copy();
const decimals = [];
while (copy._digits.length > 0) {
decimals.push(copy._mod(radix).toString(radix));
copy._divide(radix);
}
if (decimals.length === 0) { return "0"; }
return (this._sign < 0 ? "-" : "") + decimals.reverse().join("");
}
/** Returns the remainder when this number is divided by divisor. */
private _mod(divisor: number): number {
let res = 0;
let baseFactor = 1;
for (const digit of this._digits) {
res = (res + (digit % divisor) * baseFactor) % divisor;
baseFactor = (baseFactor * this._base) % divisor;
}
return res;
}
/** Divides this number in-place. */
private _divide(divisor: number): void {
if (this._digits.length === 0) { return; }
for (let i = this._digits.length - 1; i > 0; i--) {
this._digits[i - 1] += (this._digits[i] % divisor) * this._base;
this._digits[i] = Math.floor(this._digits[i] / divisor);
}
this._digits[0] = Math.floor(this._digits[0] / divisor);
while (this._digits.length > 0 && this._digits[this._digits.length - 1] === 0) {
this._digits.pop();
}
}
}

@ -0,0 +1,212 @@
import {BaseAPI, IOptions} from 'app/common/BaseAPI';
import {FullUser} from 'app/common/LoginSessionAPI';
import {StringUnion} from 'app/common/StringUnion';
import {addCurrentOrgToPath} from 'app/common/urlUtils';
import {BillingAccount, ManagerDelta, OrganizationWithoutAccessInfo} from 'app/common/UserAPI';
export const BillingSubPage = StringUnion('payment', 'plans');
export type BillingSubPage = typeof BillingSubPage.type;
export const BillingPage = StringUnion(...BillingSubPage.values, 'billing');
export type BillingPage = typeof BillingPage.type;
export const BillingTask = StringUnion('signUp', 'updatePlan', 'addCard', 'updateCard', 'updateAddress');
export type BillingTask = typeof BillingTask.type;
// Note that IBillingPlan includes selected fields from the Stripe plan object along with
// custom metadata fields that are present on plans we store in Stripe.
// For reference: https://stripe.com/docs/api/plans/object
export interface IBillingPlan {
id: string; // the Stripe plan id
nickname: string;
currency: string; // lowercase three-letter ISO currency code
interval: string; // billing frequency - one of day, week, month or year
amount: number; // amount in cents charged at each interval
metadata: {
family?: string; // groups plans for filtering by GRIST_STRIPE_FAMILY env variable
isStandard: boolean; // indicates that the plan should be returned by the API to be offered.
supportAvailable: boolean;
gristProduct: string; // name of grist product that should be used with this plan.
unthrottledApi: boolean;
customSubdomain: boolean;
workspaces: boolean;
maxDocs?: number; // if given, limit of docs that can be created
maxUsersPerDoc?: number; // if given, limit of users each doc can be shared with
};
trial_period_days: number|null; // Number of days in the trial period, or null if there is none.
product: string; // the Stripe product id.
}
// Stripe customer address information. Used to maintain the company address.
// For reference: https://stripe.com/docs/api/customers/object#customer_object-address
export interface IBillingAddress {
line1: string;
line2?: string;
city?: string;
state?: string;
postal_code?: string;
country?: string;
}
export interface IBillingCard {
funding?: 'credit'|'debit'|'prepaid'|'unknown';
brand?: string;
country?: string; // uppercase two-letter ISO country code
last4?: string; // last 4 digits of the card number
name?: string|null;
}
export interface IBillingSubscription {
// All standard plan options.
plans: IBillingPlan[];
// Index in the plans array of the plan currently in effect.
planIndex: number;
// Index in the plans array of the plan to be in effect after the current period end.
// Equal to the planIndex when the plan has not been downgraded or cancelled.
upcomingPlanIndex: number;
// Timestamp in milliseconds indicating when the current plan period ends.
// Null if the account is not signed up with Stripe.
periodEnd: number|null;
// Whether the subscription is in the trial period.
isInTrial: boolean;
// Value in cents remaining for the current subscription. This indicates the amount that
// will be discounted from a subscription upgrade.
valueRemaining: number;
// The payment card, or null if none is attached.
card: IBillingCard|null;
// The company address.
address: IBillingAddress|null;
// The effective tax rate of the customer for the given address.
taxRate: number;
// The current number of users with whom the paid org is shared.
userCount: number;
// The next total in cents that Stripe is going to charge (includes tax and discount).
nextTotal: number;
// Name of the discount if any.
discountName: string|null;
// Last plan we had a subscription for, if any.
lastPlanId: string|null;
// Whether there is a valid plan in effect
isValidPlan: boolean;
}
export interface IBillingOrgSettings {
name: string;
domain: string;
}
// Full description of billing account, including nested list of orgs and managers.
export interface FullBillingAccount extends BillingAccount {
orgs: OrganizationWithoutAccessInfo[];
managers: FullUser[];
}
export interface BillingAPI {
isDomainAvailable(domain: string): Promise<boolean>;
getTaxRate(address: IBillingAddress): Promise<number>;
getPlans(): Promise<IBillingPlan[]>;
getSubscription(): Promise<IBillingSubscription>;
getBillingAccount(): Promise<FullBillingAccount>;
// The signUp function takes the tokenId generated when card data is submitted to Stripe.
// See: https://stripe.com/docs/stripe-js/reference#stripe-create-token
signUp(planId: string, tokenId: string, address: IBillingAddress,
settings: IBillingOrgSettings): Promise<OrganizationWithoutAccessInfo>;
setCard(tokenId: string): Promise<void>;
removeCard(): Promise<void>;
setSubscription(planId: string, tokenId?: string): Promise<void>;
updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise<void>;
updateBillingManagers(delta: ManagerDelta): Promise<void>;
}
export class BillingAPIImpl extends BaseAPI implements BillingAPI {
constructor(private _homeUrl: string, options: IOptions = {}) {
super(options);
}
public async isDomainAvailable(domain: string): Promise<boolean> {
const resp = await this.request(`${this._url}/api/billing/domain`, {
method: 'POST',
body: JSON.stringify({ domain })
});
return resp.json();
}
public async getTaxRate(address: IBillingAddress): Promise<number> {
const resp = await this.request(`${this._url}/api/billing/tax`, {
method: 'POST',
body: JSON.stringify({ address })
});
return resp.json();
}
public async getPlans(): Promise<IBillingPlan[]> {
const resp = await this.request(`${this._url}/api/billing/plans`, {method: 'GET'});
return resp.json();
}
// Returns an IBillingSubscription
public async getSubscription(): Promise<IBillingSubscription> {
const resp = await this.request(`${this._url}/api/billing/subscription`, {method: 'GET'});
return resp.json();
}
public async getBillingAccount(): Promise<FullBillingAccount> {
const resp = await this.request(`${this._url}/api/billing`, {method: 'GET'});
return resp.json();
}
// Returns the new Stripe customerId.
public async signUp(
planId: string,
tokenId: string,
address: IBillingAddress,
settings: IBillingOrgSettings
): Promise<OrganizationWithoutAccessInfo> {
const resp = await this.request(`${this._url}/api/billing/signup`, {
method: 'POST',
body: JSON.stringify({ tokenId, planId, address, settings })
});
const parsed = await resp.json();
return parsed.data;
}
public async setSubscription(planId: string, tokenId?: string): Promise<void> {
await this.request(`${this._url}/api/billing/subscription`, {
method: 'POST',
body: JSON.stringify({ tokenId, planId })
});
}
public async removeSubscription(): Promise<void> {
await this.request(`${this._url}/api/billing/subscription`, {method: 'DELETE'});
}
public async setCard(tokenId: string): Promise<void> {
await this.request(`${this._url}/api/billing/card`, {
method: 'POST',
body: JSON.stringify({ tokenId })
});
}
public async removeCard(): Promise<void> {
await this.request(`${this._url}/api/billing/card`, {method: 'DELETE'});
}
public async updateAddress(address?: IBillingAddress, settings?: IBillingOrgSettings): Promise<void> {
await this.request(`${this._url}/api/billing/address`, {
method: 'POST',
body: JSON.stringify({ address, settings })
});
}
public async updateBillingManagers(delta: ManagerDelta): Promise<void> {
await this.request(`${this._url}/api/billing/managers`, {
method: 'PATCH',
body: JSON.stringify({delta})
});
}
private get _url(): string {
return addCurrentOrgToPath(this._homeUrl);
}
}

@ -0,0 +1,258 @@
/**
* Implements a binary indexed tree, aka Fenwick tree. See
* http://en.wikipedia.org/wiki/Fenwick_tree
*/
function BinaryIndexedTree(optSize) {
this.tree = [];
if (optSize > 0) {
this.tree.length = optSize;
for (var i = 0; i < optSize; i++) {
this.tree[i] = 0;
}
// The last valid index rounded down to the nearest power of 2.
this.mask = mostSignificantOne(this.tree.length - 1);
}
}
/**
* Returns a number that contains only the least significant one in `num`.
* @param {Number} num - Positive integer.
* @returns {Number} The least significant one in `num`, e.g. for 10110, returns 00010.
*/
function leastSignificantOne(num) {
return num & (-num);
}
BinaryIndexedTree.leastSignificantOne = leastSignificantOne;
/**
* Strips the least significant one from `num`.
* @param {Number} num - Positive integer.
* @returns {Number} `num` with the least significant one removed, e.g. for 10110, returns 10100.
*/
function stripLeastSignificantOne(num) {
return num & (num - 1);
}
BinaryIndexedTree.stripLeastSignificantOne = stripLeastSignificantOne;
function mostSignificantOne(num) {
if (num === 0) {
return 0;
}
var msb = 1;
while ((num >>>= 1)) {
msb <<= 1;
}
return msb;
}
BinaryIndexedTree.mostSignificantOne = mostSignificantOne;
/**
* Converts in-place an array of cumulative values to the original values.
* @param {Array[Number]} values - Array of cumulative values, or partial sums.
* @returns {Array[Number]} - same `values` array, with elements replaced by deltas.
* E.g. [1,3,6,10] is converted to [1,2,3,4].
*/
function cumulToValues(values) {
for (var i = values.length - 1; i >= 1; i--) {
values[i] -= values[i - 1];
}
return values;
}
BinaryIndexedTree.cumulToValues = cumulToValues;
/**
* Converts in-place an array of values to cumulative values, or partial sums.
* @param {Array[Number]} values - Array of numerical values.
* @returns {Array[Number]} - same `values` array, with elements replaced by partial sums.
* E.g. [1,2,3,4] is converted to [1,3,6,10].
*/
function valuesToCumul(values) {
for (var i = 1; i < values.length; i++) {
values[i] += values[i - 1];
}
return values;
}
BinaryIndexedTree.valuesToCumul = valuesToCumul;
/**
* @returns {Number} length of the tree.
*/
BinaryIndexedTree.prototype.size = function() {
return this.tree.length;
};
/**
* Converts the BinaryIndexedTree to a cumulative array.
* Takes time linear in the size of the array.
* @returns {Array[Number]} - array with each element a partial sum.
*/
BinaryIndexedTree.prototype.toCumulativeArray = function() {
var cumulValues = [this.tree[0]];
var len = cumulValues.length = this.tree.length;
for (var i = 1; i < len; i++) {
cumulValues[i] = this.tree[i] + cumulValues[stripLeastSignificantOne(i)];
}
return cumulValues;
};
/**
* Converts the BinaryIndexedTree to an array of individual values.
* Takes time linear in the size of the array.
* @returns {Array[Number]} - array with each element containing the value that was inserted.
*/
BinaryIndexedTree.prototype.toValueArray = function() {
return cumulToValues(this.toCumulativeArray());
};
/**
* Creates a tree from an array of cumulative values.
* Takes time linear in the size of the array.
* @param {Array[Number]} - array with each element a partial sum.
*/
BinaryIndexedTree.prototype.fillFromCumulative = function(cumulValues) {
var len = this.tree.length = cumulValues.length;
if (len > 0) {
this.tree[0] = cumulValues[0];
for (var i = 1; i < len; i++) {
this.tree[i] = cumulValues[i] - cumulValues[stripLeastSignificantOne(i)];
}
// The last valid index rounded down to the nearest power of 2.
this.mask = mostSignificantOne(this.tree.length - 1);
} else {
this.mask = 0;
}
};
/**
* Creates a tree from an array of invididual values.
* Takes time linear in the size of the array.
* @param {Array[Number]} - array with each element containing the value to insert.
*/
BinaryIndexedTree.prototype.fillFromValues = function(values) {
this.fillFromCumulative(valuesToCumul(values.slice()));
};
/**
* Reads the cumulative value at the given index. Takes time O(log(index)).
* @param {Number} index - index in the array.
* @returns {Number} - cumulative values up to and including `index`.
*/
BinaryIndexedTree.prototype.getCumulativeValue = function(index) {
var sum = this.tree[0];
while (index > 0) {
sum += this.tree[index];
index = stripLeastSignificantOne(index);
}
return sum;
};
/**
* Reads the cumulative value from start(inclusive) to end(exclusive). Takes time O(log(end)).
* @param {Number} start - start index
* @param {Number} end - end index
* @returns {Number} - cumulative values between start(inclusive) and end(exclusive)
*/
BinaryIndexedTree.prototype.getCumulativeValueRange = function(start, end) {
return this.getSumTo(end) - this.getSumTo(start);
};
/**
* Returns the sum of values up to the given index. Takes time O(log(index)).
* @param {Number} index - index in the array.
* @returns {Number} - cumulative values up to but not including `index`.
*/
BinaryIndexedTree.prototype.getSumTo = function(index) {
return (index > 0 ? this.getCumulativeValue(index - 1) : 0);
};
/**
* Returns the total of all values in the tree. Takes time O(log(N)).
* @returns {Number} - sum of all values.
*/
BinaryIndexedTree.prototype.getTotal = function() {
return this.getCumulativeValue(this.tree.length - 1);
};
/**
* Reads a single value at the given index. Takes time O(log(index)).
* @param {Number} index - index in the array.
* @returns {Number} - the value that was inserted at `index`.
*/
BinaryIndexedTree.prototype.getValue = function(index) {
var value = this.tree[index];
if (index > 0) {
var parent = stripLeastSignificantOne(index);
index--;
while (index !== parent) {
value -= this.tree[index];
index = stripLeastSignificantOne(index);
}
}
return value;
};
/**
* Updates a value at an index. Takes time O(log(table size)).
* @param {Number} index - index in the array.
* @param {Number} delta - value to add to the previous value at `index`.
*/
BinaryIndexedTree.prototype.addValue = function(index, delta) {
if (index === 0) {
this.tree[0] += delta;
} else {
while (index < this.tree.length) {
this.tree[index] += delta;
index += leastSignificantOne(index);
}
}
};
/**
* Sets a value at an index. Takes time O(log(table size)).
* @param {Number} index - index in the array.
* @param {Number} value - new value to set at `index`.
*/
BinaryIndexedTree.prototype.setValue = function(index, value) {
this.addValue(index, value - this.getValue(index));
};
/**
* Given a cumulative value, finds the first element whose inclusion reaches the value.
* E.g. for values [1,2,3,4] (cumulative [1,3,6,10]), getIndex(3) = 1, getIndex(3.1) = 2.
* @param {Number} cumulValue - cumulative value to exceed.
* @returns {Number} index - the first index such that getCumulativeValue(index) >= cumulValue.
* If cumulValue is too large, return one more than the highest valid index.
*/
BinaryIndexedTree.prototype.getIndex = function(cumulValue) {
if (this.tree.length === 0 || this.tree[0] >= cumulValue) {
return 0;
}
var index = 0;
var mask = this.mask;
var sum = this.tree[0];
while (mask !== 0) {
var testIndex = index + mask;
if (testIndex < this.tree.length && sum + this.tree[testIndex] < cumulValue) {
index = testIndex;
sum += this.tree[index];
}
mask >>>= 1;
}
return index + 1;
};
module.exports = BinaryIndexedTree;

@ -0,0 +1,7 @@
/**
* Describes the settings that a browser sends to the server.
*/
export interface BrowserSettings {
// The browser's timezone, must be one of `momet.tz.names()`.
timezone?: string;
}

@ -0,0 +1,26 @@
/**
*
* An interface for accessing the columns of a table by their
* ID in _grist_Tables_column, which is the ID used in sort specifications.
* Implementations of this interface can be supplied to SortFunc to
* sort the rows of a table according to such a specification.
*
*/
export interface ColumnGetters {
/**
*
* Takes a _grist_Tables_column ID and returns a function that maps
* rowIds to values for that column. Those values should be display
* values if available, drawn from a corresponding display column.
*
*/
getColGetter(colRef: number): ((rowId: number) => any) | null;
/**
*
* Returns a getter for the manual sort column if it is available.
*
*/
getManualSortGetter(): ((rowId: number) => any) | null;
}

@ -0,0 +1,28 @@
/**
* A base class which combines grainjs Disposable with mixed-in backbone Events. It includes the
* backbone Events methods, and when disposed, stops backbone listeners started with listenTo().
*/
import {Events as BackboneEvents, EventsHash} from 'backbone';
import {Disposable} from 'grainjs';
// In Typescript, mixins are awkward. This follows the recommendation here
// https://www.typescriptlang.org/docs/handbook/mixins.html
export class DisposableWithEvents extends Disposable implements BackboneEvents {
public on: (eventName: string|EventsHash, callback?: (...args: any[]) => void, context?: any) => any;
public off: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any;
public trigger: (eventName: string, ...args: any[]) => any;
public bind: (eventName: string, callback: (...args: any[]) => void, context?: any) => any;
public unbind: (eventName?: string, callback?: (...args: any[]) => void, context?: any) => any;
public once: (events: string, callback: (...args: any[]) => void, context?: any) => any;
public listenTo: (object: any, events: string, callback: (...args: any[]) => void) => any;
public listenToOnce: (object: any, events: string, callback: (...args: any[]) => void) => any;
public stopListening: (object?: any, events?: string, callback?: (...args: any[]) => void) => any;
// DisposableWithEvents knows also how to stop any backbone listeners started with listenTo().
constructor() {
super();
this.onDispose(this.stopListening, this);
}
}
Object.assign(DisposableWithEvents.prototype, BackboneEvents);

@ -0,0 +1,148 @@
/**
* This mirrors action definitions from sandbox/grist/actions.py
*/
import map = require('lodash/map');
export type AddRecord = ['AddRecord', string, number, ColValues];
export type BulkAddRecord = ['BulkAddRecord', string, number[], BulkColValues];
export type RemoveRecord = ['RemoveRecord', string, number];
export type BulkRemoveRecord = ['BulkRemoveRecord', string, number[]];
export type UpdateRecord = ['UpdateRecord', string, number, ColValues];
export type BulkUpdateRecord = ['BulkUpdateRecord', string, number[], BulkColValues];
export type ReplaceTableData = ['ReplaceTableData', string, number[], BulkColValues];
// This is the format in which data comes when we fetch a table from the sandbox.
export type TableDataAction = ['TableData', string, number[], BulkColValues];
export type AddColumn = ['AddColumn', string, string, ColInfo];
export type RemoveColumn = ['RemoveColumn', string, string];
export type RenameColumn = ['RenameColumn', string, string, string];
export type ModifyColumn = ['ModifyColumn', string, string, ColInfo];
export type AddTable = ['AddTable', string, ColInfoWithId[]];
export type RemoveTable = ['RemoveTable', string];
export type RenameTable = ['RenameTable', string, string];
export type DocAction = (
AddRecord |
BulkAddRecord |
RemoveRecord |
BulkRemoveRecord |
UpdateRecord |
BulkUpdateRecord |
ReplaceTableData |
TableDataAction |
AddColumn |
RemoveColumn |
RenameColumn |
ModifyColumn |
AddTable |
RemoveTable |
RenameTable
);
// type guards for convenience - see:
// https://www.typescriptlang.org/docs/handbook/advanced-types.html#user-defined-type-guards
export function isAddRecord(act: DocAction): act is AddRecord { return act[0] === 'AddRecord'; }
export function isBulkAddRecord(act: DocAction): act is BulkAddRecord { return act[0] === 'BulkAddRecord'; }
export function isRemoveRecord(act: DocAction): act is RemoveRecord { return act[0] === 'RemoveRecord'; }
export function isBulkRemoveRecord(act: DocAction): act is BulkRemoveRecord { return act[0] === 'BulkRemoveRecord'; }
export function isUpdateRecord(act: DocAction): act is UpdateRecord { return act[0] === 'UpdateRecord'; }
export function isBulkUpdateRecord(act: DocAction): act is BulkUpdateRecord { return act[0] === 'BulkUpdateRecord'; }
export function isReplaceTableData(act: DocAction): act is ReplaceTableData { return act[0] === 'ReplaceTableData'; }
export function isAddColumn(act: DocAction): act is AddColumn { return act[0] === 'AddColumn'; }
export function isRemoveColumn(act: DocAction): act is RemoveColumn { return act[0] === 'RemoveColumn'; }
export function isRenameColumn(act: DocAction): act is RenameColumn { return act[0] === 'RenameColumn'; }
export function isModifyColumn(act: DocAction): act is ModifyColumn { return act[0] === 'ModifyColumn'; }
export function isAddTable(act: DocAction): act is AddTable { return act[0] === 'AddTable'; }
export function isRemoveTable(act: DocAction): act is RemoveTable { return act[0] === 'RemoveTable'; }
export function isRenameTable(act: DocAction): act is RenameTable { return act[0] === 'RenameTable'; }
const SCHEMA_ACTIONS = new Set(['AddTable', 'RemoveTable', 'RenameTable', 'AddColumn',
'RemoveColumn', 'RenameColumn', 'ModifyColumn']);
/**
* Determines whether a given action is a schema action or not.
*/
export function isSchemaAction(action: DocAction): boolean {
return SCHEMA_ACTIONS.has(action[0]);
}
/**
* Returns the tableId from the action.
*/
export function getTableId(action: DocAction): string {
return action[1]; // It happens to always be in the same position in the action tuple.
}
// Helper types used in the definitions above.
export type CellValue = number|string|boolean|null|[string, any?];
export interface ColValues { [colId: string]: CellValue; }
export interface BulkColValues { [colId: string]: CellValue[]; }
export interface ColInfoMap { [colId: string]: ColInfo; }
export interface ColInfo {
type: string;
isFormula: boolean;
formula: string;
}
export interface ColInfoWithId extends ColInfo {
id: string;
}
export interface RowRecord {
id: number;
[colId: string]: CellValue;
}
// Multiple records in column-oriented format, i.e. same as BulkColValues but with a mandatory
// 'id' column. This is preferred over TableDataAction in external APIs.
export interface TableColValues {
id: number[];
[colId: string]: CellValue[];
}
// Both UserActions and DocActions are represented as [ActionName, ...actionArgs].
// TODO I think it's better to represent DocAction as a Buffer containing the marshalled action.
export type UserAction = Array<string|number|object|boolean|null|undefined>;
/**
* Gives a description for an action which involves setting values to a selection.
* @param {Array} action - The (Bulk)AddRecord/(Bulk)UpdateRecord action to describe.
* @param {Boolean} optExcludeVals - Indicates whether the values should be excluded from
* the description.
*/
export function getSelectionDesc(action: UserAction, optExcludeVals: boolean): string {
const table = action[1];
const rows = action[2];
const colValues: number[] = action[3] as any; // TODO: better typing - but code may evaporate
const columns = map(colValues, (values, col) => optExcludeVals ? col : `${col}: ${values}`);
const s = typeof rows === 'object' ? 's' : '';
return `table ${table}, row${s} ${rows}; ${columns.join(", ")}`;
}
// Convert from TableColValues (used by DocStorage and external APIs) to TableDataAction (used
// mainly by the sandbox).
export function toTableDataAction(tableId: string, colValues: TableColValues): TableDataAction {
const colData = {...colValues}; // Make a copy to avoid changing passed-in arguments.
const rowIds: number[] = colData.id;
delete colData.id;
return ['TableData', tableId, rowIds, colData];
}
// Convert from TableDataAction (used mainly by the sandbox) to TableColValues (used by DocStorage
// and external APIs).
export function fromTableDataAction(tableData: TableDataAction): TableColValues {
const rowIds: number[] = tableData[2];
const colValues: BulkColValues = tableData[3];
return {id: rowIds, ...colValues};
}

@ -0,0 +1,121 @@
/**
* DocData maintains all underlying data for a Grist document, knows how to load it,
* subscribes to actions which change it, and forwards those actions to individual tables.
* It also provides the interface to apply actions to data.
*/
import {schema} from 'app/common/schema';
import fromPairs = require('lodash/fromPairs');
import groupBy = require('lodash/groupBy');
import {ActionDispatcher} from './ActionDispatcher';
import {BulkColValues, ColInfo, ColInfoWithId, ColValues, DocAction,
RowRecord, TableDataAction} from './DocActions';
import {ColTypeMap, TableData} from './TableData';
type FetchTableFunc = (tableId: string) => Promise<TableDataAction>;
export class DocData extends ActionDispatcher {
private _tables: Map<string, TableData> = new Map();
constructor(private _fetchTableFunc: FetchTableFunc, metaTableData: {[tableId: string]: TableDataAction}) {
super();
// Create all meta tables, and populate data we already have.
for (const tableId in schema) {
if (schema.hasOwnProperty(tableId)) {
const colTypes: ColTypeMap = (schema as any)[tableId];
this._tables.set(tableId, this.createTableData(tableId, metaTableData[tableId], colTypes));
}
}
// Build a map from tableRef to [columnRecords]
const colsByTable = groupBy(this._tables.get('_grist_Tables_column')!.getRecords(), 'parentId');
for (const t of this._tables.get('_grist_Tables')!.getRecords()) {
const tableId = t.tableId as string;
const colRecords: RowRecord[] = colsByTable[t.id] || [];
const colTypes = fromPairs(colRecords.map(c => [c.colId, c.type]));
this._tables.set(tableId, this.createTableData(tableId, null, colTypes));
}
}
/**
* Creates a new TableData object. A derived class may override to return an object derived from TableData.
*/
public createTableData(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap): TableData {
return new TableData(tableId, tableData, colTypes);
}
/**
* Returns the TableData object for the requested table.
*/
public getTable(tableId: string): TableData|undefined {
return this._tables.get(tableId);
}
/**
* Returns an unsorted list of all tableIds in this doc, including both metadata and user tables.
*/
public getTables(): ReadonlyMap<string, TableData> {
return this._tables;
}
/**
* Fetches the data for tableId if needed, and returns a promise that is fulfilled when the data
* is loaded.
*/
public fetchTable(tableId: string, force?: boolean): Promise<void> {
const table = this._tables.get(tableId);
if (!table) { throw new Error(`DocData.fetchTable: unknown table ${tableId}`); }
return (!table.isLoaded || force) ? table.fetchData(this._fetchTableFunc) : Promise.resolve();
}
/**
* Handles an action received from the server, by forwarding it to the appropriate TableData
* object.
*/
public receiveAction(action: DocAction): void {
// Look up TableData before processing the action in case we rename or remove it.
const tableId: string = action[1];
const table = this._tables.get(tableId);
this.dispatchAction(action);
// Forward all actions to per-table TableData objects.
if (table) {
table.receiveAction(action);
}
}
// ---- The following methods implement ActionDispatcher interface ----
protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void {
const colTypes = fromPairs(columns.map(c => [c.id, c.type]));
this._tables.set(tableId, this.createTableData(tableId, null, colTypes));
}
protected onRemoveTable(action: DocAction, tableId: string): void {
this._tables.delete(tableId);
}
protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void {
const table = this._tables.get(oldTableId);
if (table) {
this._tables.set(newTableId, table);
this._tables.delete(oldTableId);
}
}
// tslint:disable:no-empty
protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {}
protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {}
protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {}
protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
protected onBulkRemoveRecord(action: DocAction, tableId: string, rowIds: number[]) {}
protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {}
protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {}
protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {}
protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {}
protected onModifyColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {}
}

@ -0,0 +1,83 @@
import {ActionGroup} from 'app/common/ActionGroup';
import {TableDataAction} from 'app/common/DocActions';
import {LocalPlugin} from 'app/common/plugin';
import {StringUnion} from 'app/common/StringUnion';
// Possible flavors of items in a list of documents.
export type DocEntryTag = ''|'sample'|'invite'|'shared';
export const OpenDocMode = StringUnion(
'default', // open doc with user's maximal access level
'fork', // open doc limited to view access (if user has at least that level of access)
'view' // as for 'view', but suggest a fork on any attempt to edit
);
export type OpenDocMode = typeof OpenDocMode.type;
/**
* Represents an entry in the DocList.
*/
export interface DocEntry {
docId?: string; // Set for shared docs and invites
name: string;
mtime?: Date;
size?: number;
tag: DocEntryTag;
senderName?: string;
senderEmail?: string;
}
export interface DocCreationInfo {
id: string;
title: string;
}
/**
* This documents the members of the structure returned when a local
* grist document is opened.
*/
export interface OpenLocalDocResult {
docFD: number;
clientId: string; // the docFD is meaningful only in the context of this session
doc: {[tableId: string]: TableDataAction};
log: ActionGroup[];
plugins: LocalPlugin[];
}
export interface DocListAPI {
/**
* Returns a all known Grist documents and document invites to show in the doc list.
*/
getDocList(): Promise<{docs: DocEntry[], docInvites: DocEntry[]}>;
/**
* Creates a new document, fetches it, and adds a table to it. Returns its name.
*/
createNewDoc(): Promise<string>;
/**
* Makes a copy of the given sample doc. Returns the name of the new document.
*/
importSampleDoc(sampleDocName: string): Promise<string>;
/**
* Processes an upload, containing possibly multiple files, to create a single new document, and
* returns the new document's name.
*/
importDoc(uploadId: number): Promise<string>;
/**
* Deletes a Grist document. Returns the name of the deleted document. If `deletePermanently` is
* true, the doc is deleted permanently rather than just moved to the trash.
*/
deleteDoc(docName: string, deletePermanently: boolean): Promise<string>;
/**
* Renames a document.
*/
renameDoc(oldName: string, newName: string): Promise<void>;
/**
* Opens a document, loads it, subscribes to its userAction events, and returns its metadata.
*/
openDoc(userDocName: string, openMode?: OpenDocMode): Promise<OpenLocalDocResult>;
}

@ -0,0 +1,73 @@
/**
* Types for encrypted ActionBundles that get sent between instances and hub.
*/
import {ActionInfo, Envelope} from 'app/common/ActionBundle';
import {DocAction} from 'app/common/DocActions';
// Type representing a point in time as milliseconds since Epoch.
export type Timestamp = number;
// Type representing binary data encoded as a base64 string.
export type Base64String = string;
// Metadata about a symmetric encryption key.
export interface KeyInfo {
firstActionNum: number; // ActionNum of first action for which this key was used.
firstUsedTime: Timestamp; // Timestamp of first action for which this key was used.
}
// Encrypted symmetric key with metadata, sent from hub to instance with each envelope.
export interface EncKeyInfo extends KeyInfo {
encryptedKey: Base64String; // Symmetric key encrypted with the recipient's public key.
}
// Bundle of encryptions of the symmetric key. Note that the hub will store EncKeyBundles for
// lookup, indexed by the combination {recipients: string[], firstActionNum: number}.
export interface EncKeyBundle extends KeyInfo {
encryptedKeys: {
// Map of instanceId to the symmetric key encrypted with that instance's public key.
// A single symmetric key is used for all, and only present here in encrypted form.
[instanceId: string]: Base64String;
};
}
// This allows reorganizing ActionBundle by envelope while preserving order information for
// actions. E.g. if ActionBundle contains {stored: [(0,A), (1,B), (2,C), (0,D)], then we'll have:
// - in envelopes 0: {stored: [[0, A], [3, D]]}
// - in envelopes 1: {stored: [[1, B]]}
// - in envelopes 2: {stored: [[2, C]]}
// Then recipients of multiple envelopes can sort actions by index to get their correct order.
export interface DecryptedEnvelopeContent {
info?: ActionInfo;
// number is the index into the bundle-wide array of 'stored' or 'calc' DocActions.
stored: Array<[number, DocAction]>;
calc: Array<[number, DocAction]>;
}
export type DecryptedEnvelope = Envelope & DecryptedEnvelopeContent;
// Sent from instance to hub.
export interface EncEnvelopeToHub extends Envelope {
encKeyReused?: number; // If reusing a key, firstActionNum of the key being reused.
encKeyBundle?: EncKeyBundle; // If created a new key, its encryption for all recipients.
content: Base64String; // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string.
}
// Sent from hub to instance.
export interface EncEnvelopeFromHub extends Envelope {
encKeyInfo: EncKeyInfo;
content: Base64String; // Marshalled and encrypted DecryptedEnvelopeContent as a base64 string.
}
// EncActionBundle is an encrypted version of ActionBundle. It comes in two varieties, one for
// sending ActionBundle to the hub, and one for receiving from the hub.
export interface EncActionBundle<EncEnvelope> {
actionNum: number;
actionHash: string|null;
parentActionHash: string|null;
envelopes: EncEnvelope[];
}
export type EncActionBundleToHub = EncActionBundle<EncEnvelopeToHub>;
export type EncActionBundleFromHub = EncActionBundle<EncEnvelopeFromHub>;

@ -0,0 +1,23 @@
import {OpenDocMode} from 'app/common/DocListAPI';
interface ErrorDetails {
status?: number;
accessMode?: OpenDocMode;
}
/**
*
* An error with a human-readable message and a machine-readable code.
* Makes it easier to change the human-readable message without breaking
* error handlers.
*
*/
export class ErrorWithCode extends Error {
public accessMode?: OpenDocMode;
public status?: number;
constructor(public code: string, message: string, details: ErrorDetails = {}) {
super(message);
this.status = details.status;
this.accessMode = details.accessMode;
}
}

@ -0,0 +1,45 @@
// A product is essentially a list of flags and limits that we may enforce/support.
export interface Features {
vanityDomain?: boolean; // are user-selected domains allowed (unenforced) (default: true)
workspaces?: boolean; // are workspaces shown in web interface (default: true)
// (this was intended as something we can turn off to shut down
// web access to content while leaving access to billing)
/**
* Some optional limits. Since orgs can change plans, limits will typically be checked
* at the point of creation. E.g. adding someone new to a document, or creating a
* new document. If, after an operation, the limit would be exceeded, that operation
* is denied. That means it is possible to exceed limits if the limits were not in
* place when shares/docs were originally being added. The action that would need
* to be taken when infringement is pre-existing is not so obvious.
*/
maxSharesPerDoc?: number; // Maximum number of users that can be granted access to a
// particular doc. Doesn't count users granted access at
// workspace or organization level. Doesn't count billable
// users if applicable (default: unlimited)
maxSharesPerDocPerRole?: {[role: string]: number}; // As maxSharesPerDoc, but
// for specific roles. Roles are named as in app/common/roles.
// Applied independently to maxSharesPerDoc.
// (default: unlimited)
maxSharesPerWorkspace?: number; // Maximum number of users that can be granted access to
// a particular workspace. Doesn't count users granted access
// at organizational level, or billable users (default: unlimited)
maxDocsPerOrg?: number; // Maximum number of documents allowed per org.
// (default: unlimited)
maxWorkspacesPerOrg?: number; // Maximum number of workspaces allowed per org.
// (default: unlimited)
readOnlyDocs?: boolean; // if set, docs can only be read, not written.
}
// Check whether it is possible to add members at the org level. There's no flag
// for this right now, it isn't enforced at the API level, it is just a bluff.
// For now, when maxWorkspacesPerOrg is 1, we should assume members can't be added
// to org (even though this is not enforced).
export function canAddOrgMembers(features: Features): boolean {
return features.maxWorkspacesPerOrg !== 1;
}

@ -0,0 +1,72 @@
/**
*
* This represents a formula supported under SQL for on-demand tables. This is currently
* a very small subset of the formulas supported by the data engine for regular tables.
*
* The following kinds of formula are supported:
* $refColId.colId [where colId is not itself a formula]
* $colId [where colId is not itself a formula]
* NNN [a non-negative integer]
* TODO: support a broader range of formula, by adding a parser or reusing Python parser.
* An argument for reusing Python parser: wwe already do substantial parsing of the formula code.
* E.g. Python does such amazing things as handle updating the formula when any of the columns
* referred to in Foo.lookup(bar=$baz).blah get updated.
*
*/
export type Formula = LiteralNumberFormula | ColumnFormula | ForeignColumnFormula | FormulaError;
// A simple copy of another column. E.g. "$Person"
export interface ColumnFormula {
kind: 'column';
colId: string;
}
// A copy of a column in another table (via a reference column). E.g. "$Person.FirstName"
export interface ForeignColumnFormula {
kind: 'foreignColumn';
colId: string;
refColId: string;
}
export interface LiteralNumberFormula {
kind: 'literalNumber';
value: number;
}
// A formula that couldn't be parsed.
export interface FormulaError {
kind: 'error';
msg: string;
}
/**
* Convert a string to a parsed formula. Regexes are adequate for the very few
* supported formulas, but once the syntax is at all flexible a proper parser will
* be needed. In principle, it might make sense to support python syntax, for
* compatibility with the data engine, but compatibility in corner cases will be
* fiddly given underlying differences between sqlite and python.
*/
export function parseFormula(txt: string): Formula {
// Formula of form: $x.y
let m = txt.match(/^\$([a-z]\w*)\.([a-z]\w*)$/i);
if (m) {
return {kind: 'foreignColumn', refColId: m[1], colId: m[2]};
}
// Formula of form: $x
m = txt.match(/^\$([a-z][a-z_0-9]*)$/i);
if (m) {
return {kind: 'column', colId: m[1]};
}
// Formula of form: NNN
m = txt.match(/^[0-9]+$/);
if (m) {
const value = parseInt(txt, 10);
if (isNaN(value)) { return {kind: 'error', msg: 'Cannot parse integer'}; }
return {kind: 'literalNumber', value};
}
// Everything else is an error.
return {kind: 'error', msg: 'Formula not supported'};
}

@ -0,0 +1,84 @@
import {BasketClientAPI} from 'app/common/BasketClientAPI';
import {DocListAPI} from 'app/common/DocListAPI';
import {LoginSessionAPI} from 'app/common/LoginSessionAPI';
import {EmailResult, Invite} from 'app/common/sharing';
import {UserConfig} from 'app/common/UserConfig';
export interface GristServerAPI extends
DocListAPI,
LoginSessionAPI,
BasketClientAPI,
ServerMetricsAPI,
UserAPI,
SharingAPI,
MiscAPI {}
interface ServerMetricsAPI {
/**
* Registers the list of client metric names. The calls to pushClientMetrics() send metric
* values as an array parallel to this list of names.
*/
registerClientMetrics(clientMetricsList: string[]): Promise<void>;
/**
* Sends bucketed client metric data to the server. The .values arrays contain one value for
* each of the registered metric names, as a parallel array.
*/
pushClientMetrics(clientBuckets: Array<{startTime: number, values: number[]}>): Promise<void>;
}
interface UserAPI {
/**
* Gets the Grist configuration from the server.
*/
getConfig(): Promise<UserConfig>;
/**
* Updates the user configuration and saves it to the server.
* @param {Object} config - Configuration object to save.
* @returns {Promise:Object} Configuration object as persisted by the server. You can use it to
* validate the configuration.
*/
updateConfig(config: UserConfig): Promise<UserConfig>;
/**
* Re-load plugins.
*/
reloadPlugins(): Promise<void>;
}
interface SharingAPI {
/**
* Looks up a user account by email, return an object with basic user profile information.
*/
lookupEmail(email: string): Promise<EmailResult|null>;
/**
* Fetches and saves invites from the hub which are not already present locally.
* Returns the new invites from the hub only.
*/
getNewInvites(): Promise<Invite[]>;
/**
* Fetches local invites and marks them all as read.
*/
getLocalInvites(): Promise<Invite[]>;
/**
* Marks the stored local invite belonging to the calling instance as ignored.
* Called when the user declines an invite.
*/
ignoreLocalInvite(docId: string): Promise<void>;
/**
* Downloads a shared doc by creating a new doc and applying the snapshot actions associated
* with the given docId on the sharing hub. Must be called from a logged in account and instance
* invited to download the doc. Returns the actual non-conflicting docName used.
*/
downloadSharedDoc(docId: string, docName: string): Promise<string>;
}
interface MiscAPI {
showItemInFolder(docName: string): Promise<void>;
}

@ -0,0 +1,119 @@
/**
* InactivityTimer allows to set a function that executes after a certain time of
* inactivity. Activities can be of two kinds: synchronous or asynchronous. Asynchronous activities,
* are handle with the `disableUntiFinish` method that takes in a Promise and makes sure that the
* timer does not start before the promise resolves. Synchroneous activities are monitored with the
* `ping` method which resets the timer if called during inactivity.
*
* Timer won't start before any activity happens, but you may simply call ping() after construction
* to start it. After cb is called, timer is disabled but enabled again if there is more activity.
*
* Example usage: InactivityTimer is used internally for implementing the plugins' component
* deactivation after a certain time of inactivity.
*
*/
// important to explicitly import this, or webpack --watch gets confused.
import {clearTimeout, setTimeout} from "timers";
export class InactivityTimer {
private _timeout?: NodeJS.Timer | null;
private _counter: number = 0;
private _enabled: boolean = true;
constructor(private _callback: () => void, private _delay: number) {}
// Returns the delay used by InactivityTimer, in ms.
public getDelay(): number {
return this._delay;
}
// Sets a different delay to use, in ms.
public setDelay(delayMs: number): void {
this._delay = delayMs;
this.ping();
}
/**
* Enable the InactivityTimer and schedule the callback.
*/
public enable(): void {
this._enabled = true;
this.ping();
}
/**
* Clears the timeout and prevents the callback from being called until enable() is called.
*/
public disable(): void {
this._enabled = false;
this._clearTimeout();
}
/**
* Returns whether the InactivityTimer is enabled. If not, the callback will not be scheduled.
*/
public isEnabled(): boolean {
return this._enabled;
}
/**
* Whether the callback is currently scheduled, and would trigger if there is no activity and if
* it's not disabled before it triggers.
*/
public isScheduled(): boolean {
return Boolean(this._timeout);
}
/**
* Resets the timer if called during inactivity.
*/
public ping() {
if (!this._counter && this._enabled) {
this._setTimeout();
}
}
/**
* The `disableUntilFinish` method takes in a promise and makes sure the timer won't start before
* it resolves. It returns a promise that resolves to the same object.
*/
public async disableUntilFinish<T>(promise: Promise<T>): Promise<T> {
this._beginActivity();
try {
return await promise;
} finally {
this._endActivity();
}
}
private _beginActivity() {
this._counter++;
this._clearTimeout();
}
private _endActivity() {
this._counter = Math.max(this._counter - 1, 0);
this.ping();
}
private _clearTimeout() {
if (this._timeout) {
clearTimeout(this._timeout);
this._timeout = null;
}
}
private _setTimeout() {
this._clearTimeout();
this._timeout = setTimeout(() => this._onTimeoutTriggered(), this._delay);
}
private _onTimeoutTriggered() {
this._clearTimeout();
// _counter is set to 0, even if there's no reason why it should be any thing else.
this._counter = 0;
this._callback();
}
}

@ -0,0 +1,222 @@
/**
* A class for scheduling a particular operation on resources
* identified by a key. For operations which should be applied
* some time after an event.
*/
export class KeyedOps {
private _operations = new Map<string, OperationStatus>(); // status of operations
private _history = new Map<string, OperationHistory>(); // history of operations
// (will accumulate without limit)
private _changed = new Set<string>(); // set when key needs an operation
private _operating = new Set<string>(); // set when operation is in progress for key
/**
* Provide a function to apply operation, and some optional
* parameters.
*
* - delayBeforeOperationMs: if set, a call to addOperation(key) will have
* a delayed effect. It will schedule (or reschedule) the operation to occur
* after this interval. If the operation is currently in progress, it will
* get rerun after it completes.
*
* - minDelaybetweenOperationsMs: is set, scheduling for operations will have
* additional delays inserted as necessary to keep this minimal delay between
* the start of successive operations.
*
* - retry: if `retry` is set, the operation will be retried
* indefinitely with a rather primitive retry mechanism -
* otherwise no attempt is made to retry failures.
*
* - logError: called when errors occur, with a count of number of failures so
* far.
*/
constructor(private _op: (key: string) => Promise<void>, private _options: {
delayBeforeOperationMs?: number,
minDelayBetweenOperationsMs?: number,
retry?: boolean,
logError?: (key: string, failureCount: number, err: Error) => void
}) {
}
/**
* Request an operation be done (eventually) on the specified resourse.
*/
public addOperation(key: string) {
this._changed.add(key);
this._schedule(key);
}
/**
* Check whether any work is scheduled or in progress.
*/
public hasPendingOperations() {
return this._changed.size > 0 || this._operating.size > 0;
}
/**
* Check whether any work is scheduled or in progress for a specific resource.
*/
public hasPendingOperation(key: string) {
return this._changed.has(key) || this._operating.has(key);
}
/**
* Take all scheduled operations and re-schedule them for right now. Useful
* when shutting down. Affects retries. Cannot be undone. Returns immediately.
*/
public expediteOperations() {
this._options.delayBeforeOperationMs = 0;
this._options.minDelayBetweenOperationsMs = 0;
for (const op of this._operations.values()) {
if (op.timeout) {
this._schedule(op.key, true);
}
}
}
/**
* Wait for all operations to complete. This makes most sense to use during
* shutdown - otherwise it might be a very long wait to reach a moment where
* there are no operations.
*/
public async wait(logRepeat?: (count: number) => void) {
let repeats: number = 0;
while (this.hasPendingOperations()) {
if (repeats && logRepeat) { logRepeat(repeats); }
await Promise.all([...this._operating.keys(), ...this._changed.keys()]
.map(key => this.expediteOperationAndWait(key)));
repeats++;
}
}
/**
* Re-schedules any pending operation on a resource for right now. Returns
* when operations on the resource are complete. Does not affect retries.
*/
public async expediteOperationAndWait(key: string) {
const status = this._getOperationStatus(key);
if (status.promise) {
await status.promise;
return;
}
const callback = new Promise((resolve) => {
status.callbacks.push(resolve);
});
this._schedule(key, true);
await callback;
}
/**
* Schedule an operation for a resource.
* If the operation is already in progress, we do nothing.
* If the operation has not yet happened, it is rescheduled.
* If `immediate` is set, the operation is scheduled with no delay.
*/
private _schedule(key: string, immediate: boolean = false) {
const status = this._getOperationStatus(key);
if (status.promise) { return; }
if (status.timeout) {
clearTimeout(status.timeout);
delete status.timeout;
}
let ticks = this._options.delayBeforeOperationMs || 0;
const {lastStart} = this._getOperationHistory(key);
if (lastStart && this._options.minDelayBetweenOperationsMs && !immediate) {
ticks = Math.max(ticks, lastStart + this._options.minDelayBetweenOperationsMs - Date.now());
}
// Primitive slow-down on retries.
// Will do nothing if neither delayBeforeOperationMs nor minDelayBetweenOperationsMs
// are set.
ticks *= 1 + Math.min(5, status.failures);
status.timeout = setTimeout(() => this._update(key), immediate ? 0 : ticks);
}
private _getOperationStatus(key: string): OperationStatus {
let status = this._operations.get(key);
if (!status) {
status = {
key,
failures: 0,
callbacks: []
};
this._operations.set(key, status);
}
return status;
}
private _getOperationHistory(key: string): OperationHistory {
let hist = this._history.get(key);
if (!hist) {
hist = {};
this._history.set(key, hist);
}
return hist;
}
// Implement the next scheduled operation for a resource.
private _update(key: string) {
const status = this._getOperationStatus(key);
delete status.timeout;
// We don't have to do anything if there have been no changes.
if (!this._changed.has(key)) { return; }
// We don't have to do anything (yet) if an operation is already in progress.
if (status.promise) { return; }
// Switch status from changed to operating.
this._changed.delete(key);
this._operating.add(key);
const history = this._getOperationHistory(key);
history.lastStart = Date.now();
// Store a promise for the operation.
status.promise = this._op(key).then(() => {
// Successful push! Reset failure count, notify callbacks.
status.failures = 0;
status.callbacks.forEach(callback => callback());
status.callbacks = [];
}).catch(err => {
// Operation failed. Increment failure count, notify callbacks.
status.failures++;
if (this._options.retry) {
this._changed.add(key);
}
if (this._options.logError) {
this._options.logError(key, status.failures, err);
}
status.callbacks.forEach(callback => callback(err));
status.callbacks = [];
}).then(() => {
// Clean up and schedule follow-up if necessary.
this._operating.delete(key);
delete status.promise;
if (this._changed.has(key)) {
this._schedule(key);
} else {
// No event information left to track, we can delete our OperationStatus entry.
if (status.failures === 0 && !status.timeout) {
this._operations.delete(key);
}
}
});
}
}
/**
* Status of an operation.
*/
interface OperationStatus {
timeout?: NodeJS.Timeout; // a timeout for a scheduled future operation
promise?: Promise<void>; // a promise for an operation that is under way
key: string; // the operation key
failures: number; // consecutive number of times the operation has failed
callbacks: Array<(err?: Error) => void>; // callbacks for notifications when op is done/fails
}
/**
* History of an operation.
*/
interface OperationHistory {
lastStart?: number; // last time operation was started, in ms since epoch
}

@ -0,0 +1,27 @@
// User profile info for the user. When using Cognito, it is fetched during login.
export interface UserProfile {
email: string;
name: string;
picture?: string|null; // when present, a url to a public image of unspecified dimensions.
anonymous?: boolean; // when present, asserts whether user is anonymous (not authorized).
loginMethod?: 'Google'|'Email + Password';
}
// User profile including user id. All information in it should
// have been validated against database.
export interface FullUser extends UserProfile {
id: number;
}
export interface LoginSessionAPI {
/**
* Logs out by clearing all data in the session store besides the session cookie itself.
* Broadcasts the logged out state to all clients.
*/
logout(): Promise<void>;
/**
* Replaces the user profile object in the session and broadcasts the new profile to all clients.
*/
updateProfile(profile: UserProfile): Promise<void>;
}

@ -0,0 +1,31 @@
import {parseSubdomain} from 'app/common/gristUrls';
// This interface is used by the standalone login-connect tool for knowing where to redirect to,
// by Client.ts to construct this info, and by CognitoClient to decide what to do.
export interface LoginState {
// Locally-running Grist uses localPort, while hosted uses subdomain. Login-connect uses this to
// redirect back to the localhost or to the subdomain.
localPort?: number;
subdomain?: string;
baseDomain?: string; // the domain with the (left-most) subdomain removed, e.g. ".getgrist.com".
// undefined on localhost.
// Standalone version sets clientId, used later to find the LoginSession. Hosted and dev
// versions rely on the browser cookies instead, specifically on the session cookie.
clientId?: string;
// Hosted and dev versions set redirectUrl and redirect to it when login or logout completes.
// Standalone version omits redirectUrl, and serves a page which closes the window.
redirectUrl?: string;
}
/// Allowed localhost addresses.
export const localhostRegex = /^localhost(?::(\d+))?$/i;
export function getLoginState(reqHost: string): LoginState|null {
const {org, base} = parseSubdomain(reqHost);
const matchPort = localhostRegex.exec(reqHost);
return org ? {subdomain: org, baseDomain: base} :
matchPort ? {localPort: matchPort[1] ? parseInt(matchPort[1], 10) : 80} : null;
}

@ -0,0 +1,294 @@
const gutil = require('./gutil');
const {arrayToString, stringToArray} = require('./arrayToString');
/**
* Class for a dynamic memory buffer. You can optionally pass the number of bytes
* to reserve initially.
*/
function MemBuffer(optBytesToReserve) {
this.buffer = new ArrayBuffer(optBytesToReserve || 64);
this.asArray = new Uint8Array(this.buffer);
this.asDataView = new DataView(this.buffer);
this.startPos = 0;
this.endPos = 0;
}
// These are defined in gutil now because they are used there (and to avoid a circular import),
// but were originally defined in MemBuffer and various code still uses them as MemBuffer members.
MemBuffer.arrayToString = arrayToString;
MemBuffer.stringToArray = stringToArray;
/**
* Returns the number of bytes in the buffer.
*/
MemBuffer.prototype.size = function() {
return this.endPos - this.startPos;
};
/**
* Returns the number of bytes reserved in the buffer for data. This is at least size().
*/
MemBuffer.prototype.reserved = function() {
return this.buffer.byteLength - this.startPos;
};
/**
* Reserves enough space in the buffer to hold a nbytes of data, counting the data already in the
* buffer.
*/
MemBuffer.prototype.reserve = function(nbytes) {
if (this.startPos + nbytes > this.buffer.byteLength) {
var origArray = new Uint8Array(this.buffer, this.startPos, this.size());
if (nbytes > this.buffer.byteLength) {
// At least double the size of the buffer.
var newBytes = Math.max(nbytes, this.buffer.byteLength * 2);
this.buffer = new ArrayBuffer(newBytes);
this.asArray = new Uint8Array(this.buffer);
this.asDataView = new DataView(this.buffer);
}
// If we did not allocate more space, this line will just move data to the beginning.
this.asArray.set(origArray);
this.endPos = this.size();
this.startPos = 0;
}
};
/**
* Clears the buffer.
*/
MemBuffer.prototype.clear = function() {
this.startPos = this.endPos = 0;
// If the buffer has grown somewhat big, use this chance to free the memory.
if (this.buffer.byteLength >= 256 * 1024) {
this.buffer = new ArrayBuffer(64);
this.asArray = new Uint8Array(this.buffer);
this.asDataView = new DataView(this.buffer);
}
};
/**
* Returns a Uint8Array viewing all the data in the buffer. It is the caller's responsibility to
* make a copy if needed to avoid it being affected by subsequent changes to the buffer.
*/
MemBuffer.prototype.asByteArray = function() {
return new Uint8Array(this.buffer, this.startPos, this.size());
};
/**
* Converts all buffer data to string using UTF8 encoding.
* This is mainly for testing.
*/
MemBuffer.prototype.toString = function() {
return arrayToString(this.asByteArray());
};
/*
* (Dmitry 2017/03/20. Some unittests that include timing (e.g. Sandbox.js measuring serializing
* of data using marshal.js) indicated that gutil.arrayCopyForward gets deoptimized. Narrowing it
* down, I found it was because it was used with different argument types (Arrays, Buffers,
* Uint8Arrays). To keep it optimized, we'll use a cloned copy of arrayCopyForward (for copying to
* a Uint8Array) in this module.
*/
let arrayCopyForward = gutil.cloneFunc(gutil.arrayCopyForward);
/**
* Appends an array of bytes to this MemBuffer.
* @param {Uint8Array|Buffer} bytes: Array of bytes to append. May be a Node Buffer.
*/
MemBuffer.prototype.writeByteArray = function(bytes) {
// Note that the implementation is identical for Uint8Array and a Node Buffer.
this.reserve(this.size() + bytes.length);
arrayCopyForward(this.asArray, this.endPos, bytes, 0, bytes.length);
this.endPos += bytes.length;
};
/**
* Encodes the given string in UTF8 and appends to the buffer.
*/
if (typeof TextDecoder !== 'undefined') {
MemBuffer.prototype.writeString = function(string) {
this.writeByteArray(stringToArray(string));
};
} else {
// We can write faster without using stringToArray, to avoid allocating new buffers.
// We'll encode data in chunks reusing a single buffer. The buffer is a multiple of chunk size
// to have enough space for multi-byte characters.
var encodeChunkSize = 1024;
var encodeBufferPad = Buffer.alloc(encodeChunkSize * 4);
MemBuffer.prototype.writeString = function(string) {
// Reserve one byte per character initially (common case), but we'll reserve more below as
// needed.
this.reserve(this.size() + string.length);
for (var i = 0; i < string.length; i += encodeChunkSize) {
var bytesWritten = encodeBufferPad.write(string.slice(i, i + encodeChunkSize));
this.reserve(this.size() + bytesWritten);
arrayCopyForward(this.asArray, this.endPos, encodeBufferPad, 0, bytesWritten);
this.endPos += bytesWritten;
}
};
}
function makeWriteFunc(typeName, bytes, optLittleEndian) {
var setter = DataView.prototype['set' + typeName];
return function(value) {
this.reserve(this.size() + bytes);
setter.call(this.asDataView, this.endPos, value, optLittleEndian);
this.endPos += bytes;
};
}
/**
* The following methods append a value of the given type to the buffer.
* These are analogous to Node Buffer's write* family of methods.
*/
MemBuffer.prototype.writeInt8 = makeWriteFunc('Int8', 1);
MemBuffer.prototype.writeUint8 = makeWriteFunc('Uint8', 1);
MemBuffer.prototype.writeInt16LE = makeWriteFunc('Int16', 2, true);
MemBuffer.prototype.writeInt16BE = makeWriteFunc('Int16', 2, false);
MemBuffer.prototype.writeUint16LE = makeWriteFunc('Uint16', 2, true);
MemBuffer.prototype.writeUint16BE = makeWriteFunc('Uint16', 2, false);
MemBuffer.prototype.writeInt32LE = makeWriteFunc('Int32', 4, true);
MemBuffer.prototype.writeInt32BE = makeWriteFunc('Int32', 4, false);
MemBuffer.prototype.writeUint32LE = makeWriteFunc('Uint32', 4, true);
MemBuffer.prototype.writeUint32BE = makeWriteFunc('Uint32', 4, false);
MemBuffer.prototype.writeFloat32LE = makeWriteFunc('Float32', 4, true);
MemBuffer.prototype.writeFloat32BE = makeWriteFunc('Float32', 4, false);
MemBuffer.prototype.writeFloat64LE = makeWriteFunc('Float64', 8, true);
MemBuffer.prototype.writeFloat64BE = makeWriteFunc('Float64', 8, false);
/**
* To consume data from an mbuf, the following pattern is recommended:
* var consumer = mbuf.makeConsumer();
* try {
* mbuf.readInt8(consumer);
* mbuf.readByteArray(consumer, len);
* ...
* } catch (e) {
* if (e.needMoreData) {
* ...
* }
* }
* mbuf.consume(consumer);
*/
MemBuffer.prototype.makeConsumer = function() {
return new Consumer(this);
};
/**
* After some data has been read via a consumer, mbuf.consume(consumer) will clear out the
* consumed data from the buffer.
*/
MemBuffer.prototype.consume = function(consumer) {
this.startPos = consumer.pos;
if (this.size() === 0) {
this.clear();
consumer.pos = this.startPos;
}
};
/**
* Helper class for reading data from the buffer. It keeps track of an offset into the buffer
* without changing anything in the MemBuffer itself. To affect the MemBuffer,
* mbuf.consume(consumer) should be called.
*/
function Consumer(mbuf) {
this.mbuf = mbuf;
this.pos = mbuf.startPos;
}
/**
* Helper for reading data, used by MemBuffer's read* methods.
*/
Consumer.prototype._consume = function(nbytes) {
var offset = this.pos;
if (this.pos + nbytes > this.mbuf.endPos) {
var err = new RangeError("MemBuffer: read past end");
err.needMoreData = true;
err.consumedData = this.pos - this.mbuf.startPos;
throw err;
}
this.pos += nbytes;
return offset;
};
/**
* Reads length bytes from the buffer using the passed-in consumer, as created by
* mbuf.makeConsumer(). Returns a view on the underlying data.
* @returns {Uint8Array} array of bytes viewing underlying MemBuffer data.
*/
MemBuffer.prototype.readByteArraySlice = function(cons, length) {
return new Uint8Array(this.buffer, cons._consume(length), length);
};
/**
* Reads length bytes from the buffer using the passed-in consumer.
* @returns {Uint8Array} array of bytes that's a copy of the underlying data.
*/
MemBuffer.prototype.readByteArray = function(cons, length) {
return new Uint8Array(this.readByteArraySlice(cons, length));
};
/**
* Reads length bytes from the buffer using the passed-in consumer.
* @returns {Buffer} copy of data as a Node Buffer.
*/
MemBuffer.prototype.readBuffer = function(cons, length) {
return Buffer.from(this.readByteArraySlice(cons, length));
};
/**
* Decodes byteLength bytes from the buffer using UTF8 and returns the resulting string. Uses the
* passed-in consumer, as created by mbuf.makeConsumer().
* @returns {string}
*/
if (typeof TextDecoder !== 'undefined') {
MemBuffer.prototype.readString = function(cons, byteLength) {
return arrayToString(this.readByteArraySlice(cons, byteLength));
};
} else {
var decodeBuffer = Buffer.alloc(1024);
MemBuffer.prototype.readString = function(cons, byteLength) {
var offset = cons._consume(byteLength);
if (byteLength <= decodeBuffer.length) {
gutil.arrayCopyForward(decodeBuffer, 0, this.asArray, offset, byteLength);
return decodeBuffer.toString('utf8', 0, byteLength);
} else {
return Buffer.from(new Uint8Array(this.buffer, offset, byteLength)).toString();
}
};
}
function makeReadFunc(typeName, bytes, optLittleEndian) {
var getter = DataView.prototype['get' + typeName];
return function(cons) {
return getter.call(this.asDataView, cons._consume(bytes), optLittleEndian);
};
}
/**
* The following methods read and return a value of the given type from the buffer using the
* passed-in consumer, as created by mbuf.makeConsumer(). E.g.
* var consumer = mbuf.makeConsumer();
* mbuf.readInt8(consumer);
* mbuf.consume(consumer);
* These are analogous to Node Buffer's read* family of methods.
*/
MemBuffer.prototype.readInt8 = makeReadFunc('Int8', 1);
MemBuffer.prototype.readUint8 = makeReadFunc('Uint8', 1);
MemBuffer.prototype.readInt16LE = makeReadFunc('Int16', 2, true);
MemBuffer.prototype.readUint16LE = makeReadFunc('Uint16', 2, true);
MemBuffer.prototype.readInt16BE = makeReadFunc('Int16', 2, false);
MemBuffer.prototype.readUint16BE = makeReadFunc('Uint16', 2, false);
MemBuffer.prototype.readInt32LE = makeReadFunc('Int32', 4, true);
MemBuffer.prototype.readUint32LE = makeReadFunc('Uint32', 4, true);
MemBuffer.prototype.readInt32BE = makeReadFunc('Int32', 4, false);
MemBuffer.prototype.readUint32BE = makeReadFunc('Uint32', 4, false);
MemBuffer.prototype.readFloat32LE = makeReadFunc('Float32', 4, true);
MemBuffer.prototype.readFloat32BE = makeReadFunc('Float32', 4, false);
MemBuffer.prototype.readFloat64LE = makeReadFunc('Float64', 8, true);
MemBuffer.prototype.readFloat64BE = makeReadFunc('Float64', 8, false);
module.exports = MemBuffer;

@ -0,0 +1,101 @@
const _ = require('underscore');
const metricConfig = require('./metricConfig');
const metricTools = require('./metricTools');
const gutil = require('app/common/gutil');
/**
* Base class for metrics collection used by both the server metrics collector, ServerMetrics.js,
* and the client metrics collector, ClientMetrics.js. Should not be instantiated.
* Establishes interval attempts to push metrics to the server on creation.
*/
function MetricsCollector() {
this.startTime = metricTools.getBucketStartTime(Date.now());
this.readyToExport = [];
// used (as a protected member) by the derived ServerMetrics class.
this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now()));
}
// Should return a map from metric names (as entered in metricConfig.js) to their metricTools.
MetricsCollector.prototype.getMetrics = function() {
throw new Error("Not implemented");
};
// Should return a promise that is resolved when the metrics have been pushed.
MetricsCollector.prototype.pushMetrics = function() {
throw new Error("Not implemented");
};
// Should return a bucket of metric data, formatted for either the client or server.
MetricsCollector.prototype.createBucket = function(bucketStart) {
throw new Error("Not implemented");
};
// Takes a list of metrics specifications and creates an object mapping metric names to
// a new instance of the metric gathering tool matching that metric's type.
MetricsCollector.prototype.initMetricTools = function(metricsList) {
var metrics = {};
metricsList.forEach(metricInfo => {
metrics[metricInfo.name] = new metricTools[metricInfo.type](metricInfo.name);
});
return metrics;
};
// Called each push interval.
MetricsCollector.prototype.attemptPush = function() {
this.pushMetrics(this.readyToExport);
this.readyToExport = [];
};
// Pushes bucket to the end of the readyToExport queue. Should be called sequentially, since it
// handles deletion of buckets older than the export memory limit.
MetricsCollector.prototype.queueBucket = function(bucket) {
// If readyToExport is at maximum length, delete the oldest element
this.readyToExport.push(bucket);
var length = this.readyToExport.length;
if (length > metricConfig.MAX_PENDING_BUCKETS) {
this.readyToExport.splice(0, length - metricConfig.MAX_PENDING_BUCKETS);
}
};
MetricsCollector.prototype.scheduleBucketPreparation = function() {
this.prepareCompletedBuckets(Date.now());
this._collect = setTimeout(() => this.scheduleBucketPreparation(), metricTools.getDeltaMs(Date.now()));
};
/**
* Checks if each bucket since the last update is completed and for each one adds all data and
* pushes it to the export ready array.
*/
MetricsCollector.prototype.prepareCompletedBuckets = function(now) {
var bucketStart = metricTools.getBucketStartTime(now);
while (bucketStart > this.startTime) {
this.queueBucket(this.createBucket(this.startTime));
this.startTime += metricConfig.BUCKET_SIZE;
}
};
/**
* Collects primitive metrics tools into a list.
*/
MetricsCollector.prototype.collectPrimitiveMetrics = function() {
var metricTools = [];
_.forEach(this.getMetrics(), metricTool => {
gutil.arrayExtend(metricTools, metricTool.getPrimitiveMetrics());
});
return metricTools;
};
/**
* Loops through metric tools for a chosen bucket and performs the provided callback on each.
* Resets each tool after the callback is performed.
* @param {Number} bucketStart - The desired bucket's start time in milliseconds
* @param {Function} callback - The callback to perform on each metric tool.
*/
MetricsCollector.prototype.forEachBucketMetric = function(bucketEnd, callback) {
this.collectPrimitiveMetrics().forEach(tool => {
callback(tool);
tool.reset(bucketEnd);
});
};
module.exports = MetricsCollector;

@ -0,0 +1,76 @@
/**
* Here are the most relevant formats we want to support.
* -1234.56 Plain
* -1,234.56 Number (with separators)
* 12.34% Percent
* 1.23E3 Scientific
* $(1,234.56) Accounting
* (1,234.56) Financial
* -$1,234.56 Currency
*
* We implement a button-based UI, using one selector button to choose mode:
* none = NumMode undefined (plain number, no thousand separators)
* `$` = NumMode 'currency'
* `,` = NumMode 'decimal' (plain number, with thousand separators)
* `%` = NumMode 'percent'
* `Exp` = NumMode 'scientific'
* A second toggle button is `(-)` for Sign, to use parentheses rather than "-" for negative
* numbers. It is Ignored and disabled when mode is 'scientific'.
*/
import {clamp} from 'app/common/gutil';
// Options for number formatting.
export type NumMode = 'currency' | 'decimal' | 'percent' | 'scientific';
export type NumSign = 'parens';
// TODO: In the future, locale should be a value associated with the document or the user.
const defaultLocale = 'en-US';
// TODO: The currency to use for currency formatting could be made configurable.
const defaultCurrency = 'USD';
export interface NumberFormatOptions {
numMode?: NumMode;
numSign?: NumSign;
decimals?: number; // aka minimum fraction digits
maxDecimals?: number;
}
export function buildNumberFormat(options: NumberFormatOptions): Intl.NumberFormat {
const nfOptions: Intl.NumberFormatOptions = parseNumMode(options.numMode);
// numSign is implemented outside of Intl.NumberFormat since the latter's similar 'currencySign'
// option is not well-supported, and doesn't apply to non-currency formats.
if (options.decimals !== undefined) {
// Should be at least 0
nfOptions.minimumFractionDigits = clamp(Number(options.decimals), 0, 20);
}
// maximumFractionDigits must not be less than the minimum, so we need to know the minimum
// implied by numMode.
const tmp = new Intl.NumberFormat(defaultLocale, nfOptions).resolvedOptions();
if (options.maxDecimals !== undefined) {
// Should be at least 0 and at least minimumFractionDigits.
nfOptions.maximumFractionDigits = clamp(Number(options.maxDecimals), tmp.minimumFractionDigits || 0, 20);
} else if (!options.numMode) {
// For the default format, keep max digits at 10 as we had before.
nfOptions.maximumFractionDigits = clamp(10, tmp.minimumFractionDigits || 0, 20);
}
return new Intl.NumberFormat(defaultLocale, nfOptions);
}
function parseNumMode(numMode?: NumMode): Intl.NumberFormatOptions {
switch (numMode) {
case 'currency': return {style: 'currency', currency: defaultCurrency};
case 'decimal': return {useGrouping: true};
case 'percent': return {style: 'percent'};
// TODO 'notation' option (and therefore numMode 'scientific') works on recent Firefox and
// Chrome, not on Safari or Node 10.
case 'scientific': return {notation: 'scientific'} as Intl.NumberFormatOptions;
default: return {useGrouping: false};
}
}

@ -0,0 +1,213 @@
import {IForwarderDest, IMessage, IMsgCustom, IMsgRpcCall, IRpcLogger, MsgType, Rpc} from 'grain-rpc';
import {Checker} from "ts-interface-checker";
import {InactivityTimer} from 'app/common/InactivityTimer';
import {LocalPlugin} from 'app/common/plugin';
import {BarePlugin} from 'app/plugin/PluginManifest';
import {Implementation} from 'app/plugin/PluginManifest';
import {RenderOptions, RenderTarget} from 'app/plugin/RenderOptions';
export type ComponentKind = "safeBrowser" | "safePython" | "unsafeNode";
// Describes a function that appends some html content to `containerElement` given some
// options. Usefull for provided by a plugin.
export type TargetRenderFunc = (containerElement: HTMLElement, options?: RenderOptions) => void;
/**
* The `BaseComponent` is the base implementation for a plugins' component. It exposes methods
* related to its activation. It provides basic features including the inactivity timer, activated
* state for the component. A custom component must override the `deactivateImplementation`,
* `activeImplementation` and `useRemoteAPI` methods.
*/
export abstract class BaseComponent implements IForwarderDest {
public inactivityTimer: InactivityTimer;
private _activated: boolean = false;
constructor(plugin: BarePlugin, private _logger: IRpcLogger) {
const deactivate = plugin.components.deactivate;
const delay = (deactivate && deactivate.inactivitySec) ? deactivate.inactivitySec : 300;
this.inactivityTimer = new InactivityTimer(() => this.deactivate(), delay * 1000);
}
/**
* Wether the Component component have been activated.
*/
public activated(): boolean {
return this._activated;
}
/**
* Activates the component.
*/
public async activate(): Promise<void> {
if (this._logger.info) { this._logger.info("Activating plugin component"); }
await this.activateImplementation();
this._activated = true;
this.inactivityTimer.enable();
}
/**
* Force deactivate the component.
*/
public async deactivate(): Promise<void> {
if (this._activated) {
if (this._logger.info) { this._logger.info("Deactivating plugin component"); }
this._activated = false;
// Cancel the timer to ensure we don't have an unnecessary hanging timeout (in tests it will
// prevent node from exiting, but also it's just wasteful).
this.inactivityTimer.disable();
try {
await this.deactivateImplementation();
} catch (e) {
// If it fails, we warn and swallow the exception (or it would be an unhandled rejection).
if (this._logger.warn) { this._logger.warn(`Deactivate failed: ${e.message}`); }
}
}
}
public async forwardCall(c: IMsgRpcCall): Promise<any> {
if (!this._activated) { await this.activate(); }
return await this.inactivityTimer.disableUntilFinish(this.doForwardCall(c));
}
public async forwardMessage(msg: IMsgCustom): Promise<any> {
if (!this._activated) { await this.activate(); }
this.inactivityTimer.ping();
this.doForwardMessage(msg); // tslint:disable-line:no-floating-promises TODO
}
protected abstract doForwardCall(c: IMsgRpcCall): Promise<any>;
protected abstract doForwardMessage(msg: IMsgCustom): Promise<any>;
protected abstract deactivateImplementation(): Promise<void>;
protected abstract activateImplementation(): Promise<void>;
}
/**
* Node Implementation for the PluginElement interface. A PluginInstance take care of activation of
* the the plugins's components (activating, timing and deactivating), and create the api's for each contributions.
*
* Do not try to instanciate yourself, PluginManager does it for you. Instead use the
* PluginManager.getPlugin(id) method that get instances for you.
*
*/
export class PluginInstance {
public rpc: Rpc;
public safeBrowser?: BaseComponent;
public unsafeNode?: BaseComponent;
public safePython?: BaseComponent;
private _renderTargets: Map<RenderTarget, TargetRenderFunc> = new Map();
private _nextRenderTargetId = 0;
constructor(public definition: LocalPlugin, rpcLogger: IRpcLogger) {
const rpc = this.rpc = new Rpc({logger: rpcLogger});
rpc.setSendMessage((mssg: any) => rpc.receiveMessage(mssg));
this._renderTargets.set("fullscreen", renderFullScreen);
}
/**
* Create an instance for the implementation, this implementation is specific to node environment.
*/
public getStub<Iface>(implementation: Implementation, checker: Checker): Iface {
const components: any = this.definition.manifest.components;
// the component forwarder was registered under the same relative path that was used to declare
// it in the manifest
const forwardName = components[implementation.component];
return this.rpc.getStubForward<Iface>(forwardName, implementation.name, checker);
}
/**
* Stop and clean up all components of this plugin.
*/
public async shutdown(): Promise<void> {
await Promise.all([
this.safeBrowser && this.safeBrowser.deactivate(),
this.safePython && this.safePython.deactivate(),
this.unsafeNode && this.unsafeNode.deactivate(),
]);
}
/**
* Create a render target and return its identifier. When a plugin calls `render` with `inline`
* mode and this identifier, it will append the safe browser process to `element`.
*/
public addRenderTarget(renderPluginContent: TargetRenderFunc): number {
const id = this._nextRenderTargetId++;
this._renderTargets.set(id, renderPluginContent);
return id;
}
/**
* Get the function that render an HTML element based on RenderTarget and RenderOptions.
*/
public getRenderTarget(target: RenderTarget, options?: RenderOptions): TargetRenderFunc {
const targetRenderPluginContent = this._renderTargets.get(target);
if (!targetRenderPluginContent) {
throw new Error(`Unknown render target ${target}`);
}
return (containerElement, opts) => targetRenderPluginContent(containerElement, opts || options);
}
/**
* Removes the render target.
*/
public removeRenderTarget(target: RenderTarget): boolean {
return this._renderTargets.delete(target);
}
}
/**
* Renders safe browser plugin in fullscreen.
*/
function renderFullScreen(element: Element) {
element.classList.add("plugin_instance_fullscreen");
document.body.appendChild(element);
}
// Basically the union of relevant interfaces of console and server log.
export interface BaseLogger {
log?(message: string, ...args: any[]): void;
debug?(message: string, ...args: any[]): void;
warn?(message: string, ...args: any[]): void;
}
/**
* Create IRpcLogger which logs to console or server log with the given prefix. Specifically will
* warn using baseLog.warn, and log info using baseLog.debug or baseLog.log, as available.
*/
export function createRpcLogger(baseLog: BaseLogger, prefix: string): IRpcLogger {
const info = baseLog.debug || baseLog.log;
const warn = baseLog.warn;
return {
warn: warn && ((msg: string) => warn("%s %s", prefix, msg)),
info: info && ((msg: string) => info("%s %s", prefix, msg)),
};
}
/**
* If msec milliseconds pass without receiving a Ready message, print the given message as a
* warning.
* TODO: I propose making it a method of rpc itself, as rpc.warnIfNotReady(msec, message). Until
* we have that, this implements it via an ugly hack.
*/
export function warnIfNotReady(rpc: Rpc, msec: number, message: string): void {
if (!(rpc as any)._logger.warn) { return; }
const timer = setTimeout(() => (rpc as any)._logger.warn(message), msec);
const origDispatch = (rpc as any)._dispatch;
(rpc as any)._dispatch = (msg: IMessage) => {
if (msg.mtype === MsgType.Ready) { clearTimeout(timer); }
origDispatch.call(rpc, msg);
};
}

@ -0,0 +1,123 @@
/**
* RefCountMap maintains a reference-counted key-value map. Its sole method is use(key) which
* increments the counter for the key, and returns a disposable object which exposes the value via
* the get() method, and decrements the counter back on disposal.
*
* The value is constructed on first reference using options.create(key) callback. After the last
* reference is gone, and an optional gracePeriodMs elapsed, the value is cleaned up using
* options.dispose(key, value) callback.
*/
import {IDisposable} from 'grainjs';
export interface IRefCountSub<Value> extends IDisposable {
get(): Value;
dispose(): void;
}
export class RefCountMap<Key, Value> implements IDisposable {
private _map: Map<Key, RefCountValue<Value>> = new Map();
private _createKey: (key: Key) => Value;
private _disposeKey: (key: Key, value: Value) => void;
private _gracePeriodMs: number;
/**
* Values are created using options.create(key) on first use. They are disposed after last use,
* using options.dispose(key, value). If options.gracePeriodMs is greater than zero, values
* stick around for this long after last use.
*/
constructor(options: {
create: (key: Key) => Value,
dispose: (key: Key, value: Value) => void,
gracePeriodMs: number,
}) {
this._createKey = options.create;
this._disposeKey = options.dispose;
this._gracePeriodMs = options.gracePeriodMs;
}
/**
* Use a value, constructing it if needed, or only incrementing the reference count if this key
* is already in the map. The returned subscription object has a get() method which returns the
* actual value, and a dispose() method, which must be called to release this subscription (i.e.
* decrement back the reference count).
*/
public use(key: Key): IRefCountSub<Value> {
const rcValue = this._useKey(key);
return {
get: () => rcValue.value,
dispose: () => this._releaseKey(rcValue, key),
};
}
/**
* Purge a key by immediately removing it from the map. Disposing the remaining IRefCountSub
* values will be no-ops.
*/
public purgeKey(key: Key): void {
// Note that we must be careful that disposing stale IRefCountSub values is a no-op even when
// the same key gets re-added to the map after purgeKey.
this._doDisposeKey(key);
}
/**
* Disposing clears the map immediately, and calls options.dispose on all values.
*/
public dispose(): void {
// Note that a clear() method like this one would not be OK. If the map were to continue being
// used after clear(), subscriptions created before clear() would wreak havoc when disposed.
for (const [key, r] of this._map) {
r.count = 0;
this._disposeKey.call(null, key, r.value);
}
this._map.clear();
}
private _useKey(key: Key): RefCountValue<Value> {
const r = this._map.get(key);
if (r) {
r.count += 1;
if (r.disposeTimeout) {
clearTimeout(r.disposeTimeout);
r.disposeTimeout = undefined;
}
return r;
}
const value = this._createKey.call(null, key);
const rcValue = new RefCountValue(value);
this._map.set(key, rcValue);
return rcValue;
}
private _releaseKey(r: RefCountValue<Value>, key: Key): void {
if (r.count > 0) {
r.count -= 1;
if (r.count === 0) {
if (this._gracePeriodMs > 0) {
if (!r.disposeTimeout) {
r.disposeTimeout = setTimeout(() => this._doDisposeKey(key), this._gracePeriodMs);
}
} else {
this._doDisposeKey(key);
}
}
}
}
private _doDisposeKey(key: Key): void {
const r = this._map.get(key);
if (r) {
this._map.delete(key);
r.count = 0;
this._disposeKey.call(null, key, r.value);
}
}
}
/**
* This is an implementation detail of the RefCountMap, which represents a single item.
*/
class RefCountValue<Value> {
public count: number = 1;
public disposeTimeout?: ReturnType<typeof setTimeout> = undefined;
constructor(public value: Value) {}
}

@ -0,0 +1,88 @@
/**
* SortFunc class interprets the sortSpec (as saved in viewSection.sortColRefs), exposing a
* compare(rowId1, rowId2) function that can be used to actually sort rows in a view.
*
* TODO: When an operation (such as a paste) would cause rows to jump in the sort order, this
* class should support freezing of row positions until the user chooses to re-sort. This is not
* currently implemented.
*/
import {ColumnGetters} from 'app/common/ColumnGetters';
import {nativeCompare} from 'app/common/gutil';
/**
* Compare two cell values, paying attention to types and values. Note that native JS comparison
* can't be used for sorting because it isn't transitive across types (e.g. both 1 < "2" and "2" <
* "a" are true, but 1 < "a" is false.). In addition, we handle complex values represented in
* Grist as arrays.
*
* Note that we need to handle different types of values regardless of the column type,
* because e.g. a numerical column may contain text (alttext) or null values.
*/
export function typedCompare(val1: any, val2: any): number {
// TODO: We should use Intl.Collator for string comparisons to handle accented strings.
let type1, array1;
return nativeCompare(type1 = typeof val1, typeof val2) ||
// We need to worry about Array comparisons because formulas returing Any may return null or
// object values represented as arrays (e.g. ['D', ...] for dates). Comparing those without
// distinguishing types would break the sort. Also, arrays need a special comparator.
(type1 === 'object' &&
(nativeCompare(array1 = val1 instanceof Array, val2 instanceof Array) ||
(array1 && _arrayCompare(val1, val2)))) ||
nativeCompare(val1, val2);
}
function _arrayCompare(val1: any[], val2: any[]): number {
for (let i = 0; i < val1.length; i++) {
if (i >= val2.length) {
return 1;
}
const value = nativeCompare(val1[i], val2[i]);
if (value) {
return value;
}
}
return val1.length === val2.length ? 0 : -1;
}
type ColumnGetter = (rowId: number) => any;
/**
* getters is an implementation of app.common.ColumnGetters
*/
export class SortFunc {
// updateSpec() or updateGetters() can populate these fields, used by the compare() method.
private _colGetters: ColumnGetter[] = []; // Array of column getters (mapping rowId to column value)
private _ascFlags: number[] = []; // Array of 1 (ascending) or -1 (descending) flags.
constructor(private _getters: ColumnGetters) {}
public updateSpec(sortSpec: number[]): void {
// Prepare an array of column getters for each column in sortSpec.
this._colGetters = sortSpec.map(colRef => {
return this._getters.getColGetter(Math.abs(colRef));
}).filter(getter => getter) as ColumnGetter[];
// Collect "ascending" flags as an array of 1 or -1, one for each column.
this._ascFlags = sortSpec.map(colRef => (colRef >= 0 ? 1 : -1));
const manualSortGetter = this._getters.getManualSortGetter();
if (manualSortGetter) {
this._colGetters.push(manualSortGetter);
this._ascFlags.push(1);
}
}
/**
* Returns 1 or -1 depending on whether rowId1 should be shown before rowId2.
*/
public compare(rowId1: number, rowId2: number): number {
for (let i = 0, len = this._colGetters.length; i < len; i++) {
const getter = this._colGetters[i];
const value = typedCompare(getter(rowId1), getter(rowId2));
if (value) {
return value * this._ascFlags[i];
}
}
return nativeCompare(rowId1, rowId2);
}
}

@ -0,0 +1,38 @@
/**
* TypeScript will infer a string union type from the literal values passed to
* this function. Without `extends string`, it would instead generalize them
* to the common string type.
*
* Example definition:
* const Race = StringUnion(
* "orc",
* "human",
* "night elf",
* "undead",
* );
* type Race = typeof Race.type;
*
* For more details, see:
* https://stackoverflow.com/questions/36836011/checking-validity-of-string
* -literal-union-type-at-runtime?answertab=active#tab-top
*/
export const StringUnion = <UnionType extends string>(...values: UnionType[]) => {
Object.freeze(values);
const valueSet: Set<string> = new Set(values);
const guard = (value: string): value is UnionType => {
return valueSet.has(value);
};
const check = (value: string): UnionType => {
if (!guard(value)) {
const actual = JSON.stringify(value);
const expected = values.map(s => JSON.stringify(s)).join(' | ');
throw new TypeError(`Value '${actual}' is not assignable to type '${expected}'.`);
}
return value;
};
const unionNamespace = {guard, check, values};
return Object.freeze(unionNamespace as typeof unionNamespace & {type: UnionType});
};

@ -0,0 +1,427 @@
/**
* TableData maintains a single table's data.
*/
import {getDefaultForType} from 'app/common/gristTypes';
import fromPairs = require('lodash/fromPairs');
import {ActionDispatcher} from './ActionDispatcher';
import {BulkColValues, CellValue, ColInfo, ColInfoWithId, ColValues, DocAction,
isSchemaAction, ReplaceTableData, RowRecord, TableDataAction} from './DocActions';
import {arrayRemove, arraySplice} from './gutil';
export interface ColTypeMap { [colId: string]: string; }
interface ColData {
colId: string;
type: string;
defl: any;
values: CellValue[];
}
/**
* TableData class to maintain a single table's data.
*
* In the browser's memory, table data needs a representation that's reasonably compact. We
* represent it as column-wise arrays. (An early hope was to allow use of TypedArrays, but since
* types can be mixed, those are not used.)
*/
export class TableData extends ActionDispatcher {
private _tableId: string;
private _isLoaded: boolean = false;
private _fetchPromise?: Promise<void>;
// Storage of the underlying data. Each column is an array, all of the same length. Includes
// 'id' column, containing a reference to _rowIdCol.
private _columns: Map<string, ColData> = new Map();
// Array of all ColData objects, omitting 'id'.
private _colArray: ColData[] = [];
// The `id` column is direct reference to the 'id' column, and contains row ids.
private _rowIdCol: number[] = [];
// Maps row id to index in the arrays in _columns. I.e. it's the inverse of _rowIdCol.
private _rowMap: Map<number, number> = new Map();
constructor(tableId: string, tableData: TableDataAction|null, colTypes: ColTypeMap) {
super();
this._tableId = tableId;
// Initialize all columns to empty arrays, while nothing is yet loaded.
for (const colId in colTypes) {
if (colTypes.hasOwnProperty(colId)) {
const type = colTypes[colId];
const defl = getDefaultForType(type);
const colData: ColData = { colId, type, defl, values: [] };
this._columns.set(colId, colData);
this._colArray.push(colData);
}
}
this._columns.set('id', {colId: 'id', type: 'Id', defl: 0, values: this._rowIdCol});
if (tableData) {
this.loadData(tableData);
}
// TODO: We should probably unload big sets of data when no longer needed. This can be left for
// when we support loading only parts of a table.
}
/**
* Fetch data (as long as a fetch is not in progress), and load it in memory when done.
* Returns a promise that's resolved when data finishes loading, and isLoaded becomes true.
*/
public fetchData(fetchFunc: (tableId: string) => Promise<TableDataAction>): Promise<void> {
if (!this._fetchPromise) {
this._fetchPromise = fetchFunc(this._tableId).then(data => {
this._fetchPromise = undefined;
this.loadData(data);
});
}
return this._fetchPromise;
}
/**
* Populates the data for this table. Returns the array of old rowIds that were loaded before.
*/
public loadData(tableData: TableDataAction|ReplaceTableData): number[] {
const rowIds: number[] = tableData[2];
const colValues: BulkColValues = tableData[3];
const oldRowIds: number[] = this._rowIdCol.slice(0);
reassignArray(this._rowIdCol, rowIds);
for (const colData of this._colArray) {
const values = colValues[colData.colId];
// If colId is missing from tableData, use an array of default values. Note that reusing
// default value like this is only OK because all default values we use are primitive.
reassignArray(colData.values, values || this._rowIdCol.map(() => colData.defl));
}
this._rowMap.clear();
for (let i = 0; i < rowIds.length; i++) {
this._rowMap.set(rowIds[i], i);
}
this._isLoaded = true;
return oldRowIds;
}
// Used by QuerySet to load new rows for onDemand tables.
public loadPartial(data: TableDataAction): void {
// Add the new rows, reusing BulkAddData code.
const rowIds: number[] = data[2];
this.onBulkAddRecord(data, data[1], rowIds, data[3]);
// Mark the table as loaded.
this._isLoaded = true;
}
// Used by QuerySet to remove unused rows for onDemand tables when a QuerySet is disposed.
public unloadPartial(rowIds: number[]): void {
// Remove the unneeded rows, reusing BulkRemoveRecord code.
this.onBulkRemoveRecord(['BulkRemoveRecord', this.tableId, rowIds], this.tableId, rowIds);
}
/**
* Read-only tableId.
*/
public get tableId(): string { return this._tableId; }
/**
* Boolean flag for whether the data for this table is already loaded.
*/
public get isLoaded(): boolean { return this._isLoaded; }
/**
* The number of records loaded in this table.
*/
public numRecords(): number { return this._rowIdCol.length; }
/**
* Returns the specified value from this table.
*/
public getValue(rowId: number, colId: string): CellValue|undefined {
const colData = this._columns.get(colId);
const index = this._rowMap.get(rowId);
return colData && index !== undefined ? colData.values[index] : undefined;
}
/**
* Given a column name, returns a function that takes a rowId and returns the value for that
* column of that row. The returned function is faster than getValue() calls.
*/
public getRowPropFunc(colId: string): undefined | ((rowId: number|"new") => CellValue|undefined) {
const colData = this._columns.get(colId);
if (!colData) { return undefined; }
const values = colData.values;
const rowMap = this._rowMap;
return function(rowId: number|"new") { return rowId === "new" ? "new" : values[rowMap.get(rowId)!]; };
}
/**
* Returns the list of all rowIds in this table, in unspecified and unstable order. Equivalent
* to getColValues('id').
*/
public getRowIds(): ReadonlyArray<number> {
return this._rowIdCol;
}
/**
* Sort and returns the list of all rowIds in this table.
*/
public getSortedRowIds(): number[] {
return this._rowIdCol.slice(0).sort((a, b) => a - b);
}
/**
* Returns the list of colIds in this table, including 'id'.
*/
public getColIds(): string[] {
return Array.from(this._columns.keys());
}
/**
* Returns an unsorted list of all values in the given column. With no intervening actions,
* all arrays returned by getColValues() and getRowIds() are parallel to each other, i.e. the
* values at the same index correspond to the same record.
*/
public getColValues(colId: string): ReadonlyArray<CellValue>|undefined {
const colData = this._columns.get(colId);
return colData ? colData.values : undefined;
}
/**
* Returns a limited-sized set of distinct values from a column. If count is given, limits how many
* distinct values are returned.
*/
public getDistinctValues(colId: string, count: number = Infinity): Set<CellValue>|undefined {
const valColumn = this.getColValues(colId);
if (!valColumn) { return undefined; }
const distinct: Set<CellValue> = new Set();
// Add values to the set until it reaches the desired size, or until there are no more values.
for (let i = 0; i < valColumn.length && distinct.size < count; i++) {
distinct.add(valColumn[i]);
}
return distinct;
}
/**
* Return data in TableDataAction form ['TableData', tableId, [...rowIds], {...}]
*/
public getTableDataAction(): TableDataAction {
const rowIds = this.getRowIds();
return ['TableData',
this.tableId,
rowIds as number[],
fromPairs(
this.getColIds()
.filter(colId => colId !== 'id')
.map(colId => [colId, this.getColValues(colId)! as CellValue[]]))];
}
/**
* Returns the given columns type, if the column exists, or undefined otherwise.
*/
public getColType(colId: string): string|undefined {
const colData = this._columns.get(colId);
return colData ? colData.type : undefined;
}
/**
* Builds and returns a record object for the given rowId.
*/
public getRecord(rowId: number): undefined | RowRecord {
const index = this._rowMap.get(rowId);
if (index === undefined) { return undefined; }
const ret: RowRecord = { id: this._rowIdCol[index] };
for (const colData of this._colArray) {
ret[colData.colId] = colData.values[index];
}
return ret;
}
/**
* Builds and returns the list of all records on this table, in unspecified and unstable order.
*/
public getRecords(): RowRecord[] {
const records: RowRecord[] = this._rowIdCol.map((id) => ({ id }));
for (const {colId, values} of this._colArray) {
for (let i = 0; i < records.length; i++) {
records[i][colId] = values[i];
}
}
return records;
}
/**
* Builds and returns the list of records in this table that match the given properties object.
* Properties may include 'id' and any table columns. Returned records are not sorted.
*/
public filterRecords(properties: {[key: string]: any}): RowRecord[] {
const rowIndices: number[] = [];
// Pairs of [valueToMatch, arrayOfColValues]
const props = Object.keys(properties).map(p => [properties[p], this._columns.get(p)]);
this._rowIdCol.forEach((id, i) => {
for (const p of props) {
if (p[1].values[i] !== p[0]) { return; }
}
// Collect the indices of the matching rows.
rowIndices.push(i);
});
// Convert the array of indices to an array of RowRecords.
const records: RowRecord[] = rowIndices.map(i => ({id: this._rowIdCol[i]}));
for (const {colId, values} of this._colArray) {
for (let i = 0; i < records.length; i++) {
records[i][colId] = values[rowIndices[i]];
}
}
return records;
}
/**
* Returns the rowId in the table where colValue is found in the column with the given colId.
*/
public findRow(colId: string, colValue: any): number {
const colData = this._columns.get(colId);
if (!colData) {
return 0;
}
const index = colData.values.indexOf(colValue);
return index < 0 ? 0 : this._rowIdCol[index];
}
/**
* Applies a DocAction received from the server; returns true, or false if it was skipped.
*/
public receiveAction(action: DocAction): boolean {
if (this._isLoaded || isSchemaAction(action)) {
this.dispatchAction(action);
return true;
}
return false;
}
// ---- The following methods implement ActionDispatcher interface ----
protected onAddRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
const index: number = this._rowIdCol.length;
this._rowMap.set(rowId, index);
this._rowIdCol[index] = rowId;
for (const {colId, defl, values} of this._colArray) {
values[index] = colValues.hasOwnProperty(colId) ? colValues[colId] : defl;
}
}
protected onBulkAddRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
const index: number = this._rowIdCol.length;
for (let i = 0; i < rowIds.length; i++) {
this._rowMap.set(rowIds[i], index + i);
this._rowIdCol[index + i] = rowIds[i];
}
for (const {colId, defl, values} of this._colArray) {
for (let i = 0; i < rowIds.length; i++) {
values[index + i] = colValues.hasOwnProperty(colId) ? colValues[colId][i] : defl;
}
}
}
protected onRemoveRecord(action: DocAction, tableId: string, rowId: number): void {
// Note that in this implementation, delete + undo will reorder the storage and the ordering
// of rows returned getRowIds() and similar methods.
const index = this._rowMap.get(rowId);
if (index !== undefined) {
const last: number = this._rowIdCol.length - 1;
// We keep the column-wise arrays dense by moving the last element into the freed-up spot.
for (const {values} of this._columns.values()) { // This adjusts _rowIdCol too.
values[index] = values[last];
values.pop();
}
this._rowMap.set(this._rowIdCol[index], index);
this._rowMap.delete(rowId);
}
}
protected onUpdateRecord(action: DocAction, tableId: string, rowId: number, colValues: ColValues): void {
const index = this._rowMap.get(rowId);
if (index !== undefined) {
for (const colId in colValues) {
if (colValues.hasOwnProperty(colId)) {
const colData = this._columns.get(colId);
if (colData) {
colData.values[index] = colValues[colId];
}
}
}
}
}
protected onBulkUpdateRecord(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
for (let i = 0; i < rowIds.length; i++) {
const index = this._rowMap.get(rowIds[i]);
if (index !== undefined) {
for (const colId in colValues) {
if (colValues.hasOwnProperty(colId)) {
const colData = this._columns.get(colId);
if (colData) {
colData.values[index] = colValues[colId][i];
}
}
}
}
}
}
protected onReplaceTableData(action: DocAction, tableId: string, rowIds: number[], colValues: BulkColValues): void {
this.loadData(action as ReplaceTableData);
}
protected onAddColumn(action: DocAction, tableId: string, colId: string, colInfo: ColInfo): void {
if (this._columns.has(colId)) { return; }
const type = colInfo.type;
const defl = getDefaultForType(type);
const colData: ColData = { colId, type, defl, values: this._rowIdCol.map(() => defl) };
this._columns.set(colId, colData);
this._colArray.push(colData);
}
protected onRemoveColumn(action: DocAction, tableId: string, colId: string): void {
const colData = this._columns.get(colId);
if (!colData) { return; }
this._columns.delete(colId);
arrayRemove(this._colArray, colData);
}
protected onRenameColumn(action: DocAction, tableId: string, oldColId: string, newColId: string): void {
const colData = this._columns.get(oldColId);
if (colData) {
colData.colId = newColId;
this._columns.set(newColId, colData);
this._columns.delete(oldColId);
}
}
protected onModifyColumn(action: DocAction, tableId: string, oldColId: string, colInfo: ColInfo): void {
const colData = this._columns.get(oldColId);
if (colData && colInfo.hasOwnProperty('type')) {
colData.type = colInfo.type;
colData.defl = getDefaultForType(colInfo.type);
}
}
protected onRenameTable(action: DocAction, oldTableId: string, newTableId: string): void {
this._tableId = newTableId;
}
protected onAddTable(action: DocAction, tableId: string, columns: ColInfoWithId[]): void {
// A table processing its own addition is a noop
}
protected onRemoveTable(action: DocAction, tableId: string): void {
// Stop dispatching actions if we've been deleted. We might also want to clean up in the future.
this._isLoaded = false;
}
}
function reassignArray<T>(targetArray: T[], sourceArray: T[]): void {
targetArray.length = 0;
arraySplice(targetArray, 0, sourceArray);
}

@ -0,0 +1,31 @@
/**
*
* Types for use when summarizing differences between versions of a table, with the
* diff itself presented in tabular form.
*
*/
/**
* Pairs of before/after values of cells. Values, when present, are nested in a trivial
* list since they can be literally anything - null, undefined, etc. Otherwise they
* are either null, meaning non-existent, or "?", meaning unknown. Non-existent values
* appear prior to a table/column being created, or after it has been destroyed.
* Unknown values appear when they are omitted from summaries of bulk actions, and those
* summaries are then merged with others.
*/
export type CellDelta = [[any]|"?"|null, [any]|"?"|null];
/** a special column indicating what changes happened on row (addition, update, removal) */
export type RowChangeType = string;
/** differences for an individual table */
export interface TabularDiff {
header: string[]; /** labels for columns */
cells: Array<[RowChangeType, number, CellDelta[]]>; // "number" is rowId
}
/** differences for a collection of tables */
export interface TabularDiffs {
[tableId: string]: TabularDiff;
}

@ -0,0 +1,4 @@
export interface TestState {
clipboard?: string;
anchorApplied?: boolean;
}

@ -0,0 +1,667 @@
import {ApplyUAResult} from 'app/common/ActiveDocAPI';
import {BaseAPI, IOptions} from 'app/common/BaseAPI';
import {BillingAPI, BillingAPIImpl} from 'app/common/BillingAPI';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {BulkColValues, TableColValues, UserAction} from 'app/common/DocActions';
import {DocCreationInfo} from 'app/common/DocListAPI';
import {Features} from 'app/common/Features';
import {isClient} from 'app/common/gristUrls';
import {FullUser} from 'app/common/LoginSessionAPI';
import * as roles from 'app/common/roles';
import {addCurrentOrgToPath} from 'app/common/urlUtils';
// Nominal email address of the anonymous user.
export const ANONYMOUS_USER_EMAIL = 'anon@getgrist.com';
// A special user allowed to add/remove the EVERYONE_EMAIL to/from a resource.
export const SUPPORT_EMAIL = 'support@getgrist.com';
// A special 'docId' that means to create a new document.
export const NEW_DOCUMENT_CODE = 'new';
// Properties shared by org, workspace, and doc resources.
export interface CommonProperties {
name: string;
createdAt: string; // ISO date string
updatedAt: string; // ISO date string
removedAt?: string; // ISO date string - only can appear on docs and workspaces currently
public?: boolean; // If set, resource is available to the public
}
export const commonPropertyKeys = ['createdAt', 'name', 'updatedAt'];
export interface OrganizationProperties extends CommonProperties {
domain: string|null;
}
export const organizationPropertyKeys = [...commonPropertyKeys, 'domain'];
// Basic information about an organization, excluding the user's access level
export interface OrganizationWithoutAccessInfo extends OrganizationProperties {
id: number;
owner: FullUser|null;
billingAccount?: BillingAccount;
host: string|null; // if set, org's preferred domain (e.g. www.thing.com)
}
// Organization information plus the user's access level
export interface Organization extends OrganizationWithoutAccessInfo {
access: roles.Role;
}
// Basic information about a billing account associated with an org or orgs.
export interface BillingAccount {
id: number;
individual: boolean;
product: Product;
isManager: boolean;
}
// Information about the product associated with an org or orgs.
export interface Product {
name: string;
features: Features;
}
// The upload types vary based on which fetch implementation is in use. This is
// an incomplete list. For example, node streaming types are supported by node-fetch.
export type UploadType = string | Blob | Buffer;
/**
* Returns a user-friendly org name, which is either org.name, or "@User Name" for personal orgs.
*/
export function getOrgName(org: Organization): string {
return org.owner ? `@` + org.owner.name : org.name;
}
export type WorkspaceProperties = CommonProperties;
export const workspacePropertyKeys = ['createdAt', 'name', 'updatedAt'];
export interface Workspace extends WorkspaceProperties {
id: number;
docs: Document[];
org: Organization;
access: roles.Role;
owner?: FullUser; // Set when workspaces are in the "docs" pseudo-organization,
// assembled from multiple personal organizations.
// Not set when workspaces are all from the same organization.
// Set when the workspace belongs to support@getgrist.com. We expect only one such workspace
// ("Examples & Templates"), containing sample documents.
isSupportWorkspace?: boolean;
}
export interface DocumentProperties extends CommonProperties {
isPinned: boolean;
urlId: string|null;
}
export const documentPropertyKeys = [...commonPropertyKeys, 'isPinned', 'urlId'];
export interface Document extends DocumentProperties {
id: string;
workspace: Workspace;
access: roles.Role;
}
export interface PermissionDelta {
maxInheritedRole?: roles.BasicRole|null;
users?: {
// Maps from email to group name, or null to inherit.
[email: string]: roles.NonGuestRole|null
};
}
export interface PermissionData {
maxInheritedRole?: roles.BasicRole|null;
users: UserAccessData[];
}
// A structure for modifying managers of a billing account.
export interface ManagerDelta {
users: {
// To add a manager, link their email to 'managers'.
// To remove a manager, link their email to null.
// This format is used to rhyme with the ACL PermissionDelta format.
[email: string]: 'managers'|null
};
}
// Information about a user and their access to an unspecified resource of interest.
export interface UserAccessData {
id: number;
name: string;
email: string;
picture?: string|null; // When present, a url to a public image of unspecified dimensions.
// Represents the user's direct access to the resource of interest. Lack of access to a resource
// is represented by a null value.
access: roles.Role|null;
// A user's parentAccess represent their effective inheritable access to the direct parent of the resource
// of interest. The user's effective access to the resource of interest can be determined based
// on the user's parentAccess, the maxInheritedRole setting of the resource and the user's direct
// access to the resource. Lack of access to the parent resource is represented by a null value.
// If parent has non-inheritable access, this should be null.
parentAccess?: roles.BasicRole|null;
}
export interface ActiveSessionInfo {
user: FullUser & {helpScoutSignature?: string};
org: Organization|null;
orgError?: OrgError;
}
export interface OrgError {
error: string;
status: number;
}
/**
* Options to control the source of a document being replaced. For
* example, a document could be initialized from another document
* (e.g. a fork) or from a snapshot.
*/
export interface DocReplacementOptions {
sourceDocId?: string; // docId to copy from
snapshotId?: string; // s3 VersionId
}
/**
* Information about a single document snapshot/backup.
*/
export interface DocSnapshot {
lastModified: string; // when the snapshot was made
snapshotId: string; // the id of the snapshot in the underlying store
docId: string; // an id for accessing the snapshot as a Grist document
}
/**
* A list of document snapshots.
*/
export interface DocSnapshots {
snapshots: DocSnapshot[]; // snapshots, freshest first.
}
/**
* Information about a single document state.
*/
export interface DocState {
n: number; // a sequential identifier
h: string; // a hash identifier
}
/**
* A list of document states. Most recent is first.
*/
export interface DocStates {
states: DocState[];
}
/**
* A comparison between two documents, called "left" and "right".
* The comparison is based on the action histories in the documents.
* If those histories have been truncated, the comparison may report
* two documents as being unrelated even if they do in fact have some
* shared history.
*/
export interface DocStateComparison {
left: DocState; // left / local document
right: DocState; // right / remote document
parent: DocState|null; // most recent common ancestor of left and right
// summary of the relationship between the two documents.
// same: documents have the same most recent state
// left: the left document has actions not yet in the right
// right: the right document has actions not yet in the left
// both: both documents have changes (possible divergence)
// unrelated: no common history found
summary: 'same' | 'left' | 'right' | 'both' | 'unrelated';
}
export {UserProfile} from 'app/common/LoginSessionAPI';
export interface UserAPI {
getSessionActive(): Promise<ActiveSessionInfo>;
setSessionActive(email: string): Promise<void>;
getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}>;
getOrgs(merged?: boolean): Promise<Organization[]>;
getWorkspace(workspaceId: number): Promise<Workspace>;
getOrg(orgId: number|string): Promise<Organization>;
getOrgWorkspaces(orgId: number|string): Promise<Workspace[]>;
getDoc(docId: string): Promise<Document>;
newOrg(props: Partial<OrganizationProperties>): Promise<number>;
newWorkspace(props: Partial<WorkspaceProperties>, orgId: number|string): Promise<number>;
newDoc(props: Partial<DocumentProperties>, workspaceId: number): Promise<string>;
newUnsavedDoc(options?: {timezone?: string}): Promise<string>;
renameOrg(orgId: number|string, name: string): Promise<void>;
renameWorkspace(workspaceId: number, name: string): Promise<void>;
renameDoc(docId: string, name: string): Promise<void>;
updateDoc(docId: string, props: Partial<DocumentProperties>): Promise<void>;
deleteOrg(orgId: number|string): Promise<void>;
deleteWorkspace(workspaceId: number): Promise<void>; // delete workspace permanently
softDeleteWorkspace(workspaceId: number): Promise<void>; // soft-delete workspace
undeleteWorkspace(workspaceId: number): Promise<void>; // recover soft-deleted workspace
deleteDoc(docId: string): Promise<void>; // delete doc permanently
softDeleteDoc(docId: string): Promise<void>; // soft-delete doc
undeleteDoc(docId: string): Promise<void>; // recover soft-deleted doc
updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise<void>;
updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise<void>;
updateDocPermissions(docId: string, delta: PermissionDelta): Promise<void>;
getOrgAccess(orgId: number|string): Promise<PermissionData>;
getWorkspaceAccess(workspaceId: number): Promise<PermissionData>;
getDocAccess(docId: string): Promise<PermissionData>;
pinDoc(docId: string): Promise<void>;
unpinDoc(docId: string): Promise<void>;
moveDoc(docId: string, workspaceId: number): Promise<void>;
getUserProfile(): Promise<FullUser>;
updateUserName(name: string): Promise<void>;
getWorker(key: string): Promise<string>;
getWorkerAPI(key: string): Promise<DocWorkerAPI>;
getBillingAPI(): BillingAPI;
getDocAPI(docId: string): DocAPI;
fetchApiKey(): Promise<string>;
createApiKey(): Promise<string>;
deleteApiKey(): Promise<void>;
getTable(docId: string, tableName: string): Promise<TableColValues>;
applyUserActions(docId: string, actions: UserAction[]): Promise<ApplyUAResult>;
importUnsavedDoc(material: UploadType, options?: {
filename?: string,
timezone?: string,
onUploadProgress?: (ev: ProgressEvent) => void,
}): Promise<string>;
deleteUser(userId: number, name: string): Promise<void>;
getBaseUrl(): string; // Get the prefix for all the endpoints this object wraps.
forRemoved(): UserAPI; // Get a version of the API that works on removed resources.
}
/**
* Collect endpoints related to the content of a single document that we've been thinking
* of as the (restful) "Doc API". A few endpoints that could be here are not, for historical
* reasons, such as downloads.
*/
export interface DocAPI {
getRows(tableId: string): Promise<TableColValues>;
updateRows(tableId: string, changes: TableColValues): Promise<number[]>;
addRows(tableId: string, additions: BulkColValues): Promise<number[]>;
replace(source: DocReplacementOptions): Promise<void>;
getSnapshots(): Promise<DocSnapshots>;
forceReload(): Promise<void>;
compareState(remoteDocId: string): Promise<DocStateComparison>;
}
// Operations that are supported by a doc worker.
export interface DocWorkerAPI {
readonly url: string;
importDocToWorkspace(uploadId: number, workspaceId: number, settings?: BrowserSettings): Promise<DocCreationInfo>;
upload(material: UploadType, filename?: string): Promise<number>;
downloadDoc(docId: string, template?: boolean): Promise<Response>;
copyDoc(docId: string, template?: boolean, name?: string): Promise<number>;
}
export class UserAPIImpl extends BaseAPI implements UserAPI {
constructor(private _homeUrl: string, private _options: IOptions = {}) {
super(_options);
}
public forRemoved(): UserAPI {
const extraParameters = new Map<string, string>([['showRemoved', '1']]);
return new UserAPIImpl(this._homeUrl, {...this._options, extraParameters});
}
public async getSessionActive(): Promise<ActiveSessionInfo> {
return this.requestJson(`${this._url}/api/session/access/active`, {method: 'GET'});
}
public async setSessionActive(email: string): Promise<void> {
const body = JSON.stringify({ email });
return this.requestJson(`${this._url}/api/session/access/active`, {method: 'POST', body});
}
public async getSessionAll(): Promise<{users: FullUser[], orgs: Organization[]}> {
return this.requestJson(`${this._url}/api/session/access/all`, {method: 'GET'});
}
public async getOrgs(merged: boolean = false): Promise<Organization[]> {
return this.requestJson(`${this._url}/api/orgs?merged=${merged ? 1 : 0}`, { method: 'GET' });
}
public async getWorkspace(workspaceId: number): Promise<Workspace> {
return this.requestJson(`${this._url}/api/workspaces/${workspaceId}`, { method: 'GET' });
}
public async getOrg(orgId: number|string): Promise<Organization> {
return this.requestJson(`${this._url}/api/orgs/${orgId}`, { method: 'GET' });
}
public async getOrgWorkspaces(orgId: number|string): Promise<Workspace[]> {
return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces?includeSupport=1`,
{ method: 'GET' });
}
public async getDoc(docId: string): Promise<Document> {
return this.requestJson(`${this._url}/api/docs/${docId}`, { method: 'GET' });
}
public async newOrg(props: Partial<OrganizationProperties>): Promise<number> {
return this.requestJson(`${this._url}/api/orgs`, {
method: 'POST',
body: JSON.stringify(props)
});
}
public async newWorkspace(props: Partial<WorkspaceProperties>, orgId: number|string): Promise<number> {
return this.requestJson(`${this._url}/api/orgs/${orgId}/workspaces`, {
method: 'POST',
body: JSON.stringify(props)
});
}
public async newDoc(props: Partial<DocumentProperties>, workspaceId: number): Promise<string> {
return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/docs`, {
method: 'POST',
body: JSON.stringify(props)
});
}
public async newUnsavedDoc(options: {timezone?: string} = {}): Promise<string> {
return this.requestJson(`${this._url}/api/docs`, {
method: 'POST',
body: JSON.stringify(options),
});
}
public async renameOrg(orgId: number|string, name: string): Promise<void> {
await this.request(`${this._url}/api/orgs/${orgId}`, {
method: 'PATCH',
body: JSON.stringify({ name })
});
}
public async renameWorkspace(workspaceId: number, name: string): Promise<void> {
await this.request(`${this._url}/api/workspaces/${workspaceId}`, {
method: 'PATCH',
body: JSON.stringify({ name })
});
}
public async renameDoc(docId: string, name: string): Promise<void> {
return this.updateDoc(docId, {name});
}
public async updateDoc(docId: string, props: Partial<DocumentProperties>): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}`, {
method: 'PATCH',
body: JSON.stringify(props)
});
}
public async deleteOrg(orgId: number|string): Promise<void> {
await this.request(`${this._url}/api/orgs/${orgId}`, { method: 'DELETE' });
}
public async deleteWorkspace(workspaceId: number): Promise<void> {
await this.request(`${this._url}/api/workspaces/${workspaceId}`, { method: 'DELETE' });
}
public async softDeleteWorkspace(workspaceId: number): Promise<void> {
await this.request(`${this._url}/api/workspaces/${workspaceId}/remove`, { method: 'POST' });
}
public async undeleteWorkspace(workspaceId: number): Promise<void> {
await this.request(`${this._url}/api/workspaces/${workspaceId}/unremove`, { method: 'POST' });
}
public async deleteDoc(docId: string): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}`, { method: 'DELETE' });
}
public async softDeleteDoc(docId: string): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/remove`, { method: 'POST' });
}
public async undeleteDoc(docId: string): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/unremove`, { method: 'POST' });
}
public async updateOrgPermissions(orgId: number|string, delta: PermissionDelta): Promise<void> {
await this.request(`${this._url}/api/orgs/${orgId}/access`, {
method: 'PATCH',
body: JSON.stringify({ delta })
});
}
public async updateWorkspacePermissions(workspaceId: number, delta: PermissionDelta): Promise<void> {
await this.request(`${this._url}/api/workspaces/${workspaceId}/access`, {
method: 'PATCH',
body: JSON.stringify({ delta })
});
}
public async updateDocPermissions(docId: string, delta: PermissionDelta): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/access`, {
method: 'PATCH',
body: JSON.stringify({ delta })
});
}
public async getOrgAccess(orgId: number|string): Promise<PermissionData> {
return this.requestJson(`${this._url}/api/orgs/${orgId}/access`, { method: 'GET' });
}
public async getWorkspaceAccess(workspaceId: number): Promise<PermissionData> {
return this.requestJson(`${this._url}/api/workspaces/${workspaceId}/access`, { method: 'GET' });
}
public async getDocAccess(docId: string): Promise<PermissionData> {
return this.requestJson(`${this._url}/api/docs/${docId}/access`, { method: 'GET' });
}
public async pinDoc(docId: string): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/pin`, {
method: 'PATCH'
});
}
public async unpinDoc(docId: string): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/unpin`, {
method: 'PATCH'
});
}
public async moveDoc(docId: string, workspaceId: number): Promise<void> {
await this.request(`${this._url}/api/docs/${docId}/move`, {
method: 'PATCH',
body: JSON.stringify({ workspace: workspaceId })
});
}
public async getUserProfile(): Promise<FullUser> {
return this.requestJson(`${this._url}/api/profile/user`);
}
public async updateUserName(name: string): Promise<void> {
await this.request(`${this._url}/api/profile/user/name`, {
method: 'POST',
body: JSON.stringify({name})
});
}
public async getWorker(key: string): Promise<string> {
const json = await this.requestJson(`${this._url}/api/worker/${key}`, {
method: 'GET',
credentials: 'include'
});
return json.docWorkerUrl;
}
public async getWorkerAPI(key: string): Promise<DocWorkerAPI> {
const docUrl = this._urlWithOrg(await this.getWorker(key));
return new DocWorkerAPIImpl(docUrl, this._options);
}
public getBillingAPI(): BillingAPI {
return new BillingAPIImpl(this._url, this._options);
}
public getDocAPI(docId: string): DocAPI {
return new DocAPIImpl(this._url, docId, this._options);
}
public async fetchApiKey(): Promise<string> {
const resp = await this.fetch(`${this._url}/api/profile/apiKey`, {
credentials: 'include'
});
return await resp.text();
}
public async createApiKey(): Promise<string> {
const res = await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'POST'});
return await res.text();
}
public async deleteApiKey(): Promise<void> {
await this.fetch(`${this._url}/api/profile/apiKey`, {credentials: 'include', method: 'DELETE'});
}
// This method is not strictly needed anymore, but is widely used by
// tests so supporting as a handy shortcut for getDocAPI(docId).getRows(tableName)
public async getTable(docId: string, tableName: string): Promise<TableColValues> {
return this.getDocAPI(docId).getRows(tableName);
}
public async applyUserActions(docId: string, actions: UserAction[]): Promise<ApplyUAResult> {
return this.requestJson(`${this._url}/api/docs/${docId}/apply`, {
method: 'POST',
body: JSON.stringify(actions)
});
}
public async importUnsavedDoc(material: UploadType, options?: {
filename?: string,
timezone?: string,
onUploadProgress?: (ev: ProgressEvent) => void,
}): Promise<string> {
options = options || {};
const formData = this.newFormData();
formData.append('upload', material as any, options.filename);
if (options.timezone) { formData.append('timezone', options.timezone); }
const resp = await this.requestAxios(`${this._url}/api/docs`, {
headers: this._options.headers,
method: 'POST',
data: formData,
onUploadProgress: options.onUploadProgress,
});
return resp.data;
}
public async deleteUser(userId: number, name: string) {
await this.request(`${this._url}/api/users/${userId}`,
{method: 'DELETE',
body: JSON.stringify({name})});
}
public getBaseUrl(): string { return this._url; }
// Recomputes the URL on every call to pick up changes in the URL when switching orgs.
// (Feels inefficient, but probably doesn't matter, and it's simpler than the alternatives.)
private get _url(): string {
return this._urlWithOrg(this._homeUrl);
}
private _urlWithOrg(base: string): string {
return isClient() ? addCurrentOrgToPath(base) : base.replace(/\/$/, '');
}
}
export class DocWorkerAPIImpl extends BaseAPI implements DocWorkerAPI {
constructor(readonly url: string, private _options: IOptions = {}) {
super(_options);
}
public async importDocToWorkspace(uploadId: number, workspaceId: number, browserSettings?: BrowserSettings)
: Promise<DocCreationInfo> {
return this.requestJson(`${this.url}/api/workspaces/${workspaceId}/import`, {
method: 'POST',
body: JSON.stringify({ uploadId, browserSettings })
});
}
public async upload(material: UploadType, filename?: string): Promise<number> {
const formData = this.newFormData();
formData.append('upload', material as any, filename);
const json = await this.requestJson(`${this.url}/uploads`, {
headers: this._options.headers,
method: 'POST',
body: formData
});
return json.uploadId;
}
public async downloadDoc(docId: string, template: boolean = false): Promise<Response> {
const extra = template ? '&template=1' : '';
const result = await this.request(`${this.url}/download?doc=${docId}${extra}`, {
headers: this._options.headers,
method: 'GET',
});
if (!result.ok) { throw new Error(await result.text()); }
return result;
}
public async copyDoc(docId: string, template: boolean = false, name?: string): Promise<number> {
const url = new URL(`${this.url}/copy?doc=${docId}`);
if (template) {
url.searchParams.append('template', '1');
}
if (name) {
url.searchParams.append('name', name);
}
const json = await this.requestJson(url.href, {
headers: this._options.headers,
method: 'POST',
});
return json.uploadId;
}
}
export class DocAPIImpl extends BaseAPI implements DocAPI {
private _url: string;
constructor(url: string, readonly docId: string, options: IOptions = {}) {
super(options);
this._url = `${url}/api/docs/${docId}`;
}
public async getRows(tableId: string): Promise<TableColValues> {
return this.requestJson(`${this._url}/tables/${tableId}/data`);
}
public async updateRows(tableId: string, changes: TableColValues): Promise<number[]> {
return this.requestJson(`${this._url}/tables/${tableId}/data`, {
body: JSON.stringify(changes),
method: 'PATCH'
});
}
public async addRows(tableId: string, additions: BulkColValues): Promise<number[]> {
return this.requestJson(`${this._url}/tables/${tableId}/data`, {
body: JSON.stringify(additions),
method: 'POST'
});
}
public async replace(source: DocReplacementOptions): Promise<void> {
return this.requestJson(`${this._url}/replace`, {
body: JSON.stringify(source),
method: 'POST'
});
}
public async getSnapshots(): Promise<DocSnapshots> {
return this.requestJson(`${this._url}/snapshots`);
}
public async forceReload(): Promise<void> {
await this.request(`${this._url}/force-reload`, {
method: 'POST'
});
}
public async compareState(remoteDocId: string): Promise<DocStateComparison> {
return this.requestJson(`${this._url}/compare/${remoteDocId}`);
}
}

@ -0,0 +1,31 @@
/*
* Interface for the user's config found in config.json.
*/
export interface UserConfig {
enableMetrics?: boolean;
docListSortBy?: string;
docListSortDir?: number;
features?: ISupportedFeatures;
/*
* The host serving the untrusted content: on dev environment could be
* "http://getgrist.localtest.me". Port is added at runtime and should not be included.
*/
untrustedContentOrigin?: string;
}
export interface ISupportedFeatures {
signin?: boolean;
sharing?: boolean;
proxy?: boolean; // If true, Grist will accept login information via http headers
// X-Forwarded-User and X-Forwarded-Email. Set to true only if
// Grist is behind a reverse proxy that is managing those headers,
// otherwise they could be spoofed.
formulaBar?: boolean;
// Plugin views, REPL, and Validations all need work, but are exposed here to allow existing
// tests to continue running. These only affect client-side code.
customViewPlugin?: boolean;
replTool?: boolean;
validationsTool?: boolean;
}

@ -0,0 +1,144 @@
// tslint:disable:max-classes-per-file
import {CellValue} from 'app/common/DocActions';
import * as gristTypes from 'app/common/gristTypes';
import * as gutil from 'app/common/gutil';
import {buildNumberFormat, NumberFormatOptions} from 'app/common/NumberFormat';
import * as moment from 'moment-timezone';
// Some text to show on cells whose values are pending.
export const PENDING_DATA_PLACEHOLDER = "Loading...";
/**
* Formats a custom object received as a value in a DocAction, as "Constructor(args...)".
* E.g. ["Foo", 1, 2, 3] becomes the string "Foo(1, 2, 3)".
*/
export function formatObject(args: [string, ...any[]]): string {
const objType = args[0], objArgs = args.slice(1);
switch (objType) {
case 'L': return JSON.stringify(objArgs);
// First arg is seconds since epoch (moment takes ms), second arg is timezone
case 'D': return moment.tz(objArgs[0] * 1000, objArgs[1]).format("YYYY-MM-DD HH:mm:ssZ");
case 'd': return moment.tz(objArgs[0] * 1000, 'UTC').format("YYYY-MM-DD");
case 'R': return `${objArgs[0]}[${objArgs[1]}]`;
case 'E': return gristTypes.formatError(args);
case 'P': return PENDING_DATA_PLACEHOLDER;
}
return objType + "(" + JSON.stringify(objArgs).slice(1, -1) + ")";
}
/**
* Formats a value of unknown type, using formatObject() for encoded objects.
*/
export function formatUnknown(value: any): string {
return gristTypes.isObject(value) ? formatObject(value) : (value == null ? "" : String(value));
}
export type IsRightTypeFunc = (value: CellValue) => boolean;
export class BaseFormatter {
public readonly isRightType: IsRightTypeFunc;
constructor(public type: string, public opts: object) {
this.isRightType = gristTypes.isRightType(gristTypes.extractTypeFromColType(type)) ||
gristTypes.isRightType('Any')!;
}
/**
* Formats a value that matches the type of this formatter. This should be overridden by derived
* classes to handle values in formatter-specific ways.
*/
public format(value: any): string {
return value;
}
/**
* Formats using this.format() if a value is of the right type for this formatter, or using
* AnyFormatter otherwise. This method the recommended API. There is no need to override it.
*/
public formatAny(value: any): string {
return this.isRightType(value) ? this.format(value) : formatUnknown(value);
}
}
class AnyFormatter extends BaseFormatter {
public format(value: any): string {
return formatUnknown(value);
}
}
export class NumericFormatter extends BaseFormatter {
private _numFormat: Intl.NumberFormat;
private _formatter: (val: number) => string;
constructor(type: string, options: NumberFormatOptions) {
super(type, options);
this._numFormat = buildNumberFormat(options);
this._formatter = (options.numSign === 'parens') ? this._formatParens : this._formatPlain;
}
public format(value: any): string {
return value === null ? '' : this._formatter(value);
}
public _formatPlain(value: number): string {
return this._numFormat.format(value);
}
public _formatParens(value: number): string {
// Surround positive numbers with spaces to align them visually to parenthesized numbers.
return (value >= 0) ?
` ${this._numFormat.format(value)} ` :
`(${this._numFormat.format(-value)})`;
}
}
class IntFormatter extends NumericFormatter {
constructor(type: string, opts: object) {
super(type, {decimals: 0, ...opts});
}
}
class DateFormatter extends BaseFormatter {
private _dateTimeFormat: string;
private _timezone: string;
constructor(type: string, opts: {dateFormat?: string}, timezone: string = 'UTC') {
super(type, opts);
this._dateTimeFormat = opts.dateFormat || 'YYYY-MM-DD';
this._timezone = timezone;
}
public format(value: any): string {
if (value === null) { return ''; }
const time = moment.tz(value * 1000, this._timezone);
return time.format(this._dateTimeFormat);
}
}
class DateTimeFormatter extends DateFormatter {
constructor(type: string, opts: {dateFormat?: string; timeFormat?: string}) {
const timezone = gutil.removePrefix(type, "DateTime:") || '';
const timeFormat = opts.timeFormat === undefined ? 'h:mma' : opts.timeFormat;
const dateFormat = (opts.dateFormat || 'YYYY-MM-DD') + " " + timeFormat;
super(type, {dateFormat}, timezone);
}
}
const formatters: {[name: string]: typeof BaseFormatter} = {
Numeric: NumericFormatter,
Int: IntFormatter,
Bool: BaseFormatter,
Date: DateFormatter,
DateTime: DateTimeFormatter,
// We don't list anything that maps to AnyFormatter, since that's the default.
};
/**
* Takes column type and widget options and returns a constructor with a format function that can
* properly convert a value passed to it into the right format for that column.
*/
export function createFormatter(type: string, opts: object): BaseFormatter {
const ctor = formatters[gristTypes.extractTypeFromColType(type)] || AnyFormatter;
return new ctor(type, opts);
}

@ -0,0 +1,28 @@
/**
* Functions to convert between an array of bytes and a string. The implementations are
* different for Node and for the browser.
*/
declare const TextDecoder: any, TextEncoder: any;
export let arrayToString: (data: Uint8Array) => string;
export let stringToArray: (data: string) => Uint8Array;
if (typeof TextDecoder !== 'undefined') {
// Note that constructing a TextEncoder/Decoder takes time, so it's faster to reuse.
const dec = new TextDecoder('utf8');
const enc = new TextEncoder('utf8');
arrayToString = function(uint8Array: Uint8Array): string {
return dec.decode(uint8Array);
};
stringToArray = function(str: string): Uint8Array {
return enc.encode(str);
};
} else {
arrayToString = function(uint8Array: Uint8Array): string {
return Buffer.from(uint8Array).toString('utf8');
};
stringToArray = function(str: string): Uint8Array {
return new Uint8Array(Buffer.from(str, 'utf8'));
};
}

@ -0,0 +1,5 @@
declare module "app/common/MemBuffer" {
const MemBuffer: any;
type MemBuffer = any;
export = MemBuffer;
}

@ -0,0 +1,7 @@
/**
* Returns a promise that resolves in the given number of milliseconds.
* (A replica of bluebird.delay using native promises.)
*/
export function delay(msec: number): Promise<void> {
return new Promise<void>((resolve) => setTimeout(resolve, msec));
}

@ -0,0 +1,43 @@
/**
*
* Utilities related to email normalization. Currently
* trivial, but could potentially need special per-domain
* rules in future.
*
* Email addresses are a bit slippery. Domain names are
* case insensitive, but user names may or may not be,
* depending on the mail server handling the domain.
* Other special treatment of user names may also be in
* place for particular domains (periods, plus sign, etc).
*
* We treat emails as case-insensitive for the purposes
* of determining equality of emails, and indexing users
* by email address.
*
*/
/**
*
* Convert the supplied email address to a normalized form
* that we will use for indexing and equality tests.
* Many possible email addresses could map to the same
* normalized result; as far as we are concerned those
* addresses are equivalent.
*
* The normalization we do is a simple lowercase. This
* means we won't be able to treat both Jane@x.y and
* jane@x.y as separate email addresses, even through
* they may in fact be separate mailboxes on x.y.
*
* The normalized email is not something we should show
* the user in the UI, but is rather for internal purposes.
*
* The original non-normalized email is called a
* "display email" to distinguish it from a "normalized
* email"
*
*/
export function normalizeEmail(displayEmail: string): string {
// We take the lower case, without use of locale.
return displayEmail.toLowerCase();
}

@ -0,0 +1,242 @@
import {CellValue} from 'app/common/DocActions';
import isString = require('lodash/isString');
// tslint:disable:object-literal-key-quotes
export type GristType = 'Any' | 'Attachments' | 'Blob' | 'Bool' | 'Choice' | 'Date' | 'DateTime' |
'Id' | 'Int' | 'ManualSortPos' | 'Numeric' | 'PositionNumber' | 'Ref' | 'RefList' | 'Text';
// Letter codes for CellValue types encoded as [code, args...] tuples.
export type GristObjType = 'L' | 'D' | 'd' | 'R' | 'E' | 'P';
export const MANUALSORT = 'manualSort';
// This mapping includes both the default value, and its representation for SQLite.
const _defaultValues: {[key in GristType]: [CellValue, string]} = {
'Any': [ null, "NULL" ],
'Attachments': [ null, "NULL" ],
'Blob': [ null, "NULL" ],
// Bool is only supported by SQLite as 0 and 1 values.
'Bool': [ false, "0" ],
'Choice': [ '', "''" ],
'Date': [ null, "NULL" ],
'DateTime': [ null, "NULL" ],
'Id': [ 0, "0" ],
'Int': [ 0, "0" ],
// Note that "1e999" is a way to store Infinity into SQLite. This is verified by "Defaults"
// tests in DocStorage.js. See also http://sqlite.1065341.n5.nabble.com/Infinity-td55327.html.
'ManualSortPos': [ Number.POSITIVE_INFINITY, "1e999" ],
'Numeric': [ 0, "0" ],
'PositionNumber': [ Number.POSITIVE_INFINITY, "1e999" ],
'Ref': [ 0, "0" ],
'RefList': [ null, "NULL" ],
'Text': [ '', "''" ],
};
/**
* Given a grist column type (e.g Text, Numeric, ...) returns the default value for that type.
* If options.sqlFormatted is true, returns the representation of the value for SQLite.
*/
export function getDefaultForType(colType: string, options: {sqlFormatted?: boolean} = {}) {
const type = extractTypeFromColType(colType);
return (_defaultValues[type as GristType] || _defaultValues.Any)[options.sqlFormatted ? 1 : 0];
}
/**
* Returns whether a value (as received in a DocAction) represents a custom object.
*/
export function isObject(value: CellValue): value is [string, any?] {
return Array.isArray(value);
}
/**
* Returns whether a value (as received in a DocAction) represents a raised exception.
*/
export function isRaisedException(value: CellValue): boolean {
return Array.isArray(value) && value[0] === 'E';
}
/**
* Returns whether a value (as received in a DocAction) represents a list or is null,
* which is a valid value for list types in grist.
*/
export function isListOrNull(value: CellValue): boolean {
return value === null || (Array.isArray(value) && value[0] === 'L');
}
/**
* Returns whether a value (as received in a DocAction) represents an empty list.
*/
export function isEmptyList(value: CellValue): boolean {
return Array.isArray(value) && value.length === 1 && value[0] === 'L';
}
/**
* Returns whether a value (as received in a DocAction) represents a "Pending" value.
*/
export function isPending(value: CellValue): boolean {
return Array.isArray(value) && value[0] === 'P';
}
/**
* Formats a raised exception (a value for which isRaisedException is true) for display in a cell.
* This is designed to look somewhat similar to Excel, e.g. #VALUE or #DIV/0!"
*/
export function formatError(value: [string, ...any[]]): string {
const errName = value[1];
switch (errName) {
case 'ZeroDivisionError': return '#DIV/0!';
case 'UnmarshallableError': return value[3] || ('#' + errName);
case 'InvalidTypedValue': return `#Invalid ${value[2]}: ${value[3]}`;
}
return '#' + errName;
}
function isNumber(v: CellValue) { return typeof v === 'number' || typeof v === 'boolean'; }
function isNumberOrNull(v: CellValue) { return isNumber(v) || v === null; }
function isBoolean(v: CellValue) { return typeof v === 'boolean' || v === 1 || v === 0; }
function isNormalValue(value: CellValue) {
return !(Array.isArray(value) && (value[0] === 'E' || value[0] === 'P'));
}
/**
* Map of Grist type to an "isRightType" checker function, which determines if a given values type
* matches the declared type of the column.
*/
const rightType: {[key in GristType]: (value: CellValue) => boolean} = {
Any: isNormalValue,
Attachments: isListOrNull,
Text: isString,
Blob: isString,
Int: isNumberOrNull,
Bool: isBoolean,
Date: isNumberOrNull,
DateTime: isNumberOrNull,
Numeric: isNumberOrNull,
Id: isNumber,
PositionNumber: isNumber,
ManualSortPos: isNumber,
Ref: isNumber,
RefList: isListOrNull,
Choice: (v: CellValue, options?: any) => {
// TODO widgets options should not be used outside of the client. They are an instance of
// modelUtil.jsonObservable, passed in by FieldBuilder.
if (v === '') {
// Accept empty-string values as valid
return true;
} else if (options) {
const choices = options().choices;
return Array.isArray(choices) && choices.includes(v);
} else {
return false;
}
}
};
export function isRightType(type: string): undefined | ((value: CellValue) => boolean) {
return rightType[type as GristType];
}
export function extractTypeFromColType(type: string): string {
if (!type) { return type; }
const colon = type.indexOf(':');
return (colon === -1 ? type : type.slice(0, colon));
}
/**
* Convert pureType to Grist python type name, e.g. 'Ref' to 'Reference'.
*/
export function getGristType(pureType: string): string {
switch (pureType) {
case 'Ref': return 'Reference';
case 'RefList': return 'ReferenceList';
default: return pureType;
}
}
/**
* Converts SQL type strings produced by the Sequelize library into its corresponding
* Grist type. The list of types is based on an analysis of SQL type string outputs
* produced by the Sequelize library (mostly covered in lib/data-types.js). Some
* additional engine/dialect specific types are detailed in dialect directories.
*
* TODO: A handful of exotic SQL types (mostly from PostgreSQL) will currently throw an
* Error, rather than returning a type. Further testing is required to determine
* whether Grist can manage those data types.
*
* @param {String} sqlType A string produced by Sequelize's describeTable query
* @return {String} The corresponding Grist type string
* @throws {Error} If the sqlType is unrecognized or unsupported
*/
export function sequelizeToGristType(sqlType: string): GristType {
// Sequelize type strings can include parens (e.g., `CHAR(10)`). This function
// ignores those additional details when determining the Grist type.
let endMarker = sqlType.length;
const parensMarker = sqlType.indexOf('(');
endMarker = parensMarker > 0 ? parensMarker : endMarker;
// Type strings might also include a space after the basic type description.
// The type `DOUBLE PRECISION` is one such example, but modifiers or attributes
// relevant to the type might also appear after the type itself (e.g., UNSIGNED,
// NONZERO). These are ignored when determining the Grist type.
const spaceMarker = sqlType.indexOf(' ');
endMarker = spaceMarker > 0 && spaceMarker < endMarker ? spaceMarker : endMarker;
switch (sqlType.substring(0, endMarker)) {
case 'INTEGER':
case 'BIGINT':
case 'SMALLINT':
case 'INT':
return 'Int';
case 'NUMBER':
case 'FLOAT':
case 'DECIMAL':
case 'NUMERIC':
case 'REAL':
case 'DOUBLE':
case 'DOUBLE PRECISION':
return 'Numeric';
case 'BOOLEAN':
case 'TINYINT':
return 'Bool';
case 'STRING':
case 'CHAR':
case 'TEXT':
case 'UUID':
case 'UUIDV1':
case 'UUIDV4':
case 'VARCHAR':
case 'NVARCHAR':
case 'TINYTEXT':
case 'MEDIUMTEXT':
case 'LONGTEXT':
case 'ENUM':
return 'Text';
case 'TIME':
case 'DATE':
case 'DATEONLY':
case 'DATETIME':
case 'NOW':
return 'Text';
case 'BLOB':
case 'TINYBLOB':
case 'MEDIUMBLOB':
case 'LONGBLOB':
// TODO: Passing binary data to the Sandbox is throwing Errors. Proper support
// for these Blob data types requires some more investigation.
throw new Error('SQL type: `' + sqlType + '` is currently unsupported');
case 'NONE':
case 'HSTORE':
case 'JSON':
case 'JSONB':
case 'VIRTUAL':
case 'ARRAY':
case 'RANGE':
case 'GEOMETRY':
throw new Error('SQL type: `' + sqlType + '` is currently untested');
default:
throw new Error('Unrecognized datatype: `' + sqlType + '`');
}
}

@ -1 +1,583 @@
export type ProductFlavor = 'grist';
import {BillingPage, BillingSubPage, BillingTask} from 'app/common/BillingAPI';
import {OpenDocMode} from 'app/common/DocListAPI';
import {encodeQueryParams} from 'app/common/gutil';
import {localhostRegex} from 'app/common/LoginState';
import {Document} from 'app/common/UserAPI';
import identity = require('lodash/identity');
import pickBy = require('lodash/pickBy');
import {StringUnion} from './StringUnion';
export type IDocPage = number | 'new' | 'code';
// What page to show in the user's home area. Defaults to 'workspace' if a workspace is set, and
// to 'all' otherwise.
export const HomePage = StringUnion('all', 'workspace', 'trash');
export type IHomePage = typeof HomePage.type;
export const WelcomePage = StringUnion('user', 'teams');
export type WelcomePage = typeof WelcomePage.type;
// Default subdomain for home api service if not otherwise specified.
export const DEFAULT_HOME_SUBDOMAIN = 'api';
// This is the minimum length a urlId may have if it is chosen
// as a prefix of the docId.
export const MIN_URLID_PREFIX_LENGTH = 12;
/**
* Special ways to open a document, based on what the user intends to do.
* - view: Open document in read-only mode (even if user has edit rights)
* - fork: Open document in fork-ready mode. This means that while edits are
* permitted, those edits should go to a copy of the document rather than
* the original.
*/
export const commonUrls = {
help: "https://support.getgrist.com",
plans: "https://www.getgrist.com/pricing",
efcrConnect: 'https://efc-r.com/connect',
efcrHelp: 'https://www.nioxus.info/eFCR-Help',
};
/**
* Values representable in a URL. The current state is available as urlState().state observable
* in client. Updates to this state are expected by functions such as makeUrl() and setLinkUrl().
*/
export interface IGristUrlState {
org?: string;
homePage?: IHomePage;
ws?: number;
doc?: string;
slug?: string; // if present, this is based on the document title, and is not a stable id
mode?: OpenDocMode;
fork?: UrlIdParts;
docPage?: IDocPage;
newui?: boolean;
billing?: BillingPage;
welcome?: WelcomePage;
params?: {
billingPlan?: string;
billingTask?: BillingTask;
};
hash?: HashLink; // if present, this specifies an individual row within a section of a page.
}
// Subset of GristLoadConfig used by getOrgUrlInfo(), which affects the interpretation of the
// current URL.
export interface OrgUrlOptions {
// The org associated with the current URL.
org?: string;
// Base domain for constructing new URLs, should start with "." and not include port, e.g.
// ".getgrist.com". It should be unset for localhost operation and in single-org mode.
baseDomain?: string;
// In single-org mode, this is the single well-known org.
singleOrg?: string;
}
// Result of getOrgUrlInfo().
export interface OrgUrlInfo {
hostname?: string; // If hostname should be changed to access the requested org.
orgInPath?: string; // If /o/{orgInPath} should be used to access the requested org.
}
export function isCustomHost(hostname: string, baseDomain: string) {
// .localtest.me is used by plugin tests for arcane reasons.
return hostname !== 'localhost' && !hostname.endsWith(baseDomain) && !hostname.endsWith('.localtest.me');
}
export function getOrgUrlInfo(newOrg: string, currentHostname: string, options: OrgUrlOptions): OrgUrlInfo {
if (newOrg === options.singleOrg) {
return {};
}
if (!options.baseDomain || currentHostname === 'localhost') {
return {orgInPath: newOrg};
}
if (newOrg === options.org && isCustomHost(currentHostname, options.baseDomain)) {
return {};
}
return {hostname: newOrg + options.baseDomain};
}
/**
* The actual serialization of a url state into a URL. The URL has the form
* <org-base>/
* <org-base>/ws/<ws>/
* <org-base>/doc/<doc>[/p/<docPage>]
*
* where <org-base> depends on whether subdomains are in use, e.g.
* <org>.getgrist.com
* localhost:8080/o/<org>
*/
export function encodeUrl(gristConfig: Partial<GristLoadConfig>,
state: IGristUrlState, baseLocation: Location | URL): string {
const url = new URL(baseLocation.href);
const parts = ['/'];
if (state.org) {
// We figure out where to stick the org using the gristConfig and the current host.
const {hostname, orgInPath} = getOrgUrlInfo(state.org, baseLocation.hostname, gristConfig);
if (hostname) {
url.hostname = hostname;
}
if (orgInPath) {
parts.push(`o/${orgInPath}/`);
}
}
if (state.ws) { parts.push(`ws/${state.ws}/`); }
if (state.doc) {
if (state.slug) {
parts.push(`${encodeURIComponent(state.doc)}/${encodeURIComponent(state.slug)}`);
} else {
parts.push(`doc/${encodeURIComponent(state.doc)}`);
}
if (state.mode && parseOpenDocMode(state.mode)) {
parts.push(`/m/${state.mode}`);
}
if (state.docPage) {
parts.push(`/p/${state.docPage}`);
}
} else {
if (state.homePage === 'trash') { parts.push('p/trash'); }
}
if (state.billing) {
parts.push(state.billing === 'billing' ? 'billing' : `billing/${state.billing}`);
}
if (state.welcome) {
parts.push(`welcome/${state.welcome}`);
}
const queryParams = pickBy(state.params, identity) as {[key: string]: string};
if (state.newui !== undefined) {
queryParams.newui = state.newui ? '1' : '0';
}
const hashParts: string[] = [];
if (state.hash && state.hash.rowId) {
const hash = state.hash;
hashParts.push(`a1`);
for (const key of ['sectionId', 'rowId', 'colRef'] as Array<keyof HashLink>) {
if (hash[key]) { hashParts.push(`${key[0]}${hash[key]}`); }
}
}
const queryStr = encodeQueryParams(queryParams);
url.pathname = parts.join('');
url.search = queryStr;
if (state.hash) {
// Project tests use hashes, so only set hash if there is an anchor.
url.hash = hashParts.join('.');
}
return url.href;
}
/**
* Parse a URL location into an IGristUrlState object. See encodeUrl() documentation.
*/
export function decodeUrl(gristConfig: Partial<GristLoadConfig>, location: Location | URL): IGristUrlState {
const parts = location.pathname.slice(1).split('/');
const map = new Map<string, string>();
for (let i = 0; i < parts.length; i += 2) {
map.set(parts[i], decodeURIComponent(parts[i + 1]));
}
// When the urlId is a prefix of the docId, documents are identified
// as "<urlId>/slug" instead of "doc/<urlId>". We can detect that because
// the minimum length of a urlId prefix is longer than the maximum length
// of any of the valid keys in the url.
for (const key of map.keys()) {
if (key.length >= MIN_URLID_PREFIX_LENGTH) {
map.set('doc', key);
map.set('slug', map.get(key)!);
map.delete(key);
break;
}
}
const state: IGristUrlState = {};
const subdomain = parseSubdomain(location.host);
if (gristConfig.org || gristConfig.singleOrg) {
state.org = gristConfig.org || gristConfig.singleOrg;
} else if (!gristConfig.pathOnly && subdomain.org) {
state.org = subdomain.org;
}
const sp = new URLSearchParams(location.search);
if (location.search) { state.params = {}; }
if (map.has('o')) { state.org = map.get('o'); }
if (map.has('ws')) { state.ws = parseInt(map.get('ws')!, 10); }
if (map.has('doc')) {
state.doc = map.get('doc');
const fork = parseUrlId(map.get('doc')!);
if (fork.forkId) { state.fork = fork; }
if (map.has('slug')) { state.slug = map.get('slug'); }
if (map.has('p')) { state.docPage = parseDocPage(map.get('p')!); }
} else {
if (map.has('p')) {
const p = map.get('p')!;
state.homePage = HomePage.guard(p) ? p : undefined;
}
}
if (map.has('m')) { state.mode = parseOpenDocMode(map.get('m')!); }
if (sp.has('newui')) { state.newui = useNewUI(sp.get('newui') ? sp.get('newui') === '1' : undefined); }
if (map.has('billing')) { state.billing = parseBillingPage(map.get('billing')!); }
if (map.has('welcome')) { state.welcome = parseWelcomePage(map.get('welcome')!); }
if (sp.has('billingPlan')) { state.params!.billingPlan = sp.get('billingPlan')!; }
if (sp.has('billingTask')) {
state.params!.billingTask = parseBillingTask(sp.get('billingTask')!);
}
if (location.hash) {
const hash = location.hash;
const hashParts = hash.split('.');
const hashMap = new Map<string, string>();
for (const part of hashParts) {
hashMap.set(part.slice(0, 1), part.slice(1));
}
if (hashMap.has('#') && hashMap.get('#') === 'a1') {
const link: HashLink = {};
for (const key of ['sectionId', 'rowId', 'colRef'] as Array<keyof HashLink>) {
const ch = key.substr(0, 1);
if (hashMap.has(ch)) { link[key] = parseInt(hashMap.get(ch)!, 10); }
}
state.hash = link;
}
}
return state;
}
export function useNewUI(newui: boolean|undefined) {
return newui !== false;
}
/**
* parseDocPage is a noop if p is 'new' or 'code', otherwise parse to integer
*/
function parseDocPage(p: string) {
if (['new', 'code'].includes(p)) {
return p as 'new'|'code';
}
return parseInt(p, 10);
}
/**
* parseBillingPage ensures that the billing page value is a valid BillingPageType.
*/
function parseBillingPage(p: string): BillingPage {
return BillingSubPage.guard(p) ? p : 'billing';
}
/**
* parseBillingTask ensures that the value is a valid BillingTask or undefined.
*/
function parseBillingTask(t: string): BillingTask|undefined {
return BillingTask.guard(t) ? t : undefined;
}
/**
* parseOpenDocMode ensures that the value is a valid OpenDocMode or undefined.
*/
function parseOpenDocMode(p: string): OpenDocMode|undefined {
return OpenDocMode.guard(p) ? p : undefined;
}
/**
* parse welcome page ensure that the value is a valid WelcomePage, default to 'user' if not.
*/
function parseWelcomePage(p: string): WelcomePage {
return WelcomePage.guard(p) ? p : 'user';
}
/**
* Parses the URL like "foo.bar.baz" into the pair {org: "foo", base: ".bar.baz"}.
* Port is allowed and included into base.
*
* The "base" part is required to have at least two periods. The "org" part must pass
* the subdomainRegex test.
*
* If there's no way to parse the URL into such a pair, then an empty object is returned.
*/
export function parseSubdomain(host: string|undefined): {org?: string, base?: string} {
if (!host) { return {}; }
const match = /^([^.]+)(\..+\..+)$/.exec(host.toLowerCase());
if (match) {
const org = match[1];
const base = match[2];
if (subdomainRegex.exec(org)) {
return {org, base};
}
}
// Host has nowhere to put a subdomain.
return {};
}
/**
* Like parseSubdomain, but throws an error if neither of these cases apply:
* - host can be parsed into a valid subdomain and a valid base domain.
* - host is localhost:NNNN
* An empty object is only returned when host is localhost:NNNN.
*/
export function parseSubdomainStrictly(host: string|undefined): {org?: string, base?: string} {
if (!host) { throw new Error('host not known'); }
const result = parseSubdomain(host);
if (result.org) { return result; }
if (!host.match(localhostRegex)) {
throw new Error(`host not understood: ${host}`);
}
// Host is localhost[:NNNN], no org available.
return {};
}
/**
* These settings get sent to the client along with the loaded page. At the minimum, the browser
* needs to know the URL of the home API server (e.g. api.getgrist.com).
*/
export interface GristLoadConfig {
// URL of the Home API server for the browser client to use.
homeUrl: string|null;
// When loading /doc/{docId}, we include the id used to assign the document (this is the docId).
assignmentId?: string;
// Org or "subdomain". When present, this overrides org information from the hostname. We rely
// on this for custom domains, but set it generally for all pages.
org?: string;
// Base domain for constructing new URLs, should start with "." and not include port, e.g.
// ".getgrist.com". It should be unset for localhost operation and in single-org mode.
baseDomain?: string;
// In single-org mode, this is the single well-known org. Suppress any org selection UI.
singleOrg?: string;
// When set, this directs the client to encode org information in path, not in domain.
pathOnly?: boolean;
// Type of error page to show. This is used for pages such as "signed-out" and "not-found",
// which don't include the full app.
errPage?: string;
// When errPage is a generic "other-error", this is the message to show.
errMessage?: string;
// URL for client to use for untrusted content.
pluginUrl?: string|null;
// Stripe API key for use on the client.
stripeAPIKey?: string;
// BeaconID for the support widget from HelpScout.
helpScoutBeaconId?: string;
// If set, enable anonymous sharing UI elements.
supportAnon?: boolean;
// Max upload allowed for imports (except .grist files), in bytes; 0 or omitted for unlimited.
maxUploadSizeImport?: number;
// Max upload allowed for attachments, in bytes; 0 or omitted for unlimited.
maxUploadSizeAttachment?: number;
// Pre-fetched call to getDoc for the doc being loaded.
getDoc?: {[id: string]: Document};
// Pre-fetched call to getWorker for the doc being loaded.
getWorker?: {[id: string]: string};
// The timestamp when this gristConfig was generated.
timestampMs: number;
}
// Acceptable org subdomains are alphanumeric (hyphen also allowed) and of
// non-zero length.
const subdomainRegex = /^[-a-z0-9]+$/i;
export interface OrgParts {
subdomain: string|null;
orgFromHost: string|null;
orgFromPath: string|null;
pathRemainder: string;
mismatch: boolean;
}
/**
* Returns true if code is running in client, false if running in server.
*/
export function isClient() {
return (typeof window !== 'undefined') && window && window.location && window.location.hostname;
}
/**
* Returns a known org "subdomain" if Grist is configured in single-org mode
* (GRIST_SINGLE_ORG=<org> on the server) or if the page includes an org in gristConfig.
*/
export function getKnownOrg(): string|null {
if (isClient()) {
const gristConfig: GristLoadConfig = (window as any).gristConfig;
return (gristConfig && gristConfig.org) || null;
} else {
return process.env.GRIST_SINGLE_ORG || null;
}
}
/**
* Returns true if org must be encoded in path, not in domain. Determined from
* gristConfig on the client. On on the server returns true if the host is
* supplied and is 'localhost', or if GRIST_ORG_IN_PATH is set to 'true'.
*/
export function isOrgInPathOnly(host?: string): boolean {
if (isClient()) {
const gristConfig: GristLoadConfig = (window as any).gristConfig;
return (gristConfig && gristConfig.pathOnly) || false;
} else {
if (host && host.match(/^localhost(:[0-9]+)?$/)) { return true; }
return (process.env.GRIST_ORG_IN_PATH === 'true');
}
}
// Extract an organization name from the host. Returns null if an organization name
// could not be recovered. Organization name may be overridden by server configuration.
export function getOrgFromHost(reqHost: string): string|null {
const singleOrg = getKnownOrg();
if (singleOrg) { return singleOrg; }
if (isOrgInPathOnly()) { return null; }
return parseSubdomain(reqHost).org || null;
}
/**
* Get any information about an organization that is embedded in the host name or the
* path.
* For example, on nasa.getgrist.com, orgFromHost and subdomain will be set to "nasa".
* On localhost:8000/o/nasa, orgFromPath and subdomain will be set to "nasa".
* On nasa.getgrist.com/o/nasa, orgFromHost, orgFromPath, and subdomain will all be "nasa".
* On spam.getgrist.com/o/nasa, orgFromHost will be "spam", orgFromPath will be "nasa",
* subdomain will be null, and mismatch will be true.
*/
export function extractOrgParts(reqHost: string|undefined, reqPath: string): OrgParts {
let orgFromHost: string|null = getKnownOrg();
if (!orgFromHost && reqHost) {
orgFromHost = getOrgFromHost(reqHost);
if (orgFromHost) {
// Some subdomains are shared, and do not reflect the name of an organization.
// See https://phab.getgrist.com/w/hosting/v1/urls/ for a list.
if (/^(api|v1-.*|doc-worker-.*)$/.test(orgFromHost)) {
orgFromHost = null;
}
}
}
const part = parseFirstUrlPart('o', reqPath);
if (part.value) {
const orgFromPath = part.value.toLowerCase();
const mismatch = Boolean(orgFromHost && orgFromPath && (orgFromHost !== orgFromPath));
const subdomain = mismatch ? null : orgFromPath;
return {orgFromHost, orgFromPath, pathRemainder: part.path, mismatch, subdomain};
}
return {orgFromHost, orgFromPath: null, pathRemainder: reqPath, mismatch: false, subdomain: orgFromHost};
}
/**
* When a prefix is extracted from the path, the remainder of the path may be empty.
* This method makes sure there is at least a "/".
*/
export function sanitizePathTail(path: string|undefined) {
path = path || '/';
return (path.startsWith('/') ? '' : '/') + path;
}
/*
* If path starts with /{tag}/{value}{/rest}, returns value and the remaining path (/rest).
* Otherwise, returns value of undefined and the path unchanged.
* E.g. parseFirstUrlPart('o', '/o/foo/bar') returns {value: 'foo', path: '/bar'}.
*/
export function parseFirstUrlPart(tag: string, path: string): {value?: string, path: string} {
const match = path.match(/^\/([^/?#]+)\/([^/?#]+)(.*)$/);
if (match && match[1] === tag) {
return {value: match[2], path: sanitizePathTail(match[3])};
} else {
return {path};
}
}
/**
* The internal structure of a UrlId. There is no internal structure. unless the id is
* for a fork, in which case the fork has a separate id, and a user id may also be
* embedded to track ownership.
*/
export interface UrlIdParts {
trunkId: string;
forkId?: string;
forkUserId?: number;
snapshotId?: string;
}
// Parse a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId
// or trunkId[....]~v=snapshotId
export function parseUrlId(urlId: string): UrlIdParts {
let snapshotId: string|undefined;
const parts = urlId.split('~');
const bareParts = parts.filter(part => !part.includes('='));
for (const part of parts) {
if (part.startsWith('v=')) {
snapshotId = decodeURIComponent(part.substr(2).replace(/_/g, '%'));
}
}
return {
trunkId: bareParts[0],
forkId: bareParts[1],
forkUserId: (bareParts[2] !== undefined) ? parseInt(bareParts[2], 10) : undefined,
snapshotId,
};
}
// Construct a string of the form trunkId or trunkId~forkId or trunkId~forkId~forkUserId
// or trunkId[....]~v=snapshotId
export function buildUrlId(parts: UrlIdParts): string {
let token = [parts.trunkId, parts.forkId, parts.forkUserId].filter(x => x !== undefined).join('~');
if (parts.snapshotId) {
// This could be an S3 VersionId, about which AWS makes few promises.
// encodeURIComponent leaves untouched the following:
// alphabetic; decimal; any of: - _ . ! ~ * ' ( )
// We further encode _.!~*'() to fit within existing limits on what characters
// may be in a docId (leaving just the hyphen, which is permitted). The limits
// could be loosened, but without much benefit.
const codedSnapshotId = encodeURIComponent(parts.snapshotId)
.replace(/[_.!~*'()]/g, ch => `_${ch.charCodeAt(0).toString(16).toUpperCase()}`)
.replace(/%/g, '_');
token = `${token}~v=${codedSnapshotId}`;
}
return token;
}
/**
* Values that may be encoded in a hash in a document url.
*/
export interface HashLink {
sectionId?: number;
rowId?: number;
colRef?: number;
}
// Check whether a urlId is a prefix of the docId, and adequately long to be
// a candidate for use in prettier urls.
function shouldIncludeSlug(doc: {id: string, urlId: string|null}): boolean {
if (!doc.urlId || doc.urlId.length < MIN_URLID_PREFIX_LENGTH) { return false; }
return doc.id.startsWith(doc.urlId);
}
// Convert the name of a document into a slug. Only alphanumerics are retained,
// and spaces are replaced with hyphens.
// TODO: investigate whether there's a better option with unicode than just
// deleting it, seems unfair to languages using anything other than unaccented
// Latin characters.
function nameToSlug(name: string): string {
return name.trim().replace(/ /g, '-').replace(/[^-a-zA-Z0-9]/g, '').replace(/---*/g, '-');
}
// Returns a slug for the given docId/urlId/name, or undefined if a slug should
// not be used.
export function getSlugIfNeeded(doc: {id: string, urlId: string|null, name: string}): string|undefined {
if (!shouldIncludeSlug(doc)) { return; }
return nameToSlug(doc.name);
}

@ -0,0 +1,782 @@
import {delay} from 'app/common/delay';
import {Listener, Observable} from 'grainjs';
import {Observable as KoObservable} from 'knockout';
import constant = require('lodash/constant');
import identity = require('lodash/identity');
import times = require('lodash/times');
export const UP_TRIANGLE = '\u25B2';
export const DOWN_TRIANGLE = '\u25BC';
const EMAIL_RE = new RegExp("^\\w[\\w%+/='-]*(\\.[\\w%+/='-]+)*@([A-Za-z0-9]([A-Za-z0-9-]*[A-Za-z" +
"0-9])?\\.)+[A-Za-z]{2,6}$", "u");
// Returns whether str starts with prefix. (Note that this implementation avoids creating a new
// string, and only checks a single location.)
export function startsWith(str: string, prefix: string): boolean {
return str.lastIndexOf(prefix, 0) === 0;
}
// Returns whether str ends with suffix.
export function endsWith(str: string, suffix: string): boolean {
return str.indexOf(suffix, str.length - suffix.length) !== -1;
}
// If str starts with prefix, removes it and returns what remains. Otherwise, returns null.
export function removePrefix(str: string, prefix: string): string|null {
return startsWith(str, prefix) ? str.slice(prefix.length) : null;
}
// If str ends with suffix, removes it and returns what remains. Otherwise, returns null.
export function removeSuffix(str: string, suffix: string): string|null {
return endsWith(str, suffix) ? str.slice(0, str.length - suffix.length) : null;
}
export function removeTrailingSlash(str: string): string {
const result = removeSuffix(str, '/');
return result === null ? str : result;
}
// Expose <string>.padStart. The version of node we use has it, but they typings
// need the es2017 typescript target. TODO: replace once typings in place.
export function padStart(str: string, targetLength: number, padString: string) {
return (str as any).padStart(targetLength, padString);
}
// Capitalizes every word in a string.
export function capitalize(str: string): string {
return str.replace(/\b[a-z]/gi, c => c.toUpperCase());
}
// Returns whether the string n represents a valid number.
// http://stackoverflow.com/questions/18082/validate-numbers-in-javascript-isnumeric
export function isNumber(n: string): boolean {
// This wasn't right for a long time: isFinite() is key to failing on strings like "5a".
return !isNaN(parseFloat(n)) && isFinite(n as any);
}
/**
* Returns a value clamped to the given min-max range.
* @param {Number} value - some numeric value.
* @param {Number} min - minimum value allowed.
* @param {Number} max - maximum value allowed. Must have min <= max.
* @returns {Number} - value restricted to the given range.
*/
export function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}
/**
* Checks if ele is contained within the given bounds.
* @param {Number} value
* @param {Number} bound1 - does not have to be less than/eqal to bound2
* @param {Number} bound2
* @returns {Boolean} - True/False
*/
export function between(value: number, bound1: number, bound2: number): boolean {
const lower = Math.min(bound1, bound2);
const upper = Math.max(bound1, bound2);
return lower <= value && value <= upper;
}
/**
* Returns the positive modulo of x by n. (Javascript default allows negatives)
*/
export function mod(x: number, n: number): number {
return ((x % n) + n) % n;
}
/**
* Returns a number that is n rounded down to the next nearest number divisible by m
*/
export function roundDownToMultiple(n: number, m: number): number {
return Math.floor(n / m) * m;
}
/**
* Returns the first argument unless it's undefined, in which case returns the second one.
*/
export function undefDefault<T>(x: T|undefined, y: T): T {
return (x !== void 0) ? x : y;
}
/**
* Parses json and returns the result, or returns defaultVal if parsing fails.
*/
export function safeJsonParse(json: string, defaultVal: any): any {
try {
return JSON.parse(json);
} catch (e) {
return defaultVal;
}
}
/**
* Just like encodeURIComponent, but does not encode slashes. Slashes don't hurt to be included in
* URL parameters, and look much friendlier not encoded.
*/
export function encodeQueryParam(str: string|number|undefined): string {
return encodeURIComponent(String(str === undefined ? null : str)).replace(/%2F/g, '/');
}
/**
* Encode an object into a querystring ("key=value&key2=value2").
* This is similar to JQuery's $.param, but only works on shallow objects.
*/
export function encodeQueryParams(obj: {[key: string]: string|number|undefined}): string {
return Object.keys(obj).map((k: string) => encodeQueryParam(k) + '=' + encodeQueryParam(obj[k])).join('&');
}
/**
* Return a list of the words in the string, using the given separator string. At most
* maxNumSplits splits are done, so the result will have at most maxNumSplits + 1 elements (this
* is the main difference from how JS built-in string.split() works, and similar to Python split).
* @param {String} str: String to split.
* @param {String} sep: Separator to split on.
* @param {Number} maxNumSplits: Maximum number of splits to do.
* @return {Array[String]} Array of words, of length at most maxNumSplits + 1.
*/
export function maxsplit(str: string, sep: string, maxNumSplits: number): string[] {
const result: string[] = [];
let start = 0, pos;
for (let i = 0; i < maxNumSplits; i++) {
pos = str.indexOf(sep, start);
if (pos === -1) {
break;
}
result.push(str.slice(start, pos));
start = pos + sep.length;
}
result.push(str.slice(start));
return result;
}
// Compare arrays of scalars for equality.
export function arraysEqual(a: any[], b: any[]): boolean {
if (a === b) {
return true;
}
if (!a || !b) {
return false;
}
if (a.length !== b.length) {
return false;
}
for (let i = 0; i < a.length; i++) {
if (a[i] !== b[i]) { return false; }
}
return true;
}
// Gives a set representing the set difference a - b.
export function setDifference<T>(a: Set<T>, b: Set<T>): Set<T> {
const c = new Set<T>();
for (const ai of a) {
if (!b.has(ai)) { c.add(ai); }
}
return c;
}
// Like array.indexOf, but works with array-like objects like HTMLCollection.
export function indexOf<T>(arrayLike: ArrayLike<T>, item: T): number {
return Array.prototype.indexOf.call(arrayLike, item);
}
/**
* Removes a value from the given array. Only the first instance is removed.
* Returns true on success, false if the value was not found.
*/
export function arrayRemove<T>(array: T[], value: T): boolean {
const index = array.indexOf(value);
if (index === -1) {
return false;
}
array.splice(index, 1);
return true;
}
/**
* Inserts value into the array before nextValue, or at the end if nextValue is not found.
*/
export function arrayInsertBefore<T>(array: T[], value: T, nextValue: T): void {
const index = array.indexOf(nextValue);
if (index === -1) {
array.push(value);
} else {
array.splice(index, 0, value);
}
}
/**
* Extends the first array with the second. Like native push, but adds all values in anotherArray.
*/
export function arrayExtend<T>(array: T[], anotherArray: T[]): void {
for (let i = 0, len = anotherArray.length; i < len; i++) {
array.push(anotherArray[i]);
}
}
/**
* Copies count items from fromArray to toArray, copying in a forward direction (which matters
* when the arrays are the same and source and destination indices overlap).
*
* See test/common/arraySplice.js for alternative implementations with timings, from which this
* one is chosen as consistently among the faster ones.
*/
export function arrayCopyForward<T>(toArray: T[], toStart: number,
fromArray: ArrayLike<T>, fromStart: number, count: number): void {
const end = toStart + count;
for (const xend = end - 7; toStart < xend; fromStart += 8, toStart += 8) {
toArray[toStart] = fromArray[fromStart];
toArray[toStart + 1] = fromArray[fromStart + 1];
toArray[toStart + 2] = fromArray[fromStart + 2];
toArray[toStart + 3] = fromArray[fromStart + 3];
toArray[toStart + 4] = fromArray[fromStart + 4];
toArray[toStart + 5] = fromArray[fromStart + 5];
toArray[toStart + 6] = fromArray[fromStart + 6];
toArray[toStart + 7] = fromArray[fromStart + 7];
}
for (; toStart < end; ++fromStart, ++toStart) {
toArray[toStart] = fromArray[fromStart];
}
}
/**
* Copies count items from fromArray to toArray, copying in a backward direction (which matters
* when the arrays are the same and source and destination indices overlap).
*
* See test/common/arraySplice.js for alternative implementations with timings, from which this
* one is chosen as consistently among the faster ones.
*/
export function arrayCopyBackward<T>(toArray: T[], toStart: number,
fromArray: ArrayLike<T>, fromStart: number, count: number): void {
let i = toStart + count - 1, j = fromStart + count - 1;
for (const xStart = toStart + 7; i >= xStart; i -= 8, j -= 8) {
toArray[i] = fromArray[j];
toArray[i - 1] = fromArray[j - 1];
toArray[i - 2] = fromArray[j - 2];
toArray[i - 3] = fromArray[j - 3];
toArray[i - 4] = fromArray[j - 4];
toArray[i - 5] = fromArray[j - 5];
toArray[i - 6] = fromArray[j - 6];
toArray[i - 7] = fromArray[j - 7];
}
for ( ; i >= toStart; --i, --j) {
toArray[i] = fromArray[j];
}
}
/**
* Appends a slice of fromArray to the end of toArray.
*
* See test/common/arraySplice.js for alternative implementations with timings, from which this
* one is chosen as consistently among the faster ones.
*/
export function arrayAppend<T>(toArray: T[], fromArray: ArrayLike<T>, fromStart: number, count: number): void {
if (count === 1) {
toArray.push(fromArray[fromStart]);
} else {
const len = toArray.length;
toArray.length = len + count;
arrayCopyForward(toArray, len, fromArray, fromStart, count);
}
}
/**
* Splices array arrToInsert into target starting at the given start index.
* This implementation tries to be smart by avoiding allocations, appending to the array
* contiguously, then filling in the gap.
*
* See test/common/arraySplice.js for alternative implementations with timings, from which this
* one is chosen as consistently among the faster ones.
*/
export function arraySplice<T>(target: T[], start: number, arrToInsert: ArrayLike<T>): T[] {
const origLen = target.length;
const tailLen = origLen - start;
const insLen = arrToInsert.length;
target.length = origLen + insLen;
if (insLen > tailLen) {
arrayCopyForward(target, origLen, arrToInsert, tailLen, insLen - tailLen);
arrayCopyForward(target, start + insLen, target, start, tailLen);
arrayCopyForward(target, start, arrToInsert, 0, tailLen);
} else {
arrayCopyForward(target, origLen, target, origLen - insLen, insLen);
arrayCopyBackward(target, start + insLen, target, start, tailLen - insLen);
arrayCopyForward(target, start, arrToInsert, 0, insLen);
}
return target;
}
/**
* Returns a new array of length count, filled with the given value.
*/
export function arrayRepeat<T>(count: number, value: T): T[] {
return times(count, constant(value));
}
// Type for a compare func that returns a positive, negative, or zero value, as used for sorting.
export type CompareFunc<T> = (a: T, b: T) => number;
/**
* Returns the index at which the given element can be inserted to keep the array sorted.
* This is equivalent to underscore's sortedIndex and python's bisect_left.
* @param {Array} array - sorted array of elements based on the given compareFunc
* @param {object} elem - object to be inserted in the given array
* @param {function} compareFunc - compares 2 elements. Returns a pos value if the 1st element is
* larger, 0 if they're equal, a neg value if the 2nd is larger.
*/
export function sortedIndex<T>(array: ArrayLike<T>, elem: T, compareFunc: CompareFunc<T>): number {
let lo = 0, mid;
let hi = array.length;
if (array.length === 0) { return 0; }
while (lo < hi) {
mid = Math.floor((lo + hi) / 2);
if (compareFunc(array[mid], elem) < 0) { // mid < elem
lo = mid + 1;
} else {
hi = mid;
}
}
return lo;
}
/**
* Returns true if an array contains duplicate values.
* Values are considered equal if their toString() representations are equal.
*/
export function hasDuplicates(array: any[]): boolean {
const prevVals = Object.create(null);
for (const value of array) {
if (value in prevVals) {
return true;
}
prevVals[value] = true;
}
return false;
}
/**
* Counts the number of items in array which satisfy the callback.
*/
export function countIf<T>(array: ReadonlyArray<T>, callback: (item: T) => boolean): number {
let count = 0;
array.forEach(item => {
if (callback(item)) { count++; }
});
return count;
}
/**
* For two parallel arrays, calls mapFunc(a[i], b[i]) for each pair of corresponding elements, and
* returns an array of the results.
*/
export function map2<T, U, V>(array1: ArrayLike<T>, array2: ArrayLike<U>, mapFunc: (a: T, b: U) => V): V[] {
const len = array1.length;
const result: V[] = new Array(len);
for (let i = 0; i < len; i++) {
result[i] = mapFunc(array1[i], array2[i]);
}
return result;
}
/**
* Takes a 2d array returns a new matrix with r rows and c columns
* @param [Array] dataMatrix: a 2d array
* @param [Number] r: final row length
* @param [Number] c: final column length
*/
export function growMatrix<T>(dataMatrix: T[][], r: number, c: number): T[][] {
const colArr = dataMatrix.map(colVals =>
Array.from({length: c}, (_v, k) => colVals[k % colVals.length])
);
return Array.from({length: r}, (_v, k) => colArr[k % colArr.length]);
}
/**
* Returns a function that compares two elements based on multiple sort keys and the
* given compare functions.
* Elements are compared using the sort key functions with index 0 having the greatest priority.
* Subsequent sort key functions are used as tie breakers.
* @param {function Array} sortKeyFuncs - a list of sort key functions.
* @param {function Array} compareKeyFuncs - a list of comparison functions parallel to sortKeyFuncs
* Each compare function must satisfy the comparison invariant:
* If compare(a, b) > 0 then a > b,
* If compare(a, b) < 0 then a < b,
* If compare(a, b) == 0 then a == b,
* @param {Array of 1/-1's} optAscending - Comparison on sortKeyFuncs[i] is inverted if optAscending[i] == -1
*/
export function multiCompareFunc<T, U>(sortKeyFuncs: ReadonlyArray<(a: T) => U>,
compareFuncs: ArrayLike<CompareFunc<U>>,
optAscending?: number[]): CompareFunc<T> {
if (sortKeyFuncs.length !== compareFuncs.length) {
throw new Error('Number of sort key funcs must be the same as the number of compare funcs');
}
const ascending = optAscending || sortKeyFuncs.map(() => 1);
return function(a: T, b: T): number {
let compareOutcome, keyA, keyB;
for (let i = 0; i < compareFuncs.length; i++) {
keyA = sortKeyFuncs[i](a);
keyB = sortKeyFuncs[i](b);
compareOutcome = compareFuncs[i](keyA, keyB);
if (compareOutcome !== 0) { return ascending[i] * compareOutcome; }
}
return 0;
};
}
export function nativeCompare<T>(a: T, b: T): number {
return (a < b ? -1 : (a > b ? 1 : 0));
}
/**
* A copy of python`s `setdefault` function.
* Sets key in mapInst to value, if key is not already set.
* @param {Map} mapInst: Instance of Map.
* @param {Object} key: Key into the map.
* @param {Object} value: Value to insert, possibly.
*/
export function setDefault<K, V>(mapInst: Map<K, V>, key: K, val: V): V {
if (!mapInst.has(key)) { mapInst.set(key, val); }
return mapInst.get(key)!;
}
/**
* Similar to Python's `setdefault`: returns the key `key` from `mapInst`, or if it's not there, sets
* it to the result buildValue().
*/
export function getSetMapValue<K, V>(mapInst: Map<K, V>, key: K, buildValue: () => V): V {
if (!mapInst.has(key)) { mapInst.set(key, buildValue()); }
return mapInst.get(key)!;
}
/**
* If key is in mapInst, remove it and return its value, else return `undefined`.
* @param {Map} mapInst: Instance of Map.
* @param {Object} key: Key into the map to remove.
*/
export function popFromMap<K, V>(mapInst: Map<K, V>, key: K): V|undefined {
const value = mapInst.get(key);
mapInst.delete(key);
return value;
}
/**
* For each encountered value in `values`, increment the corresponding counter in `valueCounts`.
*/
export function addCountsToMap<T>(valueCounts: Map<T, number>, values: Iterable<T>,
mapFunc: (v: any) => any = identity) {
for (const v of values) {
const mappedValue = mapFunc(v);
valueCounts.set(mappedValue, (valueCounts.get(mappedValue) || 0) + 1);
}
}
/**
* Returns whether one Set is a subset of another.
*/
export function isSubset(smaller: Set<any>, larger: Set<any>): boolean {
for (const value of smaller) {
if (!larger.has(value)) {
return false;
}
}
return true;
}
/**
* Merges the contents of two or more objects together into the first object, recursing into
* nested objects and arrays (like jquery.extend(true, ...)).
* @param {Object} target - The object to modify. Use {} to create a new merged object.
* @param {Object} ... - Additional objects from which to copy properties into target.
* @returns {Object} The first argument, target, modified.
*/
export function deepExtend(target: any, _varArgObjects: any): any {
for (let i = 1; i < arguments.length; i++) {
const object = arguments[i];
// Extend the base object
for (const name in object) {
if (!object.hasOwnProperty(name)) { continue; }
let src = object[name];
if (src === target || src === undefined) {
// Prevent one kind of infinite loop, as JQuery's extend does, and skip undefined values.
continue;
}
if (src) {
// Recurse if we're merging plain objects or arrays
const tgt = target[name];
if (Array.isArray(src)) {
src = deepExtend(tgt && Array.isArray(tgt) ? tgt : [], src);
} else if (typeof src === 'object') {
src = deepExtend(tgt && typeof tgt === 'object' ? tgt : {}, src);
}
}
target[name] = src;
}
}
// Return the modified object
return target;
}
/**
* Returns a human-readable string containing a number of bytes, KB, or MB.
* @param {Number} bytes. Number of bytes.
* @returns {String} A description such as "4.1KB".
*/
export function byteString(bytes: number): string {
if (bytes < 1024) {
return bytes + 'B';
} else if (bytes < 1024 * 1024) {
return (bytes / 1024).toFixed(1) + 'KB';
} else {
return (bytes / 1024 / 1024).toFixed(1) + 'MB';
}
}
/**
* Creates a new object mapping each key in keysArray to the value returned by callback.
* @param {Array} keysArray - Array of strings to use as the properties of the returned object.
* @param {Function} callback - Function that produces the value for each key. Called in the same
* way as array.map() calls its callbacks.
* @param {Object} optThisArg - Value to use as `this` when executing callback.
* @returns {Object} - object mapping keys from `keysArray` to values returned by `callback`.
*/
export function mapToObject<T>(keysArray: string[], callback: (key: string) => T,
optThisArg: any): {[key: string]: T} {
const values: T[] = keysArray.map(callback, optThisArg);
const map: {[key: string]: T} = {};
for (let i = 0; i < keysArray.length; i++) {
map[keysArray[i]] = values[i];
}
return map;
}
/**
* A List of python identifiers; the result of running keywords.kwlist in Python 2.7.6,
* plus additional illegal identifiers None, False, True
* Using [] instead of new Array causes a "comprehension error" for some reason
*/
const _kwlist = ['and', 'as', 'assert', 'break', 'class', 'continue', 'def',
'del', 'elif', 'else', 'except', 'exec', 'finally', 'for', 'from', 'global',
'if', 'import', 'in', 'is', 'lambda', 'not', 'or', 'pass', 'print', 'raise',
'return', 'try', 'while', 'with', 'yield', 'None', 'False', 'True'];
/**
* Given an arbitrary string, makes substitutions to make it a valid SQL/Python identifier.
* Corresponds to sandbox/grist/gencode.sanitize_ident
*/
export function sanitizeIdent(ident: string, prefix?: string) {
prefix = prefix || 'c';
// Remove non-alphanumeric non-_ chars
ident = ident.replace(/[^a-zA-Z0-9_]+/g, '_');
// Remove leading and trailing _
ident = ident.replace(/^_+|_+$/g, '');
// Place prefix at front if the beginning isn't a number
ident = ident.replace(/^(?=[0-9])/g, prefix);
// Append prefix until it is not python keyword
while (_kwlist.includes(ident)) {
ident = prefix + ident;
}
return ident;
}
/**
* Clone a function, returning a function object that represents a brand new function with the
* same code. If the same function is used with different argument types, it would prevent JS V8
* engine optimizations (or cause it to deoptimize it). If different clones are called with
* different argument types, they can be optimized independently.
*
* As with all micro-optimizations, only do this when the optimization matters.
*/
export function cloneFunc(fn: Function): Function { // tslint:disable-line:ban-types
/* jshint evil:true */ // suppress eval warning.
return eval('(' + fn.toString() + ')'); // tslint:disable-line:no-eval
}
/**
* Generates a random id using a sequence of uppercase alphanumeric characters
* preceeded by an optional prefix.
*/
export function genRandomId(len: number, optPrefix?: string): string {
const chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ';
let ret = optPrefix || '';
for (let i = 0; i < len; i++) {
ret += chars[Math.floor(Math.random() * chars.length)];
}
return ret;
}
/**
* Scans through two sorted arrays, calling a function on each item or pair of items
* for every present key in order.
* @param {Array} arrA - First array to scan. NOTE: Should be sorted by the key value.
* @param {Array} arrB - Second array to scan. NOTE: Should be sorted by the key value.
* @param {Function} callback - Called with an item from arrA as the first argument and an
* item from arrB as the second. Called for every unique key in order, either with one of the
* arguments null if the key is present only in one array, or both non-null if the key is
* present in both arrays. NOTE: Key values should not be null.
* @param {Function} optKeyFunc - Optional function to map each array value to a sort key.
* Defaults to the identity function.
*/
export function sortedScan<T, U>(arrA: ArrayLike<T>, arrB: ArrayLike<U>,
callback: (a: T|null, B: U|null) => void,
optKeyFunc?: (item: T|U) => any) {
const keyFunc = optKeyFunc || identity;
let i = 0, j = 0;
while (i < arrA.length || j < arrB.length) {
const a = arrA[i], b = arrB[j];
const keyA = i < arrA.length ? keyFunc(a) : null;
const keyB = j < arrB.length ? keyFunc(b) : null;
if (keyA !== null && (keyB === null || keyA < keyB)) {
callback(a, null);
i++;
} else if (keyA === null || keyA > keyB) {
callback(null, b);
j++;
} else {
callback(a, b);
i++;
j++;
}
}
}
/**
* Returns the time in ms to wait until attempting another connection.
* @param {Number} attemptNumber - Reconnect attempt number starting at 0.
* @param {Array} intervals - Array of reconnect intervals in ms.
* @returns {Number}
*/
export function getReconnectTimeout(attemptNumber: number, intervals: ArrayLike<number>): number {
if (attemptNumber >= intervals.length) {
// Add an additional wait time if already at max attempts.
const timeout = intervals[intervals.length - 1];
return timeout + Math.random() * timeout;
} else {
return intervals[attemptNumber];
}
}
/**
* Returns whether the given email is a valid formatted email string.
* @param {String} email - Email to test.
* @returns {Boolean}
*/
export function isEmail(email: string): boolean {
return EMAIL_RE.test(email.toLowerCase());
}
/*
* Takes an observable and returns a promise for when the observable's value matches the given
* predicate. It then unsubscribes from the observable, and returns its value.
* If a predicate is not given, resolves to the observable values as soon as it's truthy.
*/
export function waitObs<T>(observable: KoObservable<T>, predicate: (value: T) => boolean = Boolean): Promise<T> {
return new Promise((resolve, _reject) => {
const value = observable.peek();
if (predicate(value)) { return resolve(value); }
const sub = observable.subscribe((val: T) => {
if (predicate(val)) {
sub.dispose();
resolve(val);
}
});
});
}
/**
* Same as waitObs but for grainjs observables.
*/
export async function waitGrainObs<T>(observable: Observable<T>): Promise<NonNullable<T>>;
export async function waitGrainObs<T>(observable: Observable<T>, predicate?: (value: T) => boolean): Promise<T>;
export async function waitGrainObs<T>(observable: Observable<T>,
predicate: (value: T) => boolean = Boolean): Promise<T> {
let sub: Listener|undefined;
const res: T = await new Promise((resolve, _reject) => {
const value = observable.get();
if (predicate(value)) { return resolve(value); }
sub = observable.addListener((val: T) => {
if (predicate(val)) {
resolve(val);
}
});
});
if (sub) { sub.dispose(); }
return res;
}
/**
* Class to maintain a chain of promise-returning callbacks. All scheduled callbacks will be
* called in order as long as the previous one is successful. If a callback fails is rejected,
* already-scheduled callbacks will be skipped, but newly-scheduled ones will be run.
*/
export class PromiseChain<T> {
private _last: Promise<T|void> = Promise.resolve();
// Adds a callback to the chain. If the callback runs, the return value is the return value of
// the callback. If it's skipped due to a failure earlier in the chain, the return value is the
// rejection with the message "Skipped due to an earlier error".
public add(nextCB: () => Promise<T>): Promise<T> {
const next = this._last.catch(() => { throw new Error("Skipped due to an earlier error"); }).then(nextCB);
// If any callback fails, all queued ones will be skipped. Here we reset the chain, so that
// callbacks added later do get run.
next.catch(() => { this._last = Promise.resolve(); });
this._last = next;
return next;
}
}
/**
* Indicates if a hex color value, e.g. '#000000', is darker than the given value.
* Darkness is measured from 0..255, where 0 is the darkest and 255 is the lightest.
*
* Taken from: https://stackoverflow.com/questions/12043187/how-to-check-if-hex-color-is-too-black
*/
export function isColorDark(hexColor: string, isDarkBelow: number = 220): boolean {
const c = hexColor.substring(1); // strip #
const rgb = parseInt(c, 16); // convert rrggbb to decimal
// Extract RGB components
const r = (rgb >> 16) & 0xff; // tslint:disable-line:no-bitwise
const g = (rgb >> 8) & 0xff; // tslint:disable-line:no-bitwise
const b = (rgb >> 0) & 0xff; // tslint:disable-line:no-bitwise
const luma = 0.2126 * r + 0.7152 * g + 0.0722 * b; // per ITU-R BT.709
return luma < isDarkBelow;
}
/**
* Returns a promise that resolves to true if promise takes longer than timeoutMsec to resolve. If not
* or if promise throws returns false.
*/
export async function isLongerThan(promise: Promise<any>, timeoutMsec: number): Promise<boolean> {
let isPending = true;
const done = () => {isPending = false; };
await Promise.race([
promise.then(done, done),
delay(timeoutMsec)
]);
return isPending;
}

@ -0,0 +1,502 @@
/**
* Module for serializing data in the format of Python 'marshal' module. It's used for
* communicating with the Python-based formula engine running in a Pypy sandbox. It supports
* version 0 of python marshalling format, which is what the Pypy sandbox supports.
*
* Usage:
* Marshalling:
* const marshaller = new Marshaller({version: 2});
* marshaller.marshal(value);
* marshaller.marshal(value);
* const buf = marshaller.dump(); // Leaves the marshaller empty.
*
* Unmarshalling:
* const unmarshaller = new Unmarshaller();
* unmarshaller.on('value', function(value) { ... });
* unmarshaller.push(buffer);
* unmarshaller.push(buffer);
*
* In Python, and in the marshalled format, there is a distinction between strings and unicode
* objects. In JS, there is a good correspondence to Uint8Array objects and strings, respectively.
* Python unicode objects always become JS strings. JS Uint8Arrays always become Python strings.
*
* JS strings become Python unicode objects, but can be marshalled to Python strings with
* 'stringToBuffer' option. Similarly, Python strings become JS Uint8Arrays, but can be
* unmarshalled to JS strings if 'bufferToString' option is set.
*/
import {BigInt} from 'app/common/BigInt';
import * as MemBuffer from 'app/common/MemBuffer';
import {EventEmitter} from 'events';
import * as util from 'util';
export interface MarshalOptions {
stringToBuffer?: boolean;
version?: number;
}
export interface UnmarshalOptions {
bufferToString?: boolean;
}
function ord(str: string): number {
return str.charCodeAt(0);
}
/**
* Type codes used for python marshalling of values.
* See pypy: rpython/translator/sandbox/_marshal.py.
*/
const marshalCodes = {
NULL : ord('0'),
NONE : ord('N'),
FALSE : ord('F'),
TRUE : ord('T'),
STOPITER : ord('S'),
ELLIPSIS : ord('.'),
INT : ord('i'),
INT64 : ord('I'),
/*
BFLOAT, for 'binary float', is an encoding of float that just encodes the bytes of the
double in standard IEEE 754 float64 format. It is used by Version 2+ of Python's marshal
module. Previously (in versions 0 and 1), the FLOAT encoding is used, which stores floats
through their string representations.
Version 0 (FLOAT) is mandatory for system calls within the sandbox, while Version 2 (BFLOAT)
is recommended for Grist's communication because it is more efficient and faster to
encode/decode
*/
BFLOAT : ord('g'),
FLOAT : ord('f'),
COMPLEX : ord('x'),
LONG : ord('l'),
STRING : ord('s'),
INTERNED : ord('t'),
STRINGREF: ord('R'),
TUPLE : ord('('),
LIST : ord('['),
DICT : ord('{'),
CODE : ord('c'),
UNICODE : ord('u'),
UNKNOWN : ord('?'),
SET : ord('<'),
FROZENSET: ord('>'),
};
type MarshalCode = keyof typeof marshalCodes;
// A little hack to test if the value is a 32-bit integer. Actually, for Python, int might be up
// to 64 bits (if that's the native size), but this is simpler.
// See http://stackoverflow.com/questions/3885817/how-to-check-if-a-number-is-float-or-integer.
function isInteger(n: number): boolean {
// Float have +0.0 and -0.0. To represent -0.0 precisely, we have to use a float, not an int
// (see also https://stackoverflow.com/questions/7223359/are-0-and-0-the-same).
// tslint:disable-next-line:no-bitwise
return n === +n && n === (n | 0) && !Object.is(n, -0.0);
}
// ----------------------------------------------------------------------
/**
* To force a value to be serialized using a particular representation (e.g. a number as INT64),
* wrap it into marshal.wrap('INT64', value) and serialize that.
*/
export function wrap(codeStr: MarshalCode, value: unknown) {
return new WrappedObj(marshalCodes[codeStr], value);
}
export class WrappedObj {
constructor(public code: number, public value: unknown) {}
public inspect() {
return util.inspect(this.value);
}
}
// ----------------------------------------------------------------------
/**
* @param {Boolean} options.stringToBuffer - If set, JS strings will become Python strings rather
* than unicode objects (as if each JS string is wrapped into MemBuffer.stringToArray(str)).
* This flag becomes a same-named property of Marshaller, which can be set at any time.
* @param {Number} options.version - If version >= 2, uses binary representation for floats. The
* default version 0 formats floats as strings.
*
* TODO: The default should be version 2. (0 was used historically because it was needed for
* communication with PyPy-based sandbox.)
*/
export class Marshaller {
private memBuf: MemBuffer;
private readonly floatCode: number;
private readonly stringCode: number;
constructor(options?: MarshalOptions) {
this.memBuf = new MemBuffer(undefined);
this.floatCode = options && options.version && options.version >= 2 ? marshalCodes.BFLOAT : marshalCodes.FLOAT;
this.stringCode = options && options.stringToBuffer ? marshalCodes.STRING : marshalCodes.UNICODE;
}
public dump(): Uint8Array {
// asByteArray returns a view on the underlying data, and the constructor creates a new copy.
// For some usages, we may want to avoid making the copy.
const bytes = new Uint8Array(this.memBuf.asByteArray());
this.memBuf.clear();
return bytes;
}
public dumpAsBuffer(): Buffer {
const bytes = Buffer.from(this.memBuf.asByteArray());
this.memBuf.clear();
return bytes;
}
public getCode(value: any) {
switch (typeof value) {
case 'number': return isInteger(value) ? marshalCodes.INT : this.floatCode;
case 'string': return this.stringCode;
case 'boolean': return value ? marshalCodes.TRUE : marshalCodes.FALSE;
case 'undefined': return marshalCodes.NONE;
case 'object': {
if (value instanceof WrappedObj) {
return value.code;
} else if (value === null) {
return marshalCodes.NONE;
} else if (value instanceof Uint8Array) {
return marshalCodes.STRING;
} else if (Buffer.isBuffer(value)) {
return marshalCodes.STRING;
} else if (Array.isArray(value)) {
return marshalCodes.LIST;
}
return marshalCodes.DICT;
}
default: {
throw new Error("Marshaller: Unsupported value of type " + (typeof value));
}
}
}
public marshal(value: any): void {
const code = this.getCode(value);
if (value instanceof WrappedObj) {
value = value.value;
}
this.memBuf.writeUint8(code);
switch (code) {
case marshalCodes.NULL: return;
case marshalCodes.NONE: return;
case marshalCodes.FALSE: return;
case marshalCodes.TRUE: return;
case marshalCodes.INT: return this.memBuf.writeInt32LE(value);
case marshalCodes.INT64: return this._writeInt64(value);
case marshalCodes.FLOAT: return this._writeStringFloat(value);
case marshalCodes.BFLOAT: return this.memBuf.writeFloat64LE(value);
case marshalCodes.STRING:
return (value instanceof Uint8Array || Buffer.isBuffer(value) ?
this._writeByteArray(value) :
this._writeUtf8String(value));
case marshalCodes.TUPLE: return this._writeList(value);
case marshalCodes.LIST: return this._writeList(value);
case marshalCodes.DICT: return this._writeDict(value);
case marshalCodes.UNICODE: return this._writeUtf8String(value);
// None of the following are supported.
case marshalCodes.STOPITER:
case marshalCodes.ELLIPSIS:
case marshalCodes.COMPLEX:
case marshalCodes.LONG:
case marshalCodes.INTERNED:
case marshalCodes.STRINGREF:
case marshalCodes.CODE:
case marshalCodes.UNKNOWN:
case marshalCodes.SET:
case marshalCodes.FROZENSET: throw new Error("Marshaller: Can't serialize code " + code);
default: throw new Error("Marshaller: Can't serialize code " + code);
}
}
private _writeInt64(value: number) {
if (!isInteger(value)) {
// TODO We could actually support 53 bits or so.
throw new Error("Marshaller: int64 still only supports 32-bit ints for now: " + value);
}
this.memBuf.writeInt32LE(value);
this.memBuf.writeInt32LE(value >= 0 ? 0 : -1);
}
private _writeStringFloat(value: number) {
// This could be optimized a bit, but it's only used in V0 marshalling, which is only used in
// sandbox system calls, which don't really ever use floats anyway.
const bytes = MemBuffer.stringToArray(value.toString());
if (bytes.byteLength >= 127) {
throw new Error("Marshaller: Trying to write a float that takes " + bytes.byteLength + " bytes");
}
this.memBuf.writeUint8(bytes.byteLength);
this.memBuf.writeByteArray(bytes);
}
private _writeByteArray(value: Uint8Array|Buffer) {
// This works for both Uint8Arrays and Node Buffers.
this.memBuf.writeInt32LE(value.length);
this.memBuf.writeByteArray(value);
}
private _writeUtf8String(value: string) {
const offset = this.memBuf.size();
// We don't know the length until we write the value.
this.memBuf.writeInt32LE(0);
this.memBuf.writeString(value);
const byteLength = this.memBuf.size() - offset - 4;
// Overwrite the 0 length we wrote earlier with the correct byte length.
this.memBuf.asDataView.setInt32(this.memBuf.startPos + offset, byteLength, true);
}
private _writeList(array: unknown[]) {
this.memBuf.writeInt32LE(array.length);
for (const item of array) {
this.marshal(item);
}
}
private _writeDict(obj: {[key: string]: any}) {
const keys = Object.keys(obj);
keys.sort();
for (const key of keys) {
this.marshal(key);
this.marshal(obj[key]);
}
this.memBuf.writeUint8(marshalCodes.NULL);
}
}
// ----------------------------------------------------------------------
const TwoTo32 = 0x100000000; // 2**32
const TwoTo15 = 0x8000; // 2**15
/**
* @param {Boolean} options.bufferToString - If set, Python strings will become JS strings rather
* than Buffers (as if each decoded buffer is wrapped into `buf.toString()`).
* This flag becomes a same-named property of Unmarshaller, which can be set at any time.
* Note that options.version isn't needed, since this will decode both formats.
* TODO: Integers (such as int64 and longs) that are too large for JS are currently represented as
* decimal strings. They may need a better representation, or a configurable option.
*/
export class Unmarshaller extends EventEmitter {
public memBuf: MemBuffer;
private consumer: any = null;
private _lastCode: number|null = null;
private readonly bufferToString: boolean;
private emitter: (v: any) => boolean;
private stringTable: Array<string|Uint8Array> = [];
constructor(options?: UnmarshalOptions) {
super();
this.memBuf = new MemBuffer(undefined);
this.bufferToString = Boolean(options && options.bufferToString);
this.emitter = this.emit.bind(this, 'value');
}
/**
* Adds more data for parsing. Parsed values will be emitted as 'value' events.
* @param {Uint8Array|Buffer} byteArray: Uint8Array or Node Buffer with bytes to parse.
*/
public push(byteArray: Uint8Array|Buffer) {
this.parse(byteArray, this.emitter);
}
/**
* Adds data to parse, and calls valueCB(value) for each value parsed. If valueCB returns the
* Boolean false, stops parsing and returns.
*/
public parse(byteArray: Uint8Array|Buffer, valueCB: (val: any) => boolean|void) {
this.memBuf.writeByteArray(byteArray);
try {
while (this.memBuf.size() > 0) {
this.consumer = this.memBuf.makeConsumer();
// Have to reset stringTable for interned strings before each top-level parse call.
this.stringTable.length = 0;
const value = this._parse();
this.memBuf.consume(this.consumer);
if (valueCB(value) === false) {
return;
}
}
} catch (err) {
// If the error is `needMoreData`, we silently return. We'll retry by reparsing the message
// from scratch after the next push(). If buffers contain complete serialized messages, the
// cost should be minor. But this design might get very inefficient if we have big messages
// of arrays or dictionaries.
if (err.needMoreData) {
if (!err.consumedData || err.consumedData > 1024) {
// tslint:disable-next-line:no-console
console.log("Unmarshaller: Need more data; wasted parsing of %d bytes", err.consumedData);
}
} else {
err.message = "Unmarshaller: " + err.message;
throw err;
}
}
}
private _parse(): unknown {
const code = this.memBuf.readUint8(this.consumer);
this._lastCode = code;
switch (code) {
case marshalCodes.NULL: return null;
case marshalCodes.NONE: return null;
case marshalCodes.FALSE: return false;
case marshalCodes.TRUE: return true;
case marshalCodes.INT: return this._parseInt();
case marshalCodes.INT64: return this._parseInt64();
case marshalCodes.FLOAT: return this._parseStringFloat();
case marshalCodes.BFLOAT: return this._parseBinaryFloat();
case marshalCodes.STRING: return this._parseByteString();
case marshalCodes.TUPLE: return this._parseList();
case marshalCodes.LIST: return this._parseList();
case marshalCodes.DICT: return this._parseDict();
case marshalCodes.UNICODE: return this._parseUnicode();
case marshalCodes.INTERNED: return this._parseInterned();
case marshalCodes.STRINGREF: return this._parseStringRef();
case marshalCodes.LONG: return this._parseLong();
// None of the following are supported.
// case marshalCodes.STOPITER:
// case marshalCodes.ELLIPSIS:
// case marshalCodes.COMPLEX:
// case marshalCodes.CODE:
// case marshalCodes.UNKNOWN:
// case marshalCodes.SET:
// case marshalCodes.FROZENSET:
default:
throw new Error(`Unmarshaller: unsupported code "${String.fromCharCode(code)}" (${code})`);
}
}
private _parseInt() {
return this.memBuf.readInt32LE(this.consumer);
}
private _parseInt64() {
const low = this.memBuf.readInt32LE(this.consumer);
const hi = this.memBuf.readInt32LE(this.consumer);
if ((hi === 0 && low >= 0) || (hi === -1 && low < 0)) {
return low;
}
const unsignedLow = low < 0 ? TwoTo32 + low : low;
if (hi >= 0) {
return new BigInt(TwoTo32, [unsignedLow, hi], 1).toNative();
} else {
// This part is tricky. See unittests for check of correctness.
return new BigInt(TwoTo32, [TwoTo32 - unsignedLow, -hi - 1], -1).toNative();
}
}
private _parseLong() {
// The format is a 32-bit size whose sign is the sign of the result, followed by 16-bit digits
// in base 2**15.
const size = this.memBuf.readInt32LE(this.consumer);
const sign = size < 0 ? -1 : 1;
const numDigits = size < 0 ? -size : size;
const digits = [];
for (let i = 0; i < numDigits; i++) {
digits.push(this.memBuf.readInt16LE(this.consumer));
}
return new BigInt(TwoTo15, digits, sign).toNative();
}
private _parseStringFloat() {
const len = this.memBuf.readUint8(this.consumer);
const buf = this.memBuf.readString(this.consumer, len);
return parseFloat(buf);
}
private _parseBinaryFloat() {
return this.memBuf.readFloat64LE(this.consumer);
}
private _parseByteString(): string|Uint8Array {
const len = this.memBuf.readInt32LE(this.consumer);
return (this.bufferToString ?
this.memBuf.readString(this.consumer, len) :
this.memBuf.readByteArray(this.consumer, len));
}
private _parseInterned() {
const s = this._parseByteString();
this.stringTable.push(s);
return s;
}
private _parseStringRef() {
const index = this._parseInt();
return this.stringTable[index];
}
private _parseList() {
const len = this.memBuf.readInt32LE(this.consumer);
const value = [];
for (let i = 0; i < len; i++) {
value[i] = this._parse();
}
return value;
}
private _parseDict() {
const dict: {[key: string]: any} = {};
while (true) { // eslint-disable-line no-constant-condition
let key = this._parse() as string|Uint8Array;
if (key === null && this._lastCode === marshalCodes.NULL) {
break;
}
const value = this._parse();
if (key !== null) {
if (key instanceof Uint8Array) {
key = MemBuffer.arrayToString(key);
}
dict[key as string] = value;
}
}
return dict;
}
private _parseUnicode() {
const len = this.memBuf.readInt32LE(this.consumer);
return this.memBuf.readString(this.consumer, len);
}
}
/**
* Similar to python's marshal.loads(). Parses the given bytes and returns the parsed value. There
* must not be any trailing data beyond the single marshalled value.
*/
export function loads(byteArray: Uint8Array|Buffer, options?: UnmarshalOptions): any {
const unmarshaller = new Unmarshaller(options);
let parsedValue;
unmarshaller.parse(byteArray, function(value) {
parsedValue = value;
return false;
});
if (typeof parsedValue === 'undefined') {
throw new Error("loads: input data truncated");
} else if (unmarshaller.memBuf.size() > 0) {
throw new Error("loads: extra bytes past end of input");
}
return parsedValue;
}
/**
* Serializes arbitrary data by first marshalling then converting to a base64 string.
*/
export function dumpBase64(data: any, options?: MarshalOptions) {
const marshaller = new Marshaller(options || {version: 2});
marshaller.marshal(data);
return marshaller.dumpAsBuffer().toString('base64');
}
/**
* Loads data from a base64 string, as serialized by dumpBase64().
*/
export function loadBase64(data: string, options?: UnmarshalOptions) {
return loads(Buffer.from(data, 'base64'), options);
}

@ -0,0 +1,252 @@
/**
* File for configuring the metric collection bucket duration, data push intervals between client, server,
* and Grist Metrics EC2 instance, as well as individual metrics collected in the client and server.
*/
// Time interval settings (ms)
exports.BUCKET_SIZE = 60 * 1000;
exports.CLIENT_PUSH_INTERVAL = 120 * 1000;
exports.SERVER_PUSH_INTERVAL = 120 * 1000;
exports.MAX_PENDING_BUCKETS = 40;
exports.CONN_RETRY = 20 * 1000;
// Metrics use the general form:
// <category>.<short desc>
// With prefixes, measurement type, and clientId/serverId added automatically on send.
// 'type' is the measurement tool type, with options 'Switch', 'Counter', 'Gauge', 'Timer', and
// 'ExecutionTimer'. (See metricTools.js for details)
// Suffixes are added to the metric names depending on their measurement tool.
// 'Switch' => '.instances'
// 'Gauge' => '.total'
// 'Counter' => '.count'
// 'Timer' => '.time'
// 'ExecutionTimer' => '.execution_time', '.count' (Execution timer automatically records a count)
exports.clientMetrics = [
// General
{
name: 'sidepane.opens',
type: 'Counter',
desc: 'Number of times the side pane is opened'
},
{
name: 'app.client_active_span',
type: 'Timer',
desc: 'Total client time spent using grist'
},
{
name: 'app.connected_to_server_span',
type: 'Timer',
desc: 'Total time spent connected to the server'
},
{
name: 'app.disconnected_from_server_span',
type: 'Timer',
desc: 'Total time spent disconnected from the server'
},
// Docs
{
name: 'docs.num_open_6+_tables',
type: 'SamplingGauge',
desc: 'Number of open docs with more than 5 tables'
},
{
name: 'docs.num_open_0-5_tables',
type: 'SamplingGauge',
desc: 'Number of open docs with 0-5 tables'
},
// Tables
{
name: 'tables.num_tables',
type: 'SamplingGauge',
desc: 'Number of open tables'
},
{
name: 'tables.num_summary_tables',
type: 'SamplingGauge',
desc: 'Number of open sections in the current view'
},
// Views
{
name: 'views.code_view_open_span',
type: 'Timer',
desc: 'Time spent with code viewer open'
},
// Sections
{
name: 'sections.grid_open_span',
type: 'Timer',
desc: 'Time spent with gridview open'
},
{
name: 'sections.detail_open_span',
type: 'Timer',
desc: 'Time spent with gridview open'
},
{
name: 'sections.num_grid_sections',
type: 'SamplingGauge',
desc: 'Number of open sections in the current view'
},
{
name: 'sections.num_detail_sections',
type: 'SamplingGauge',
desc: 'Number of open sections in the current view'
},
{
name: 'sections.num_chart_sections',
type: 'SamplingGauge',
desc: 'Number of open sections in the current view'
},
{
name: 'sections.multiple_open_span',
type: 'Timer',
desc: 'Time spent with multiple sections open'
},
// Performance
{
name: 'performance.server_action',
type: 'ExecutionTimer',
desc: 'Time for a server action to complete'
},
{
name: 'performance.doc_load',
type: 'ExecutionTimer',
desc: 'Time to load a document'
},
// Columns
{
name: 'cols.num_formula_cols',
type: 'SamplingGauge',
desc: 'Number of formula columns in open documents'
},
{
name: 'cols.num_text_cols',
type: 'SamplingGauge',
desc: 'Number of text columns in open documents'
},
{
name: 'cols.num_int_cols',
type: 'SamplingGauge',
desc: 'Number of integer columns in open documents'
},
{
name: 'cols.num_numeric_cols',
type: 'SamplingGauge',
desc: 'Number of numeric columns in open documents'
},
{
name: 'cols.num_date_cols',
type: 'SamplingGauge',
desc: 'Number of date columns in open documents'
},
{
name: 'cols.num_datetime_cols',
type: 'SamplingGauge',
desc: 'Number of datetime columns in open documents'
},
{
name: 'cols.num_ref_cols',
type: 'SamplingGauge',
desc: 'Number of reference columns in open documents'
},
{
name: 'cols.num_attachments_cols',
type: 'SamplingGauge',
desc: 'Number of attachments columns in open documents'
},
{
name: 'performance.front_end_errors',
type: 'Counter',
desc: 'Number of frontend errors'
}
// TODO: Implement the following:
// {
// name: 'grist-rt.performance.view_swap',
// type: 'ExecutionTimer',
// desc: 'Time to swap views'
// }
];
exports.serverMetrics = [
// General
{
name: 'app.server_active',
type: 'Switch',
desc: 'Number of users currently using grist'
},
{
name: 'app.server_active_span',
type: 'Timer',
desc: 'Total server time spent using grist'
},
{
name: 'app.have_doc_open',
type: 'Switch',
desc: 'Number of users with at least one doc open'
},
{
name: 'app.doc_open_span',
type: 'Timer',
desc: 'Total time spent with at least one doc open'
},
// Docs
{
name: 'docs.num_open',
type: 'Gauge',
desc: 'Number of open docs'
},
{
name: 'performance.node_memory_usage',
type: 'SamplingGauge',
desc: 'Memory utilization in bytes of the node process'
}
// TODO: Implement the following:
// {
// name: 'grist-rt.docs.total_size_open',
// type: 'Gauge',
// desc: 'Cumulative size of open docs'
// }
// {
// name: 'grist-rt.performance.open_standalone_app',
// type: 'ExecutionTimer',
// desc: 'Time to start standalone app'
// }
// {
// name: 'grist-rt.performance.sandbox_recalculation',
// type: 'ExecutionTimer',
// desc: 'Time for sandbox recalculation to occur'
// }
// {
// name: 'grist-rt.performance.open_standalone_app',
// type: 'ExecutionTimer',
// desc: 'Time to start standalone app'
// }
// {
// name: 'grist-rt.performance.node_cpu_usage',
// type: 'SamplingGauge',
// desc: 'Amount of time node was using the cpu in the interval'
// }
// {
// name: 'grist-rt.performance.sandbox_cpu_usage',
// type: 'SamplingGauge',
// desc: 'Amount of time the sandbox was using the cpu in the interval'
// }
// {
// name: 'grist-rt.performance.chrome_cpu_usage',
// type: 'SamplingGauge',
// desc: 'Amount of time chrome was using the cpu in the interval'
// }
// {
// name: 'grist-rt.performance.sandbox_memory_usage',
// type: 'SamplingGauge',
// desc: 'Memory utilization in bytes of the sandbox process'
// }
// {
// name: 'grist-rt.performance.chrome_memory_usage',
// type: 'SamplingGauge',
// desc: 'Memory utilization in bytes of the chrome process'
// }
];

@ -0,0 +1,261 @@
const _ = require('underscore');
const gutil = require('./gutil');
const metricConfig = require('./metricConfig');
// TODO: Create a metric test class and write tests for each metric tool.
/**
* Base class for tools to gather metrics. Should not be instantiated.
*/
function MetricTool(name) {
this.name = name;
}
// Should be implemented by extending classes
MetricTool.prototype._getSuffix = function() {
throw new Error("Not implemented");
};
// Should be overridden by extending classes depending on desired reset behavior
MetricTool.prototype.reset = _.noop;
// Returns the name of the metric with its suffix appended to the end.
// NOTE: Should return names in the same order as getValues.
MetricTool.prototype.getName = function() {
return this.name + '.' + this._getSuffix();
};
// Should be implemented by extending classes. Returns the value of the tool for a bucket.
// @param {Number} bucketEndTime - The desired bucket's end time in milliseconds
MetricTool.prototype.getValue = function(bucketEndTime) {
throw new Error("Not implemented");
};
// Returns a list of all primitive metrics this tool is made up of.
// Only requires overridding by non-primitive metrics.
MetricTool.prototype.getPrimitiveMetrics = function() {
return [this];
};
/**
* Counts the number of times an event has occurred in the current bucket.
*/
function Counter(name) {
MetricTool.call(this, name);
this.val = 0;
}
_.extend(Counter.prototype, MetricTool.prototype);
Counter.prototype.inc = function() {
this.val += 1;
};
Counter.prototype._getSuffix = function() {
return 'count';
};
Counter.prototype.getValue = function(bucketEndTime) {
// If the bucket is more recent than the last one where counting occurred, return 0
return this.val;
};
Counter.prototype.reset = function(bucketEndTime) {
this.val = 0;
};
exports.Counter = Counter;
/**
* Keeps track of a count that persists across buckets.
*/
function Gauge(name) {
MetricTool.call(this, name);
this.val = null;
}
_.extend(Gauge.prototype, MetricTool.prototype);
Gauge.prototype.set = function(num) {
this.val = num;
};
Gauge.prototype.inc = function() {
this.val = (this.val ? this.val + 1 : 1);
};
Gauge.prototype.dec = function() {
this.val -= 1;
};
Gauge.prototype._getSuffix = function() {
return 'total';
};
Gauge.prototype.getValue = function(bucketEndTime) {
return this.val;
};
exports.Gauge = Gauge;
/**
* A gauge that pulls samples using a callback function
*/
function SamplingGauge(name) {
MetricTool.call(this, name);
this.callback = _.constant(null);
}
_.extend(SamplingGauge.prototype, MetricTool.prototype);
SamplingGauge.prototype.assignCallback = function(callback) {
this.callback = callback;
};
SamplingGauge.prototype._getSuffix = function() {
return 'total';
};
SamplingGauge.prototype.getValue = function(bucketEndTime) {
return this.callback();
};
exports.SamplingGauge = SamplingGauge;
/**
* Keeps track of whether or not a certain condition is met. Useful for statistics
* which measure the number of users who meet a certain criteria. Persists across buckets.
*/
function Switch(name) {
MetricTool.call(this, name);
this.val = null;
}
_.extend(Switch.prototype, Gauge.prototype);
Switch.prototype.set = function(bool) {
this.val = bool ? 1 : 0;
};
Switch.prototype._getSuffix = function() {
return 'instances';
};
Switch.prototype.getValue = function(bucketEndTime) {
return this.val;
};
exports.Switch = Switch;
/**
* Keeps track of the amount of time in each bucket that an event is occurring (ms).
*/
function Timer(name) {
MetricTool.call(this, name);
this.val = 0; // The sum of all runtimes in the last updated bucket
this.startTime = 0; // The time (in ms since the bucket started) when the timer was started
this.running = false;
}
_.extend(Timer.prototype, MetricTool.prototype);
Timer.prototype.setRunning = function(bool) {
return bool ? this.start() : this.stop();
};
Timer.prototype.start = function() {
if (this.running) {
return;
}
// Record start time and set to running
this.startTime = Date.now();
this.running = true;
};
Timer.prototype.stop = function() {
if (!this.running) {
return;
}
// Add time since start to value and set running to false
var stopTime = Date.now();
this.val += stopTime - this.startTime;
this.running = false;
};
Timer.prototype._getSuffix = function() {
return 'time';
};
Timer.prototype.getValue = function(bucketEndTime) {
// Add the value and the time to the end of the bucket if the timer is running
return this.val + (this.running ? Math.max(0, bucketEndTime - this.startTime) : 0);
};
Timer.prototype.reset = function(bucketEndTime) {
this.val = 0;
this.startTime = Math.max(bucketEndTime, this.startTime);
};
exports.Timer = Timer;
/**
* Keeps track of the amount of time in an event takes, and the number of times that event occurs (ms).
*/
function ExecutionTimer(name) {
MetricTool.call(this, name);
this.startTime = 0; // The last time (in ms) the timer was started
this.val = 0;
this.running = false;
// Counter keeps track of the total number of executions in the current bucket.
// An execution is in a bucket if it ended in that bucket.
this.counter = new Counter(name);
}
_.extend(ExecutionTimer.prototype, MetricTool.prototype);
ExecutionTimer.prototype.setRunning = function(bool) {
return bool ? this.start() : this.stop();
};
ExecutionTimer.prototype.start = function() {
if (this.running) {
return;
}
this.startTime = Date.now();
this.running = true;
};
ExecutionTimer.prototype.stop = function() {
if (!this.running) {
return;
}
var stopTime = Date.now();
this.val += stopTime - this.startTime;
this.counter.inc();
this.running = false;
};
ExecutionTimer.prototype._getSuffix = function() {
return 'execution_time';
};
ExecutionTimer.prototype.getValue = function(bucketEndTime) {
return this.val;
};
ExecutionTimer.prototype.reset = function(bucketEndTime) {
this.val = 0;
this.counter.reset();
};
ExecutionTimer.prototype.getPrimitiveMetrics = function() {
return [this, this.counter];
};
exports.ExecutionTimer = ExecutionTimer;
// Returns the time rounded down to the start of the current bucket's time window (in ms).
function getBucketStartTime(now) {
return gutil.roundDownToMultiple(now, metricConfig.BUCKET_SIZE);
}
exports.getBucketStartTime = getBucketStartTime;
// Returns the time until the start of the next bucket (in ms).
function getDeltaMs(now) {
return getBucketStartTime(now) + metricConfig.BUCKET_SIZE - now;
}
exports.getDeltaMs = getDeltaMs;

@ -0,0 +1,56 @@
const BLACKLISTED_SUBDOMAINS = new Set([
// from wiki page as of 2018-12-14
'aws',
'gristlogin',
'issues',
'metrics',
'phab',
'releases',
'test',
'vpn',
'www',
// A few more reserved just in case. The minimum length requirement would eliminate
// some in any case, but specified here also in case that minimum changes.
'w', 'ww', 'wwww', 'wwwww',
'docs', 'api', 'static',
'ftp', 'imap', 'pop', 'smtp', 'mail', 'git', 'blog', 'wiki', 'support', 'kb', 'help',
'admin', 'store', 'dev', 'beta', 'dev',
// a few random tech brands
'google', 'apple', 'microsoft', 'ms', 'facebook', 'fb', 'twitter', 'youtube', 'yt',
// updates for new special domains
'current', 'staging', 'prod', 'login', 'login-dev',
]);
/**
*
* Checks whether the subdomain is on the list of forbidden subdomains.
* See https://phab.getgrist.com/w/hosting/v1/urls/#organization-subdomains
*
* Also enforces various sanity checks.
*
* Throws if the subdomain is invalid.
*
*/
export function checkSubdomainValidity(subdomain: string): void {
// stick with limited alphanumeric subdomains.
if (!(/^[a-z][-a-z0-9]*$/.test(subdomain))) {
throw new Error('Domain must include letters, numbers, and dashes only.');
}
// 'docs-*' is reserved for personal orgs.
if (subdomain.startsWith('docs-')) { throw new Error('Domain cannot use reserved prefix "docs-".'); }
// 'doc-worker-*' is reserved for doc workers.
if (subdomain.startsWith('doc-worker-')) { throw new Error('Domain cannot use reserved prefix "doc-worker-".'); }
// special subdomains like _domainkey.
if (subdomain.startsWith('_')) { throw new Error('Domain cannot use reserved prefix "_".'); }
// some domains are currently in use for testing v1.
if (subdomain.startsWith('v1-')) { throw new Error('Domain cannot use reserved prefix "v1-".'); }
// check limit of 63 characters on dns label.
if (subdomain.length > 63) { throw new Error('Domain must contain less than 64 characters.'); }
// check the subdomain isn't too short.
if (subdomain.length <= 2) { throw new Error('Domain must contain more than 2 characters.'); }
// a small blacklist prepared by hand.
if (BLACKLISTED_SUBDOMAINS.has(subdomain)) { throw new Error('Invalid domain value.'); }
}

@ -0,0 +1,107 @@
import * as moment from 'moment-timezone';
// Order of formats to try if the date cannot be parsed as the currently set format.
// Formats are parsed in momentjs strict mode, but separator matching and the MM/DD
// two digit requirement are ignored. Also, partial completion is permitted, so formats
// may match even if only beginning elements are provided.
// TODO: These should be affected by the user's locale/settings.
// TODO: We may want to consider adding default time formats as well to support more
// time formats.
const PARSER_FORMATS: string[] = [
'M D YYYY',
'M D YY',
'M D',
'M',
'MMMM D YYYY',
'MMMM D',
'MMMM Do YYYY',
'MMMM Do',
'MMMM',
'MMM D YYYY',
'MMM D',
'D MMM YYYY',
'D MMM',
'MMM',
'YYYY M D',
'YYYY M',
'YYYY',
'D M YYYY',
'D M YY',
'D M',
'D'
];
interface ParseOptions {
time?: string;
dateFormat?: string;
timeFormat?: string;
timezone?: string;
}
/**
* parseDate - Attempts to parse a date string using several common formats. Returns the
* timestamp of the parsed date in seconds since epoch, or returns null on failure.
* @param {String} date - The date string to parse.
* @param {String} options.dateFormat - The preferred momentjs format to use to parse the
* date. This is attempted before the default formats.
* @param {String} options.time - The time string to parse.
* @param {String} options.timeFormat - The momentjs format to use to parse the time. This
* must be given if options.time is given.
* @param {String} options.timezone - The timezone string for the date/time, which affects
* the resulting timestamp.
*/
export function parseDate(date: string, options: ParseOptions = {}): number | null {
// If no date, return null.
if (!date) {
return null;
}
// Not picky about separators, so replace them in the date and format strings to be spaces.
const separators = /\W+/g;
const dateFormats = PARSER_FORMATS.slice();
// If a preferred parse format is given, set that to be the first parser used.
if (options.dateFormat) {
// Momentjs has an undesirable feature in strict mode where MM and DD
// matches require two digit numbers. Change MM, DD to M, D.
const format = options.dateFormat.replace(/\bMM\b/g, 'M')
.replace(/\bDD\b/g, 'D')
.replace(separators, ' ');
dateFormats.unshift(_getPartialFormat(date, format));
}
const cleanDate = date.replace(separators, ' ');
const datetime = (options.time ? `${cleanDate} ${options.time}` : cleanDate).trim();
for (const f of dateFormats) {
// Momentjs has an undesirable feature in strict mode where HH, mm, and ss
// matches require two digit numbers. Change HH, mm, and ss to H, m, and s.
const timeFormat = options.timeFormat ? options.timeFormat.replace(/\bHH\b/g, 'H')
.replace(/\bmm\b/g, 'm')
.replace(/\bss\b/g, 's') : null;
const fullFormat = options.time && timeFormat ? `${f} ${timeFormat}` : f;
const m = moment.tz(datetime, fullFormat, true, options.timezone || 'UTC');
if (m.isValid()) {
return m.valueOf() / 1000;
}
}
return null;
}
// Helper function to get the partial format string based on the input. Momentjs has a feature
// which allows defaulting to the current year, month and/or day if not accounted for in the
// parser. We remove any parts of the parser not given in the input to take advantage of this
// feature.
function _getPartialFormat(input: string, format: string): string {
// Define a regular expression to match contiguous separators.
const re = /\W+/g;
// Clean off any whitespace from the ends, and count the number of separators.
const inputMatch = input.trim().match(re);
const numInputSeps = inputMatch ? inputMatch.length : 0;
// Find the separator matches in the format string.
let formatMatch;
for (let i = 0; i < numInputSeps + 1; i++) {
formatMatch = re.exec(format);
if (!formatMatch) {
break;
}
}
// Get the format string up until the corresponding input ends.
return formatMatch ? format.slice(0, formatMatch.index) : format;
}

@ -0,0 +1,77 @@
/**
* Plugin's utilities common to server and client.
*/
import {BarePlugin, Implementation} from 'app/plugin/PluginManifest';
export type LocalPluginKind = "installed"|"builtIn";
export interface ImplDescription {
localPluginId: string;
implementation: Implementation;
}
export interface FileParser {
fileExtensions: string[];
parseOptions?: ImplDescription;
fileParser: ImplDescription;
}
// Deprecated, use FileParser or ImportSource instead.
export interface FileImporter {
id: string;
fileExtensions?: string[];
script?: string;
scriptFullPath?: string;
filePicker?: string;
filePickerFullPath?: string;
}
/**
* Manifest parsing error.
*/
export interface ManifestParsingError {
yamlError?: any;
jsonError?: any;
cannotReadError?: any;
missingEntryErrors?: string;
}
/**
* Whether the importer provides a file picker.
*/
export function isPicker(importer: FileImporter): boolean {
return importer.filePicker !== undefined;
}
/**
* A Plugin that was found in the system, either installed or builtin.
*/
export interface LocalPlugin {
/**
* the plugin's manifest
*/
manifest: BarePlugin;
/**
* The path to the plugin's folder.
*/
path: string;
/**
* A name to uniquely identify a LocalPlugin.
*/
readonly id: string;
}
export interface DirectoryScanEntry {
manifest?: BarePlugin;
/**
* User-friendly error messages.
*/
errors?: any[];
path: string;
id: string;
}
/**
* The contributions type.
*/
export type Contribution = "importSource" | "fileParser";

@ -0,0 +1,44 @@
import {ManagerDelta, PermissionDelta, UserAPI} from 'app/common/UserAPI';
/**
* A utility to reset an organization into the state it would have when first
* created - no docs, one workspace called "Home", a single user. Should be
* called by a user who is both an owner of the org and a billing manager.
*/
export async function resetOrg(api: UserAPI, org: string|number) {
const session = await api.getSessionActive();
if (!(session.org && session.org.access === 'owners')) {
throw new Error('user must be an owner of the org to be reset');
}
const billing = api.getBillingAPI();
const account = await billing.getBillingAccount();
if (!account.managers.some(manager => (manager.id === session.user.id))) {
throw new Error('user must be a billing manager');
}
const wss = await api.getOrgWorkspaces(org);
for (const ws of wss) {
if (!ws.isSupportWorkspace) {
await api.deleteWorkspace(ws.id);
}
}
await api.newWorkspace({name: 'Home'}, org);
const permissions: PermissionDelta = { users: {} };
for (const user of (await api.getOrgAccess(org)).users) {
if (user.id !== session.user.id) {
permissions.users![user.email] = null;
}
}
await api.updateOrgPermissions(org, permissions);
// For non-individual accounts, update billing managers (individual accounts will
// throw an error if we try to do this).
if (!account.individual) {
const managers: ManagerDelta = { users: {} };
for (const user of account.managers) {
if (user.id !== session.user.id) {
managers.users[user.email] = null;
}
}
await billing.updateBillingManagers(managers);
}
return api;
}

@ -0,0 +1,85 @@
export const OWNER = 'owners';
export const EDITOR = 'editors';
export const VIEWER = 'viewers';
export const GUEST = 'guests';
export const MEMBER = 'members';
// Roles ordered from most to least permissive.
const roleOrder: Array<Role|null> = [OWNER, EDITOR, VIEWER, MEMBER, GUEST, null];
export type BasicRole = 'owners'|'editors'|'viewers';
export type NonMemberRole = BasicRole|'guests';
export type NonGuestRole = BasicRole|'members';
export type Role = NonMemberRole|'members';
// Returns the BasicRole (or null) with the same effective access as the given role.
export function getEffectiveRole(role: Role|null): BasicRole|null {
if (role === GUEST || role === MEMBER) {
return VIEWER;
} else {
return role;
}
}
export function canEditAccess(role: string|null): boolean {
return role === OWNER;
}
// Note that while canEdit has the same return value as canDelete, the functions are
// kept separate as they may diverge in the future.
export function canEdit(role: string|null): boolean {
return role === OWNER || role === EDITOR;
}
export function canDelete(role: string|null): boolean {
return role === OWNER || role === EDITOR;
}
export function canView(role: string|null): boolean {
return role !== null;
}
// Returns true if the role string is a valid role or null.
export function isValidRole(role: string|null): role is Role|null {
return (roleOrder as Array<string|null>).includes(role);
}
// Returns true if the role string is a valid non-Guest, non-Member, non-null role.
export function isBasicRole(role: string|null): role is BasicRole {
return Boolean(role && role !== GUEST && role !== MEMBER && isValidRole(role));
}
// Returns true if the role string is a valid non-Guest, non-null role.
export function isNonGuestRole(role: string|null): role is NonGuestRole {
return Boolean(role && role !== GUEST && isValidRole(role));
}
/**
* Returns out of any number of group role names the one that offers more permissions. The function
* is overloaded so that the output type matches the specificity of the input values.
*/
export function getStrongestRole<T extends Role|null>(...args: T[]): T {
return getFirstMatchingRole(roleOrder, args);
}
/**
* Returns out of any number of group role names the one that offers fewer permissions. The function
* is overloaded so that the output type matches the specificity of the input values.
*/
export function getWeakestRole<T extends Role|null>(...args: T[]): T {
return getFirstMatchingRole(roleOrder.slice().reverse(), args);
}
// Returns which of the `anyOf` args comes first in `array`. Helper for getStrongestRole
// and getWeakestRole.
function getFirstMatchingRole<T extends Role|null>(array: Array<Role|null>, anyOf: T[]): T {
if (anyOf.length === 0) {
throw new Error(`getFirstMatchingRole: No roles given`);
}
for (const role of anyOf) {
if (!isValidRole(role)) {
throw new Error(`getFirstMatchingRole: Invalid role ${role}`);
}
}
return array.find((item) => anyOf.includes(item as T)) as T;
}

@ -0,0 +1,336 @@
/*** THIS FILE IS AUTO-GENERATED BY sandbox/gen_js_schema.py ***/
// tslint:disable:object-literal-key-quotes
export const schema = {
"_grist_DocInfo": {
docId : "Text",
peers : "Text",
basketId : "Text",
schemaVersion : "Int",
timezone : "Text",
},
"_grist_Tables": {
tableId : "Text",
primaryViewId : "Ref:_grist_Views",
summarySourceTable : "Ref:_grist_Tables",
onDemand : "Bool",
},
"_grist_Tables_column": {
parentId : "Ref:_grist_Tables",
parentPos : "PositionNumber",
colId : "Text",
type : "Text",
widgetOptions : "Text",
isFormula : "Bool",
formula : "Text",
label : "Text",
untieColIdFromLabel : "Bool",
summarySourceCol : "Ref:_grist_Tables_column",
displayCol : "Ref:_grist_Tables_column",
visibleCol : "Ref:_grist_Tables_column",
},
"_grist_Imports": {
tableRef : "Ref:_grist_Tables",
origFileName : "Text",
parseFormula : "Text",
delimiter : "Text",
doublequote : "Bool",
escapechar : "Text",
quotechar : "Text",
skipinitialspace : "Bool",
encoding : "Text",
hasHeaders : "Bool",
},
"_grist_External_database": {
host : "Text",
port : "Int",
username : "Text",
dialect : "Text",
database : "Text",
storage : "Text",
},
"_grist_External_table": {
tableRef : "Ref:_grist_Tables",
databaseRef : "Ref:_grist_External_database",
tableName : "Text",
},
"_grist_TableViews": {
tableRef : "Ref:_grist_Tables",
viewRef : "Ref:_grist_Views",
},
"_grist_TabItems": {
tableRef : "Ref:_grist_Tables",
viewRef : "Ref:_grist_Views",
},
"_grist_TabBar": {
viewRef : "Ref:_grist_Views",
tabPos : "PositionNumber",
},
"_grist_Pages": {
viewRef : "Ref:_grist_Views",
indentation : "Int",
pagePos : "PositionNumber",
},
"_grist_Views": {
name : "Text",
type : "Text",
layoutSpec : "Text",
},
"_grist_Views_section": {
tableRef : "Ref:_grist_Tables",
parentId : "Ref:_grist_Views",
parentKey : "Text",
title : "Text",
defaultWidth : "Int",
borderWidth : "Int",
theme : "Text",
options : "Text",
chartType : "Text",
layoutSpec : "Text",
filterSpec : "Text",
sortColRefs : "Text",
linkSrcSectionRef : "Ref:_grist_Views_section",
linkSrcColRef : "Ref:_grist_Tables_column",
linkTargetColRef : "Ref:_grist_Tables_column",
embedId : "Text",
},
"_grist_Views_section_field": {
parentId : "Ref:_grist_Views_section",
parentPos : "PositionNumber",
colRef : "Ref:_grist_Tables_column",
width : "Int",
widgetOptions : "Text",
displayCol : "Ref:_grist_Tables_column",
visibleCol : "Ref:_grist_Tables_column",
filter : "Text",
},
"_grist_Validations": {
formula : "Text",
name : "Text",
tableRef : "Int",
},
"_grist_REPL_Hist": {
code : "Text",
outputText : "Text",
errorText : "Text",
},
"_grist_Attachments": {
fileIdent : "Text",
fileName : "Text",
fileType : "Text",
fileSize : "Int",
imageHeight : "Int",
imageWidth : "Int",
timeUploaded : "DateTime",
},
"_grist_ACLRules": {
resource : "Ref:_grist_ACLResources",
permissions : "Int",
principals : "Text",
aclFormula : "Text",
aclColumn : "Ref:_grist_Tables_column",
},
"_grist_ACLResources": {
tableId : "Text",
colIds : "Text",
},
"_grist_ACLPrincipals": {
type : "Text",
userEmail : "Text",
userName : "Text",
groupName : "Text",
instanceId : "Text",
},
"_grist_ACLMemberships": {
parent : "Ref:_grist_ACLPrincipals",
child : "Ref:_grist_ACLPrincipals",
},
};
export interface SchemaTypes {
"_grist_DocInfo": {
docId: string;
peers: string;
basketId: string;
schemaVersion: number;
timezone: string;
};
"_grist_Tables": {
tableId: string;
primaryViewId: number;
summarySourceTable: number;
onDemand: boolean;
};
"_grist_Tables_column": {
parentId: number;
parentPos: number;
colId: string;
type: string;
widgetOptions: string;
isFormula: boolean;
formula: string;
label: string;
untieColIdFromLabel: boolean;
summarySourceCol: number;
displayCol: number;
visibleCol: number;
};
"_grist_Imports": {
tableRef: number;
origFileName: string;
parseFormula: string;
delimiter: string;
doublequote: boolean;
escapechar: string;
quotechar: string;
skipinitialspace: boolean;
encoding: string;
hasHeaders: boolean;
};
"_grist_External_database": {
host: string;
port: number;
username: string;
dialect: string;
database: string;
storage: string;
};
"_grist_External_table": {
tableRef: number;
databaseRef: number;
tableName: string;
};
"_grist_TableViews": {
tableRef: number;
viewRef: number;
};
"_grist_TabItems": {
tableRef: number;
viewRef: number;
};
"_grist_TabBar": {
viewRef: number;
tabPos: number;
};
"_grist_Pages": {
viewRef: number;
indentation: number;
pagePos: number;
};
"_grist_Views": {
name: string;
type: string;
layoutSpec: string;
};
"_grist_Views_section": {
tableRef: number;
parentId: number;
parentKey: string;
title: string;
defaultWidth: number;
borderWidth: number;
theme: string;
options: string;
chartType: string;
layoutSpec: string;
filterSpec: string;
sortColRefs: string;
linkSrcSectionRef: number;
linkSrcColRef: number;
linkTargetColRef: number;
embedId: string;
};
"_grist_Views_section_field": {
parentId: number;
parentPos: number;
colRef: number;
width: number;
widgetOptions: string;
displayCol: number;
visibleCol: number;
filter: string;
};
"_grist_Validations": {
formula: string;
name: string;
tableRef: number;
};
"_grist_REPL_Hist": {
code: string;
outputText: string;
errorText: string;
};
"_grist_Attachments": {
fileIdent: string;
fileName: string;
fileType: string;
fileSize: number;
imageHeight: number;
imageWidth: number;
timeUploaded: number;
};
"_grist_ACLRules": {
resource: number;
permissions: number;
principals: string;
aclFormula: string;
aclColumn: number;
};
"_grist_ACLResources": {
tableId: string;
colIds: string;
};
"_grist_ACLPrincipals": {
type: string;
userEmail: string;
userName: string;
groupName: string;
instanceId: string;
};
"_grist_ACLMemberships": {
parent: number;
child: number;
};
}

@ -0,0 +1,43 @@
import {EncActionBundleFromHub} from 'app/common/EncActionBundle';
export const allToken: string = '#ALL';
/**
* Messages received from SQS
*/
export interface Message {
type: MessageType;
content: Invite | EncActionBundleFromHub;
docId: string; // The docId to which the message pertains.
}
export enum MessageType {
invite = 1,
accept,
decline,
action
}
export interface Invite {
senderEmail: string;
senderName?: string;
docId: string; // Indicates the doc to which the user is being invited to join.
docName: string; // Indicates the docName at the time of sending for user doc recognition.
isUnread?: boolean;
isIgnored?: boolean;
}
/**
* Contains information about someone who may or may not be a Grist user.
*/
export interface Peer {
email: string;
name?: string;
instIds?: string[];
}
export interface EmailResult {
email: string;
instIds: string[];
name?: string;
}

@ -0,0 +1,15 @@
/**
* A version of Function.bind() that preserves types.
*/
// tslint:disable:max-line-length
// Bind just the context for a function of up to 4 args.
export function tbind<T, R, Args extends any[]>(func: (this: T, ...a: Args) => R, context: T): (...a: Args) => R;
// Bind context and first arg for a function of up to 5 args.
export function tbind<T, R, X, Args extends any[]>(func: (this: T, x: X, ...a: Args) => R, context: T, x: X): (...a: Args) => R;
export function tbind(func: any, context: any, ...boundArgs: any[]): any {
return func.bind(context, ...boundArgs);
}

@ -0,0 +1,56 @@
/**
* timeFormat(format, date) formats the passed-in Date object using the format string. The format
* string may contain the following:
* 'h': hour (00 - 23)
* 'm': minute (00 - 59)
* 's': second (00 - 59)
* 'd': day of the month (01 - 31)
* 'n': month (01 - 12)
* 'y': 4-digit year
* 'M': milliseconds (000 - 999)
* 'Y': date as 20140212
* 'D': date as 2014-02-12
* 'T': time as 00:51:06
* 'A': full time and date, as 2014-02-12 00:51:06.123
* @param {String} format The format string.
* @param {Date} date The date/time object to format.
* @returns {String} The formatted date and/or time.
*/
function pad(num: number, len: number): string {
const s = num.toString();
return s.length >= len ? s : "00000000".slice(0, len - s.length) + s;
}
type FormatHelper = (out: string[], date: Date) => void;
const timeFormatKeys: {[spec: string]: FormatHelper} = {
h: (out, date) => out.push(pad(date.getHours(), 2)),
m: (out, date) => out.push(pad(date.getMinutes(), 2)),
s: (out, date) => out.push(pad(date.getSeconds(), 2)),
d: (out, date) => out.push(pad(date.getDate(), 2)),
n: (out, date) => out.push(pad(date.getMonth() + 1, 2)),
y: (out, date) => out.push("" + date.getFullYear()),
M: (out, date) => out.push(pad(date.getMilliseconds(), 3)),
Y: (out, date) => timeFormatHelper(out, 'ynd', date),
D: (out, date) => timeFormatHelper(out, 'y-n-d', date),
T: (out, date) => timeFormatHelper(out, 'h:m:s', date),
A: (out, date) => timeFormatHelper(out, 'D T.M', date),
};
function timeFormatHelper(out: string[], format: string, date: Date) {
for (let i = 0, len = format.length; i < len; i++) {
const c = format[i];
const helper = timeFormatKeys[c];
if (helper) {
helper(out, date);
} else {
out.push(c);
}
}
}
export function timeFormat(format: string, date: Date): string {
const out: string[] = [];
timeFormatHelper(out, format, date);
return out.join("");
}

@ -0,0 +1,25 @@
// tslint:disable:max-line-length
// credits: https://stackoverflow.com/questions/49998665/promisified-function-type
// Generic Function definition
type AnyFunction = (...args: any[]) => any;
// Extracts the type if wrapped by a Promise
type Unpacked<T> = T extends Promise<infer U> ? U : T;
type PromisifiedFunction<T extends AnyFunction> =
T extends () => infer U ? () => Promise<Unpacked<U>> :
T extends (a1: infer A1) => infer U ? (a1: A1) => Promise<Unpacked<U>> :
T extends (a1: infer A1, a2: infer A2) => infer U ? (a1: A1, a2: A2) => Promise<Unpacked<U>> :
T extends (a1: infer A1, a2: infer A2, a3: infer A3) => infer U ? (a1: A1, a2: A2, a3: A3) => Promise<Unpacked<U>> :
T extends (a1: infer A1, a2: infer A2, a3: infer A3, a4: infer A4) => infer U ? (a1: A1, a2: A2, a3: A3, a4: A4) => Promise<Unpacked<U>> :
// ...
T extends (...args: any[]) => infer U ? (...args: any[]) => Promise<Unpacked<U>> : T;
/**
* `Promisified<T>` has the same methods as `T` but they all return promises. This is useful when
* creating a stub with `grain-rpc` for an api which is synchronous.
*/
export type Promisified<T> = {
[K in keyof T]: T[K] extends AnyFunction ? PromisifiedFunction<T[K]> : never
};

@ -1,3 +1,6 @@
{
"extends": "../../buildtools/tsconfig-base.json",
"references": [
{ "path": "../plugin" }
]
}

@ -0,0 +1,58 @@
/**
* Given a 2D array of strings, encodes them in tab-separated format.
* Certain values are quoted; when quoted, internal quotes get doubled. The behavior attempts to
* match Excel's tsv encoding and parsing when using copy-paste.
*/
export function tsvEncode(data: any[][]): string {
return data.map(row => row.map(value => encode(value)).join("\t")).join("\n");
}
function encode(rawValue: any): string {
// For encoding-decoding symmetry, we should also encode any values that start with '"',
// but neither Excel nor Google Sheets do it. They both decode such values to something
// different than what produced them (e.g. `"foo""bar"` is encoded into `"foo""bar"`, and
// that is decoded into `foo"bar`).
const value: string = typeof rawValue === 'string' ? rawValue :
(rawValue == null ? "" : String(rawValue));
if (value.includes("\t") || value.includes("\n")) {
return '"' + value.replace(/"/g, '""') + '"';
}
return value;
}
/**
* Given a tab-separated string, decodes it and returns a 2D array of strings.
* TODO: This does not yet deal with Windows line endings (\r or \r\n).
*/
export function tsvDecode(tsvString: string): string[][] {
const lines: string[][] = [];
let row: string[] = [];
// This is a complex regexp but it does the job of a lot of parsing code. Here are the parts:
// A: [^\t\n]* Sequence of character that does not require the field to get quoted.
// B: ([^"]*"")*[^"]* Sequence of characters containing all double-quotes in pairs (i.e. `""`)
// C: "B"(?!") Quoted sequence, with all double-quotes inside paired up, and ending in a single quote.
// D: C?A A value for one field, a relaxation of C|A (to cope with not-quite expected data)
// E: D(\t|\n|$) Field value with field, line, or file terminator.
const fieldRegexp = /(("([^"]*"")*[^"]*"(?!"))?[^\t\n]*)(\t|\n|$)/g;
for (;;) {
const m = fieldRegexp.exec(tsvString);
if (!m) { break; }
const sep = m[4];
let value = m[1];
if (value.startsWith('"')) {
// It's a quoted value, so doubled-up quotes should became individual quotes, and individual
// quotes should be removed.
value = value.replace(/"([^"]*"")*[^"]*"(?!")/, q => q.slice(1, -1).replace(/""/g, '"'));
}
row.push(value);
if (sep !== '\t') {
lines.push(row);
row = [];
if (sep === '') {
break;
}
}
}
return lines;
}

@ -0,0 +1,44 @@
/**
* Code and declarations shared by browser and server-side code for handling uploads.
*
* Browser code has several functions available in app/client/lib/uploads.ts which return an
* UploadResult that represents an upload. An upload may contain multiple files.
*
* An upload is identified by a numeric uploadId which is unique within an UploadSet. An UploadSet
* is collection of uploads tied to a browser session (as maintained by app/server/lib/Client).
* When the session ends, all uploads are cleaned up.
*
* The uploadId is useful to identify the upload to the server, which can then consume the actual
* files there. It may also be used to clean up the upload once it is no longer needed.
*
* Files within an upload can be identified by their index in UploadResult.files array. The
* origName available for files is not guaranteed to be unique.
*
* Implementation detail: The upload is usually a temporary directory on the server, but may be a
* collection of non-temporary files when files are selected using Electron's native file picker.
*/
/**
* Represents a single upload, containing one or more files. Empty uploads are never created.
*/
export interface UploadResult {
uploadId: number;
files: FileUploadResult[];
}
/**
* Represents a single file within an upload. This is the only information made available to the
* browser. (In particular, while the server knows also the actual path of the file on the server,
* the browser has no need for it and should not know it.)
*/
export interface FileUploadResult {
origName: string; // The filename that the user reports for the file (not guaranteed unique).
size: number; // The size of the file in bytes.
ext: string; // The extension of the file, starting with "."
}
/**
* Path where the server accepts POST requests with uploads. Don't include a leading / so that
* the page's <base> will be respected.
*/
export const UPLOAD_URL_PATH = 'uploads';

@ -0,0 +1,88 @@
import {extractOrgParts, GristLoadConfig} from 'app/common/gristUrls';
export function getGristConfig(): GristLoadConfig {
return (window as any).gristConfig || {};
}
/**
*
* Adds /o/ORG to the supplied path, with ORG extracted from current URL if possible.
* If not, path is returned as is, but with any trailing / removed for consistency.
*
*/
export function addCurrentOrgToPath(path: string, skipIfInDomain: boolean = false) {
if (typeof window === 'undefined' || !window) { return path; }
return addOrgToPath(path, window.location.href, skipIfInDomain);
}
/**
*
* Adds /o/ORG to the supplied path, with ORG extracted from the page URL if possible.
* If not, path is returned as is, but with any trailing / removed for consistency.
*
*/
export function addOrgToPath(path: string, page: string, skipIfInDomain: boolean = false) {
if (typeof window === 'undefined' || !window) { return path; }
if (path.includes('/o/')) { return path; }
const src = new URL(page);
const srcParts = extractOrgParts(src.host, src.pathname);
if (srcParts.mismatch) {
throw new Error('Cannot figure out what organization the URL is for.');
}
path = path.replace(/\/$/, '');
if (!srcParts.subdomain) {
return path;
}
if (skipIfInDomain && srcParts.orgFromHost) {
return path;
}
return `${path}/o/${srcParts.subdomain}`;
}
/**
* Expands an endpoint path to a full url anchored to the given doc worker base url.
*/
export function docUrl(docWorkerUrl: string|null|undefined, path?: string) {
const base = document.querySelector('base');
const baseHref = base && base.href;
const baseUrl = new URL(docWorkerUrl || baseHref || window.location.origin);
return baseUrl.toString().replace(/\/$/, '') + (path ? `/${path}` : '');
}
// Get a url on the same webserver as the current page, adding a prefix to encode
// the current organization if necessary.
export function getOriginUrl(path: string) {
return `${window.location.origin}${addCurrentOrgToPath('/', true)}${path}`;
}
// Return a string docId if server has provided one (as in hosted Grist), otherwise null
// (as in classic Grist).
export function getInitialDocAssignment(): string|null {
return getGristConfig().assignmentId || null;
}
// Return true if we are on a page that can send metrics.
// TODO: all pages should send suitable metrics.
export function pageHasMetrics(): boolean {
// No metric support on hosted grist.
return !getGristConfig().homeUrl;
}
// Return true if we are on a page that can supply a doc list.
// TODO: the doclist object isn't relevant to hosted grist and should be factored out.
export function pageHasDocList(): boolean {
// No doc list support on hosted grist.
return !getGristConfig().homeUrl;
}
// Return true if we are on a page that has access to home api.
export function pageHasHome(): boolean {
return Boolean(getGristConfig().homeUrl);
}
// Construct a url by adding `path` to the home url (adding in the part to the current
// org if needed), and fetch from it.
export function fetchFromHome(path: string, opts: RequestInit): Promise<Response> {
const baseUrl = addCurrentOrgToPath(getGristConfig().homeUrl!);
return window.fetch(`${baseUrl}${path}`, opts);
}

@ -0,0 +1,3 @@
export const version = "1.0.2-dev";
export const channel = "devtest";
export const gitcommit = "e33c4e5aeM";

@ -0,0 +1,475 @@
import * as crypto from 'crypto';
import * as express from 'express';
import {EntityManager} from 'typeorm';
import {ApiError} from 'app/common/ApiError';
import {FullUser} from 'app/common/LoginSessionAPI';
import {OrganizationProperties} from 'app/common/UserAPI';
import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession';
import {expressWrap} from 'app/server/lib/expressWrap';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import * as log from 'app/server/lib/log';
import {getDocScope, getScope, integerParam, isParameterOn, sendOkReply,
sendReply, stringParam} from 'app/server/lib/requestUtils';
import {Request} from 'express';
import {User} from './entity/User';
import {HomeDBManager} from './lib/HomeDBManager';
// exposed for testing purposes
export const Deps = {
apiKeyGenerator: () => crypto.randomBytes(20).toString('hex')
};
// Fetch the org this request was made for, or null if it isn't tied to a particular org.
// Early middleware should have put the org in the request object for us.
export function getOrgFromRequest(req: Request): string|null {
return (req as RequestWithOrg).org || null;
}
/**
* Compute the signature of the user's email address using HelpScout's secret key, to prove to
* HelpScout the user identity for identifying customer information and conversation history.
*/
function helpScoutSign(email: string): string|undefined {
const secretKey = process.env.HELP_SCOUT_SECRET_KEY;
if (!secretKey) { return undefined; }
return crypto.createHmac('sha256', secretKey).update(email).digest('hex');
}
/**
* Fetch an identifier for an organization from the "oid" parameter of the request.
* - Integers are accepted, and will be compared with values in orgs.id column
* - Strings are accepted, and will be compared with values in orgs.domain column
* (or, if they match the pattern docs-NNNN, will check orgs.owner_id)
* - The special string "current" is replaced with the current org domain embedded
* in the url
* - If there is no identifier available, a 400 error is thrown.
*/
export function getOrgKey(req: Request): string|number {
let orgKey: string|null = stringParam(req.params.oid);
if (orgKey === 'current') {
orgKey = getOrgFromRequest(req);
}
if (!orgKey) {
throw new ApiError("No organization chosen", 400);
} else if (/^\d+$/.test(orgKey)) {
return parseInt(orgKey, 10);
}
return orgKey;
}
// Adds an non-personal org with a new billingAccout, with the given name and domain.
// Returns a QueryResult with the orgId on success.
export function addOrg(
dbManager: HomeDBManager,
userId: number,
props: Partial<OrganizationProperties>,
): Promise<number> {
return dbManager.connection.transaction(async manager => {
const user = await manager.findOne(User, userId);
if (!user) { return handleDeletedUser(); }
const query = await dbManager.addOrg(user, props, false, true, manager);
if (query.status !== 200) { throw new ApiError(query.errMessage!, query.status); }
return query.data!;
});
}
/**
* Provides a REST API for the landing page, which returns user's workspaces, organizations and documents.
* Temporarily sqlite database is used. Later it will be changed to RDS Aurora or PostgreSQL.
*/
export class ApiServer {
/**
* Add API endpoints to the specified connection. An error handler is added to /api to make sure
* all error responses have a body in json format.
*
* Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside
* to apply to these routes, and trustOrigin too for cross-domain requests.
*/
constructor(
private _app: express.Application,
private _dbManager: HomeDBManager,
) {
this._addEndpoints();
}
private _addEndpoints(): void {
// GET /api/orgs
// Get all organizations user may have some access to.
this._app.get('/api/orgs', expressWrap(async (req, res) => {
const userId = getUserId(req);
const domain = getOrgFromRequest(req);
const merged = Boolean(req.query.merged);
const query = merged ?
await this._dbManager.getMergedOrgs(userId, userId, domain) :
await this._dbManager.getOrgs(userId, domain);
return sendReply(req, res, query);
}));
// GET /api/workspace/:wid
// Get workspace by id, returning nested documents that user has access to.
this._app.get('/api/workspaces/:wid', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
const query = await this._dbManager.getWorkspace(getScope(req), wsId);
return sendReply(req, res, query);
}));
// GET /api/orgs/:oid
// Get organization by id
this._app.get('/api/orgs/:oid', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.getOrg(getScope(req), org);
return sendReply(req, res, query);
}));
// GET /api/orgs/:oid/workspaces
// Get all workspaces and nested documents of organization that user has access to.
this._app.get('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.getOrgWorkspaces(getScope(req), org);
return sendReply(req, res, query);
}));
// POST /api/orgs
// Body params: name (required), domain
// Create a new org.
this._app.post('/api/orgs', expressWrap(async (req, res) => {
// Don't let anonymous users end up owning organizations, it will be confusing.
// Maybe if the user has presented credentials this would be ok - but addOrg
// doesn't have access to that information yet, so punting on this.
// TODO: figure out who should be allowed to create organizations
const userId = getAuthorizedUserId(req);
const orgId = await addOrg(this._dbManager, userId, req.body);
return sendOkReply(req, res, orgId);
}));
// PATCH /api/orgs/:oid
// Body params: name, domain
// Update the specified org.
this._app.patch('/api/orgs/:oid', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.updateOrg(getScope(req), org, req.body);
return sendReply(req, res, query);
}));
// // DELETE /api/orgs/:oid
// Delete the specified org and all included workspaces and docs.
this._app.delete('/api/orgs/:oid', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.deleteOrg(getScope(req), org);
return sendReply(req, res, query);
}));
// POST /api/orgs/:oid/workspaces
// Body params: name
// Create a new workspace owned by the specific organization.
this._app.post('/api/orgs/:oid/workspaces', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.addWorkspace(getScope(req), org, req.body);
return sendReply(req, res, query);
}));
// PATCH /api/workspaces/:wid
// Body params: name
// Update the specified workspace.
this._app.patch('/api/workspaces/:wid', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
const query = await this._dbManager.updateWorkspace(getScope(req), wsId, req.body);
return sendReply(req, res, query);
}));
// // DELETE /api/workspaces/:wid
// Delete the specified workspace and all included docs.
this._app.delete('/api/workspaces/:wid', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
return sendReply(req, res, query);
}));
// POST /api/workspaces/:wid/remove
// Soft-delete the specified workspace. If query parameter "permanent" is set,
// delete permanently.
this._app.post('/api/workspaces/:wid/remove', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
if (isParameterOn(req.query.permanent)) {
const query = await this._dbManager.deleteWorkspace(getScope(req), wsId);
return sendReply(req, res, query);
} else {
await this._dbManager.softDeleteWorkspace(getScope(req), wsId);
return sendOkReply(req, res);
}
}));
// POST /api/workspaces/:wid/unremove
// Recover the specified workspace if it was previously soft-deleted and is
// still available.
this._app.post('/api/workspaces/:wid/unremove', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
await this._dbManager.undeleteWorkspace(getScope(req), wsId);
return sendOkReply(req, res);
}));
// POST /api/workspaces/:wid/docs
// Create a new doc owned by the specific workspace.
this._app.post('/api/workspaces/:wid/docs', expressWrap(async (req, res) => {
const wsId = integerParam(req.params.wid);
const query = await this._dbManager.addDocument(getScope(req), wsId, req.body);
return sendReply(req, res, query);
}));
// PATCH /api/docs/:did
// Update the specified doc.
this._app.patch('/api/docs/:did', expressWrap(async (req, res) => {
const query = await this._dbManager.updateDocument(getDocScope(req), req.body);
return sendReply(req, res, query);
}));
// POST /api/docs/:did/unremove
// Recover the specified doc if it was previously soft-deleted and is
// still available.
this._app.post('/api/docs/:did/unremove', expressWrap(async (req, res) => {
await this._dbManager.undeleteDocument(getDocScope(req));
return sendOkReply(req, res);
}));
// PATCH /api/orgs/:oid/access
// Update the specified org acl rules.
this._app.patch('/api/orgs/:oid/access', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const delta = req.body.delta;
const query = await this._dbManager.updateOrgPermissions(getScope(req), org, delta);
return sendReply(req, res, query);
}));
// PATCH /api/workspaces/:wid/access
// Update the specified workspace acl rules.
this._app.patch('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
const workspaceId = integerParam(req.params.wid);
const delta = req.body.delta;
const query = await this._dbManager.updateWorkspacePermissions(getScope(req), workspaceId, delta);
return sendReply(req, res, query);
}));
// GET /api/docs/:did
// Get information about a document.
this._app.get('/api/docs/:did', expressWrap(async (req, res) => {
const query = await this._dbManager.getDoc(getDocScope(req));
return sendOkReply(req, res, query);
}));
// PATCH /api/docs/:did/access
// Update the specified doc acl rules.
this._app.patch('/api/docs/:did/access', expressWrap(async (req, res) => {
const delta = req.body.delta;
const query = await this._dbManager.updateDocPermissions(getDocScope(req), delta);
return sendReply(req, res, query);
}));
// PATCH /api/docs/:did/move
// Move the doc to the workspace specified in the body.
this._app.patch('/api/docs/:did/move', expressWrap(async (req, res) => {
const workspaceId = req.body.workspace;
const query = await this._dbManager.moveDoc(getDocScope(req), workspaceId);
return sendReply(req, res, query);
}));
this._app.patch('/api/docs/:did/pin', expressWrap(async (req, res) => {
const query = await this._dbManager.pinDoc(getDocScope(req), true);
return sendReply(req, res, query);
}));
this._app.patch('/api/docs/:did/unpin', expressWrap(async (req, res) => {
const query = await this._dbManager.pinDoc(getDocScope(req), false);
return sendReply(req, res, query);
}));
// GET /api/orgs/:oid/access
// Get user access information regarding an org
this._app.get('/api/orgs/:oid/access', expressWrap(async (req, res) => {
const org = getOrgKey(req);
const query = await this._dbManager.getOrgAccess(getScope(req), org);
return sendReply(req, res, query);
}));
// GET /api/workspaces/:wid/access
// Get user access information regarding a workspace
this._app.get('/api/workspaces/:wid/access', expressWrap(async (req, res) => {
const workspaceId = integerParam(req.params.wid);
const query = await this._dbManager.getWorkspaceAccess(getScope(req), workspaceId);
return sendReply(req, res, query);
}));
// GET /api/docs/:did/access
// Get user access information regarding a doc
this._app.get('/api/docs/:did/access', expressWrap(async (req, res) => {
const query = await this._dbManager.getDocAccess(getDocScope(req));
return sendReply(req, res, query);
}));
// GET /api/profile/user
// Get user's profile
this._app.get('/api/profile/user', expressWrap(async (req, res) => {
const fullUser = await this._getFullUser(req);
return sendOkReply(req, res, fullUser);
}));
// POST /api/profile/user/name
// Body params: string
// Update users profile.
this._app.post('/api/profile/user/name', expressWrap(async (req, res) => {
const userId = getUserId(req);
if (!(req.body && req.body.name)) {
throw new ApiError('Name expected in the body', 400);
}
const name = req.body.name;
await this._dbManager.updateUserName(userId, name);
res.sendStatus(200);
}));
// GET /api/profile/apikey
// Get user's apiKey
this._app.get('/api/profile/apikey', expressWrap(async (req, res) => {
const userId = getUserId(req);
const user = await User.findOne(userId);
if (user) {
// The null value is of no interest to the user, let's show empty string instead.
res.send(user.apiKey || '');
return;
}
handleDeletedUser();
}));
// POST /api/profile/apikey
// Update user's apiKey
this._app.post('/api/profile/apikey', expressWrap(async (req, res) => {
const userId = getAuthorizedUserId(req);
const force = req.body ? req.body.force : false;
const manager = this._dbManager.connection.manager;
let user = await manager.findOne(User, userId);
if (!user) { return handleDeletedUser(); }
if (!user.apiKey || force) {
user = await updateApiKeyWithRetry(manager, user);
res.status(200).send(user.apiKey);
} else {
res.status(400).send({error: "An apikey is already set, use `{force: true}` to override it."});
}
}));
// DELETE /api/profile/apiKey
// Delete apiKey
this._app.delete('/api/profile/apikey', expressWrap(async (req, res) => {
const userId = getAuthorizedUserId(req);
await this._dbManager.connection.transaction(async manager => {
const user = await manager.findOne(User, userId);
if (!user) {return handleDeletedUser(); }
user.apiKey = null;
await manager.save(User, user);
});
res.sendStatus(200);
}));
// GET /api/session/access/active
// Returns active user and active org (if any)
this._app.get('/api/session/access/active', expressWrap(async (req, res) => {
const fullUser = await this._getFullUser(req);
const domain = getOrgFromRequest(req);
const org = domain ? (await this._dbManager.getOrg(getScope(req), domain || null)) : null;
const orgError = (org && org.errMessage) ? {error: org.errMessage, status: org.status} : undefined;
return sendOkReply(req, res, {
user: {...fullUser, helpScoutSignature: helpScoutSign(fullUser.email)},
org: (org && org.data) || null,
orgError
});
}));
// POST /api/session/access/active
// Body params: email (required)
// Sets active user for active org
this._app.post('/api/session/access/active', expressWrap(async (req, res) => {
const mreq = req as RequestWithLogin;
const domain = getOrgFromRequest(mreq);
const email = req.body.email;
if (!email) { throw new ApiError('email required', 400); }
try {
// Modify session copy in request. Will be saved to persistent storage before responding
// by express-session middleware.
linkOrgWithEmail(mreq.session, req.body.email, domain || '');
return sendOkReply(req, res, {email});
} catch (e) {
throw new ApiError('email not available', 403);
}
}));
// GET /api/session/access/all
// Returns all user profiles (with ids) and all orgs they can access.
// Flattens personal orgs into a single org.
this._app.get('/api/session/access/all', expressWrap(async (req, res) => {
const domain = getOrgFromRequest(req);
const users = getUserProfiles(req);
const userId = getUserId(req);
const orgs = await this._dbManager.getMergedOrgs(userId, users, domain);
if (orgs.errMessage) { throw new ApiError(orgs.errMessage, orgs.status); }
return sendOkReply(req, res, {
users: await this._dbManager.completeProfiles(users),
orgs: orgs.data
});
}));
// DELETE /users/:uid
// Delete the specified user, their personal organization, removing them from all groups.
// Not available to the anonymous user.
// TODO: should orphan orgs, inaccessible by anyone else, get deleted when last user
// leaves?
this._app.delete('/api/users/:uid', expressWrap(async (req, res) => {
const userIdToDelete = parseInt(req.params.uid, 10);
if (!(req.body && req.body.name !== undefined)) {
throw new ApiError('to confirm deletion of a user, provide their name', 400);
}
const query = await this._dbManager.deleteUser(getScope(req), userIdToDelete, req.body.name);
return sendReply(req, res, query);
}));
}
private async _getFullUser(req: Request): Promise<FullUser> {
const mreq = req as RequestWithLogin;
const userId = getUserId(mreq);
const fullUser = await this._dbManager.getFullUser(userId);
const domain = getOrgFromRequest(mreq);
const sessionUser = getSessionUser(mreq.session, domain || '');
const loginMethod = sessionUser && sessionUser.profile ? sessionUser.profile.loginMethod : undefined;
return {...fullUser, loginMethod};
}
}
/**
* Throw the error for when a user has been deleted since point of call (very unlikely to happen).
*/
function handleDeletedUser(): never {
throw new ApiError("user not known", 401);
}
/**
* Helper to update a user's apiKey. Update might fail because of the DB uniqueness constraint on
* the apiKey (although it is very unlikely according to `crypto`), we retry until success. Fails
* after 5 unsuccessful attempts.
*/
async function updateApiKeyWithRetry(manager: EntityManager, user: User): Promise<User> {
const currentKey = user.apiKey;
for (let i = 0; i < 5; ++i) {
user.apiKey = Deps.apiKeyGenerator();
try {
// if new key is the same as the current, the db update won't fail so we check it here (very
// unlikely to happen but but still better to handle)
if (user.apiKey === currentKey) {
throw new Error('the new key is the same as the current key');
}
return await manager.save(User, user);
} catch (e) {
// swallow and retry
log.warn(`updateApiKeyWithRetry: failed attempt ${i}/5, %s`, e);
}
}
throw new Error('Could not generate a valid api key.');
}

@ -0,0 +1,58 @@
import {BaseEntity, ChildEntity, Column, Entity, JoinColumn, ManyToOne, OneToOne,
PrimaryGeneratedColumn, RelationId, TableInheritance} from "typeorm";
import {Document} from "./Document";
import {Group} from "./Group";
import {Organization} from "./Organization";
import {Workspace} from "./Workspace";
@Entity('acl_rules')
@TableInheritance({ column: { type: "int", name: "type" } })
export class AclRule extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@Column()
public permissions: number;
@OneToOne(type => Group, group => group.aclRule)
@JoinColumn({name: "group_id"})
public group: Group;
}
@ChildEntity()
export class AclRuleWs extends AclRule {
@ManyToOne(type => Workspace, workspace => workspace.aclRules)
@JoinColumn({name: "workspace_id"})
public workspace: Workspace;
@RelationId((aclRule: AclRuleWs) => aclRule.workspace)
public workspaceId: number;
}
@ChildEntity()
export class AclRuleOrg extends AclRule {
@ManyToOne(type => Organization, organization => organization.aclRules)
@JoinColumn({name: "org_id"})
public organization: Organization;
@RelationId((aclRule: AclRuleOrg) => aclRule.organization)
public orgId: number;
}
@ChildEntity()
export class AclRuleDoc extends AclRule {
@ManyToOne(type => Document, document => document.aclRules)
@JoinColumn({name: "doc_id"})
public document: Document;
@RelationId((aclRule: AclRuleDoc) => aclRule.document)
public docId: number;
}

@ -0,0 +1,27 @@
import {BaseEntity, Column, CreateDateColumn, Entity, JoinColumn, ManyToOne,
PrimaryColumn} from 'typeorm';
import {Document} from './Document';
import {Organization} from './Organization';
@Entity({name: 'aliases'})
export class Alias extends BaseEntity {
@PrimaryColumn({name: 'org_id'})
public orgId: number;
@PrimaryColumn({name: 'url_id'})
public urlId: string;
@Column({name: 'doc_id'})
public docId: string;
@ManyToOne(type => Document)
@JoinColumn({name: 'doc_id'})
public doc: Document;
@ManyToOne(type => Organization)
@JoinColumn({name: 'org_id'})
public org: Organization;
@CreateDateColumn({name: 'created_at'})
public createdAt: Date;
}

@ -0,0 +1,65 @@
import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from 'typeorm';
import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
import {Organization} from 'app/gen-server/entity/Organization';
import {Product} from 'app/gen-server/entity/Product';
import {nativeValues} from 'app/gen-server/lib/values';
// This type is for billing account status information. Intended for stuff
// like "free trial running out in N days".
interface BillingAccountStatus {
stripeStatus?: string;
currentPeriodEnd?: Date;
message?: string;
}
/**
* This relates organizations to products. It holds any stripe information
* needed to be able to update and pay for the product that applies to the
* organization. It has a list of managers detailing which users have the
* right to view and edit these settings.
*/
@Entity({name: 'billing_accounts'})
export class BillingAccount extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@ManyToOne(type => Product)
@JoinColumn({name: 'product_id'})
public product: Product;
@Column()
public individual: boolean;
// A flag for when all is well with the user's subscription.
// Probably shouldn't use this to drive whether service is provided or not.
// Strip recommends updating an end-of-service datetime every time payment
// is received, adding on a grace period of some days.
@Column({name: 'in_good_standing', default: nativeValues.trueValue})
public inGoodStanding: boolean;
@Column({type: nativeValues.jsonEntityType, nullable: true})
public status: BillingAccountStatus;
@Column({name: 'stripe_customer_id', type: String, nullable: true})
public stripeCustomerId: string | null;
@Column({name: 'stripe_subscription_id', type: String, nullable: true})
public stripeSubscriptionId: string | null;
@Column({name: 'stripe_plan_id', type: String, nullable: true})
public stripePlanId: string | null;
@OneToMany(type => BillingAccountManager, manager => manager.billingAccount)
public managers: BillingAccountManager[];
@OneToMany(type => Organization, org => org.billingAccount)
public orgs: Organization[];
// A calculated column that is true if it looks like there is a paid plan.
@Column({name: 'paid', type: 'boolean', insert: false, select: false})
public paid?: boolean;
// A calculated column summarizing whether active user is a manager of the billing account.
// (No @Column needed since calculation is done in javascript not sql)
public isManager?: boolean;
}

@ -0,0 +1,26 @@
import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn} from 'typeorm';
import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
import {User} from 'app/gen-server/entity/User';
/**
* A list of users with the right to modify a giving billing account.
*/
@Entity({name: 'billing_account_managers'})
export class BillingAccountManager extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@Column({name: 'billing_account_id'})
public billingAccountId: number;
@ManyToOne(type => BillingAccount, { onDelete: 'CASCADE' })
@JoinColumn({name: 'billing_account_id'})
public billingAccount: BillingAccount;
@Column({name: 'user_id'})
public userId: number;
@ManyToOne(type => User, { onDelete: 'CASCADE' })
@JoinColumn({name: 'user_id'})
public user: User;
}

@ -0,0 +1,69 @@
import {ApiError} from 'app/common/ApiError';
import {Role} from 'app/common/roles';
import {DocumentProperties, documentPropertyKeys, NEW_DOCUMENT_CODE} from "app/common/UserAPI";
import {nativeValues} from 'app/gen-server/lib/values';
import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryColumn} from "typeorm";
import {AclRuleDoc} from "./AclRule";
import {Alias} from "./Alias";
import {Resource} from "./Resource";
import {Workspace} from "./Workspace";
// Acceptable ids for use in document urls.
const urlIdRegex = /^[-a-z0-9]+$/i;
function isValidUrlId(urlId: string) {
if (urlId === NEW_DOCUMENT_CODE) { return false; }
return urlIdRegex.exec(urlId);
}
@Entity({name: 'docs'})
export class Document extends Resource {
@PrimaryColumn()
public id: string;
@ManyToOne(type => Workspace)
@JoinColumn({name: 'workspace_id'})
public workspace: Workspace;
@OneToMany(type => AclRuleDoc, aclRule => aclRule.document)
public aclRules: AclRuleDoc[];
// Indicates whether the doc is pinned to the org it lives in.
@Column({name: 'is_pinned', default: false})
public isPinned: boolean;
// Property that may be returned when the doc is fetched to indicate the access the
// fetching user has on the doc, i.e. 'owners', 'editors', 'viewers'
public access: Role|null;
// a computed column with permissions.
// {insert: false} makes sure typeorm doesn't try to put values into such
// a column when creating documents.
@Column({name: 'permissions', type: 'text', select: false, insert: false, update: false})
public permissions?: any;
@Column({name: 'url_id', type: 'text', nullable: true})
public urlId: string|null;
@Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
public removedAt: Date|null;
@OneToMany(type => Alias, alias => alias.doc)
public aliases: Alias[];
public checkProperties(props: any): props is Partial<DocumentProperties> {
return super.checkProperties(props, documentPropertyKeys);
}
public updateFromProperties(props: Partial<DocumentProperties>) {
super.updateFromProperties(props);
if (props.isPinned !== undefined) { this.isPinned = props.isPinned; }
if (props.urlId !== undefined) {
if (props.urlId !== null && !isValidUrlId(props.urlId)) {
throw new ApiError('invalid urlId', 400);
}
this.urlId = props.urlId;
}
}
}

@ -0,0 +1,33 @@
import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToOne, PrimaryGeneratedColumn} from "typeorm";
import {AclRule} from "./AclRule";
import {User} from "./User";
@Entity({name: 'groups'})
export class Group extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@Column()
public name: string;
@ManyToMany(type => User)
@JoinTable({
name: 'group_users',
joinColumn: {name: 'group_id'},
inverseJoinColumn: {name: 'user_id'}
})
public memberUsers: User[];
@ManyToMany(type => Group)
@JoinTable({
name: 'group_groups',
joinColumn: {name: 'group_id'},
inverseJoinColumn: {name: 'subgroup_id'}
})
public memberGroups: Group[];
@OneToOne(type => AclRule, aclRule => aclRule.group)
public aclRule: AclRule;
}

@ -0,0 +1,25 @@
import {BaseEntity, Column, Entity, JoinColumn, ManyToOne, PrimaryColumn} from "typeorm";
import {User} from "./User";
@Entity({name: 'logins'})
export class Login extends BaseEntity {
@PrimaryColumn()
public id: number;
// This is the normalized email address we use for equality and indexing.
@Column()
public email: string;
// This is how the user's email address should be displayed.
@Column({name: 'display_email'})
public displayEmail: string;
@Column({name: 'user_id'})
public userId: number;
@ManyToOne(type => User)
@JoinColumn({name: 'user_id'})
public user: User;
}

@ -0,0 +1,79 @@
import {Column, Entity, JoinColumn, ManyToOne, OneToMany, OneToOne,
PrimaryGeneratedColumn, RelationId} from "typeorm";
import {Role} from "app/common/roles";
import {OrganizationProperties, organizationPropertyKeys} from "app/common/UserAPI";
import {AclRuleOrg} from "./AclRule";
import {BillingAccount} from "./BillingAccount";
import {Resource} from "./Resource";
import {User} from "./User";
import {Workspace} from "./Workspace";
// Information about how an organization may be accessed.
export interface AccessOption {
id: number; // a user id
email: string; // a user email
name: string; // a user name
perms: number; // permissions the user would have on organization
}
export interface AccessOptionWithRole extends AccessOption {
access: Role; // summary of permissions
}
@Entity({name: 'orgs'})
export class Organization extends Resource {
@PrimaryGeneratedColumn()
public id: number;
@Column({
nullable: true
})
public domain: string;
@OneToOne(type => User, user => user.personalOrg)
@JoinColumn({name: 'owner_id'})
public owner: User;
@RelationId((org: Organization) => org.owner)
public ownerId: number;
@OneToMany(type => Workspace, workspace => workspace.org)
public workspaces: Workspace[];
@OneToMany(type => AclRuleOrg, aclRule => aclRule.organization)
public aclRules: AclRuleOrg[];
@Column({name: 'billing_account_id'})
public billingAccountId: number;
@ManyToOne(type => BillingAccount)
@JoinColumn({name: 'billing_account_id'})
public billingAccount: BillingAccount;
// Property that may be returned when the org is fetched to indicate the access the
// fetching user has on the org, i.e. 'owners', 'editors', 'viewers'
public access: string;
// Property that may be used internally to track multiple ways an org can be accessed
public accessOptions?: AccessOptionWithRole[];
// a computed column with permissions.
// {insert: false} makes sure typeorm doesn't try to put values into such
// a column when creating organizations.
@Column({name: 'permissions', type: 'text', select: false, insert: false})
public permissions?: any;
// For custom domains, this is the preferred host associated with this org/team.
@Column({name: 'host', type: 'text', nullable: true})
public host: string|null;
public checkProperties(props: any): props is Partial<OrganizationProperties> {
return super.checkProperties(props, organizationPropertyKeys);
}
public updateFromProperties(props: Partial<OrganizationProperties>) {
super.updateFromProperties(props);
if (props.domain) { this.domain = props.domain; }
}
}

@ -0,0 +1,176 @@
import {Features} from 'app/common/Features';
import {nativeValues} from 'app/gen-server/lib/values';
import * as assert from 'assert';
import {BaseEntity, Column, Connection, Entity, PrimaryGeneratedColumn} from 'typeorm';
/**
* A summary of features used in 'starter' plans.
*/
export const starterFeatures: Features = {
workspaces: true,
// no vanity domain
maxDocsPerOrg: 10,
maxSharesPerDoc: 2,
maxWorkspacesPerOrg: 1
};
/**
* A summary of features used in 'team' plans.
*/
export const teamFeatures: Features = {
workspaces: true,
vanityDomain: true,
maxSharesPerWorkspace: 0, // all workspace shares need to be org members.
maxSharesPerDoc: 2
};
/**
* A summary of features used in unrestricted grandfathered accounts, and also
* in some test settings.
*/
export const grandfatherFeatures: Features = {
workspaces: true,
vanityDomain: true,
};
export const suspendedFeatures: Features = {
workspaces: true,
vanityDomain: true,
readOnlyDocs: true,
// clamp down on new docs/workspaces/shares
maxDocsPerOrg: 0,
maxSharesPerDoc: 0,
maxWorkspacesPerOrg: 0,
};
/**
* Basic fields needed for products supported by Grist.
*/
export interface IProduct {
name: string;
features: Features;
}
/**
*
* Products are a bundle of enabled features. Most products in
* Grist correspond to products in stripe. The correspondence is
* established by a gristProduct metadata field on stripe plans.
*
* In addition, there are the following products in Grist that don't
* exist in stripe:
* - The product named 'Free'. This is a product used for organizations
* created prior to the billing system being set up.
* - The product named 'stub'. This is product assigned to new
* organizations that should not be usable until a paid plan
* is set up for them.
*
* TODO: change capitalization of name of grandfather product.
*
*/
const PRODUCTS: IProduct[] = [
// This is a product for grandfathered accounts/orgs.
{
name: 'Free',
features: grandfatherFeatures,
},
// This is a product for newly created accounts/orgs.
{
name: 'stub',
features: {},
},
// These are products set up in stripe.
{
name: 'starter',
features: starterFeatures,
},
{
name: 'professional', // deprecated, can be removed once no longer referred to in stripe.
features: teamFeatures,
},
{
name: 'team',
features: teamFeatures,
},
// This is a product for a team site that is no longer in good standing, but isn't yet
// to be removed / deactivated entirely.
{
name: 'suspended',
features: suspendedFeatures,
},
];
/**
* Get names of products for different situations.
*/
export function getDefaultProductNames() {
return {
personal: 'starter', // Personal site start off on a functional plan.
teamInitial: 'stub', // Team site starts off on a limited plan, requiring subscription.
team: 'team', // Functional team site
};
}
/**
* A Grist product. Corresponds to a set of enabled features and a choice of limits.
*/
@Entity({name: 'products'})
export class Product extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@Column()
public name: string;
@Column({type: nativeValues.jsonEntityType})
public features: Features;
}
/**
* Make sure the products defined for the current stripe setup are
* in the database and up to date. Other products in the database
* are untouched.
*
* If `apply` is set, the products are changed in the db, otherwise
* the are left unchanged. A summary of affected products is returned.
*/
export async function synchronizeProducts(connection: Connection, apply: boolean): Promise<string[]> {
try {
await connection.query('select name, features, stripe_product_id from products limit 1');
} catch (e) {
// No usable products table, do not try to synchronize.
return [];
}
const changingProducts: string[] = [];
await connection.transaction(async transaction => {
const desiredProducts = new Map(PRODUCTS.map(p => [p.name, p]));
const existingProducts = new Map((await transaction.find(Product))
.map(p => [p.name, p]));
for (const product of desiredProducts.values()) {
if (existingProducts.has(product.name)) {
const p = existingProducts.get(product.name)!;
try {
assert.deepStrictEqual(p.features, product.features);
} catch (e) {
if (apply) {
p.features = product.features;
await transaction.save(p);
}
changingProducts.push(p.name);
}
} else {
if (apply) {
const p = new Product();
p.name = product.name;
p.features = product.features;
await transaction.save(p);
}
changingProducts.push(product.name);
}
}
});
return changingProducts;
}

@ -0,0 +1,46 @@
import {BaseEntity, Column} from "typeorm";
import {ApiError} from 'app/common/ApiError';
import {CommonProperties} from "app/common/UserAPI";
export class Resource extends BaseEntity {
@Column()
public name: string;
@Column({name: 'created_at', default: () => "CURRENT_TIMESTAMP"})
public createdAt: Date;
@Column({name: 'updated_at', default: () => "CURRENT_TIMESTAMP"})
public updatedAt: Date;
// a computed column which, when present, means the entity should be filtered out
// of results.
@Column({name: 'filtered_out', type: 'boolean', select: false, insert: false})
public filteredOut?: boolean;
public updateFromProperties(props: Partial<CommonProperties>) {
if (props.createdAt) { this.createdAt = _propertyToDate(props.createdAt); }
if (props.updatedAt) {
this.updatedAt = _propertyToDate(props.updatedAt);
} else {
this.updatedAt = new Date();
}
if (props.name) { this.name = props.name; }
}
protected checkProperties(props: any, keys: string[]): props is Partial<CommonProperties> {
for (const key of Object.keys(props)) {
if (!keys.includes(key)) {
throw new ApiError(`unrecognized property ${key}`, 400);
}
}
return true;
}
}
// Ensure iso-string-or-date value is converted to a date.
function _propertyToDate(d: string|Date): Date {
if (typeof(d) === 'string') {
return new Date(d);
}
return d;
}

@ -0,0 +1,54 @@
import {BaseEntity, Column, Entity, JoinTable, ManyToMany, OneToMany, OneToOne,
PrimaryGeneratedColumn} from "typeorm";
import {Group} from "./Group";
import {Login} from "./Login";
import {Organization} from "./Organization";
@Entity({name: 'users'})
export class User extends BaseEntity {
@PrimaryGeneratedColumn()
public id: number;
@Column()
public name: string;
@Column({name: 'api_key', type: String, nullable: true})
// Found how to make a type nullable in this discussion: https://github.com/typeorm/typeorm/issues/2567
// todo: adds constraint for api_key not to equal ''
public apiKey: string | null;
@Column({name: 'picture', type: String, nullable: true})
public picture: string | null;
@Column({name: 'first_login_at', type: Date, nullable: true})
public firstLoginAt: Date | null;
@OneToOne(type => Organization, organization => organization.owner)
public personalOrg: Organization;
@OneToMany(type => Login, login => login.user)
public logins: Login[];
@ManyToMany(type => Group)
@JoinTable({
name: 'group_users',
joinColumn: {name: 'user_id'},
inverseJoinColumn: {name: 'group_id'}
})
public groups: Group[];
@Column({name: 'is_first_time_user', default: false})
public isFirstTimeUser: boolean;
/**
* Get user's email. Returns undefined if logins has not been joined, or no login
* is available
*/
public get loginEmail(): string|undefined {
const login = this.logins && this.logins[0];
if (!login) { return undefined; }
return login.email;
}
}

@ -0,0 +1,49 @@
import {Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn} from "typeorm";
import {WorkspaceProperties, workspacePropertyKeys} from "app/common/UserAPI";
import {nativeValues} from 'app/gen-server/lib/values';
import {AclRuleWs} from "./AclRule";
import {Document} from "./Document";
import {Organization} from "./Organization";
import {Resource} from "./Resource";
@Entity({name: 'workspaces'})
export class Workspace extends Resource {
@PrimaryGeneratedColumn()
public id: number;
@ManyToOne(type => Organization)
@JoinColumn({name: 'org_id'})
public org: Organization;
@OneToMany(type => Document, document => document.workspace)
public docs: Document[];
@OneToMany(type => AclRuleWs, aclRule => aclRule.workspace)
public aclRules: AclRuleWs[];
// Property that may be returned when the workspace is fetched to indicate the access the
// fetching user has on the workspace, i.e. 'owners', 'editors', 'viewers'
public access: string;
// A computed column that is true if the workspace is a support workspace.
@Column({name: 'support', type: 'boolean', insert: false, select: false})
public isSupportWorkspace?: boolean;
// a computed column with permissions.
// {insert: false} makes sure typeorm doesn't try to put values into such
// a column when creating workspaces.
@Column({name: 'permissions', type: 'text', select: false, insert: false})
public permissions?: any;
@Column({name: 'removed_at', type: nativeValues.dateTimeType, nullable: true})
public removedAt: Date|null;
public checkProperties(props: any): props is Partial<WorkspaceProperties> {
return super.checkProperties(props, workspacePropertyKeys);
}
public updateFromProperties(props: Partial<WorkspaceProperties>) {
super.updateFromProperties(props);
}
}

@ -0,0 +1,103 @@
import * as express from "express";
import fetch, { RequestInit } from 'node-fetch';
import { ApiError } from 'app/common/ApiError';
import { removeTrailingSlash } from 'app/common/gutil';
import { HomeDBManager } from "app/gen-server/lib/HomeDBManager";
import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer';
import { IDocWorkerMap } from "app/server/lib/DocWorkerMap";
import { expressWrap } from "app/server/lib/expressWrap";
import { getAssignmentId } from "app/server/lib/idUtils";
/**
* Forwards all /api/docs/:docId/tables requests to the doc worker handling the :docId document. Makes
* sure the user has at least view access to the document otherwise rejects the request. For
* performance reason we stream the body directly from the request, which requires that no-one reads
* the req before, in particular you should register DocApiForwarder before bodyParser.
*
* Use:
* const home = new ApiServer(false);
* const docApiForwarder = new DocApiForwarder(getDocWorkerMap(), home);
* app.use(docApiForwarder.getMiddleware());
*
* Note that it expects userId, and jsonErrorHandler middleware to be set up outside
* to apply to these routes.
*/
export class DocApiForwarder {
constructor(private _docWorkerMap: IDocWorkerMap, private _dbManager: HomeDBManager) {
}
public addEndpoints(app: express.Application) {
// Middleware to forward a request about an existing document that user has access to.
// We do not check whether the document has been soft-deleted; that will be checked by
// the worker if needed.
const withDoc = expressWrap(this._forwardToDocWorker.bind(this, true));
// Middleware to forward a request without a pre-existing document (for imports/uploads).
const withoutDoc = expressWrap(this._forwardToDocWorker.bind(this, false));
app.use('/api/docs/:docId/tables', withDoc);
app.use('/api/docs/:docId/force-reload', withDoc);
app.use('/api/docs/:docId/remove', withDoc);
app.delete('/api/docs/:docId', withDoc);
app.use('/api/docs/:docId/download', withDoc);
app.use('/api/docs/:docId/apply', withDoc);
app.use('/api/docs/:docId/attachments', withDoc);
app.use('/api/docs/:docId/snapshots', withDoc);
app.use('/api/docs/:docId/replace', withDoc);
app.use('/api/docs/:docId/flush', withDoc);
app.use('/api/docs/:docId/states', withDoc);
app.use('/api/docs/:docId/compare', withDoc);
app.use('^/api/docs$', withoutDoc);
}
private async _forwardToDocWorker(withDocId: boolean, req: express.Request, res: express.Response): Promise<void> {
let docId: string|null = null;
if (withDocId) {
const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, req.params.docId);
assertAccess('viewers', docAuth, {allowRemoved: true});
docId = docAuth.docId;
}
// Use the docId for worker assignment, rather than req.params.docId, which could be a urlId.
const assignmentId = getAssignmentId(this._docWorkerMap, docId === null ? 'import' : docId);
if (!this._docWorkerMap) {
throw new ApiError('no worker map', 404);
}
const docStatus = await this._docWorkerMap.assignDocWorker(assignmentId);
// Construct new url by keeping only origin and path prefixes of `docWorker.internalUrl`,
// and otherwise reflecting fully the original url (remaining path, and query params).
const docWorkerUrl = new URL(docStatus.docWorker.internalUrl);
const url = new URL(req.originalUrl, docWorkerUrl.origin);
url.pathname = removeTrailingSlash(docWorkerUrl.pathname) + url.pathname;
const headers: {[key: string]: string} = {
...getTransitiveHeaders(req),
'Content-Type': req.get('Content-Type') || 'application/json',
};
for (const key of ['X-Sort', 'X-Limit']) {
const hdr = req.get(key);
if (hdr) { headers[key] = hdr; }
}
const options: RequestInit = {
method: req.method,
headers,
};
if (['POST', 'PATCH'].includes(req.method)) {
// uses `req` as a stream
options.body = req;
}
const docWorkerRes = await fetch(url.href, options);
res.status(docWorkerRes.status);
for (const key of ['content-type', 'content-disposition', 'cache-control']) {
const value = docWorkerRes.headers.get(key);
if (value) { res.set(key, value); }
}
return new Promise<void>((resolve, reject) => {
docWorkerRes.body.on('error', reject);
res.on('error', reject);
res.on('finish', resolve);
docWorkerRes.body.pipe(res);
});
}
}

@ -0,0 +1,440 @@
import {MapWithTTL} from 'app/common/AsyncCreate';
import * as version from 'app/common/version';
import {DocStatus, DocWorkerInfo, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import * as log from 'app/server/lib/log';
import {checkPermitKey, formatPermitKey, Permit} from 'app/server/lib/Permit';
import {promisifyAll} from 'bluebird';
import mapValues = require('lodash/mapValues');
import {createClient, Multi, RedisClient} from 'redis';
import * as Redlock from 'redlock';
import * as uuidv4 from 'uuid/v4';
promisifyAll(RedisClient.prototype);
promisifyAll(Multi.prototype);
// Max time for which we will hold a lock, by default. In milliseconds.
const LOCK_TIMEOUT = 3000;
// How long do checksums stored in redis last. In milliseconds.
// Should be long enough to allow S3 to reach consistency with very high probability.
// Consistency failures shorter than this interval will be detectable, failures longer
// than this interval will not be detectable.
const CHECKSUM_TTL_MSEC = 24 * 60 * 60 * 1000; // 24 hours
// How long do permits stored in redis last, in milliseconds.
const PERMIT_TTL_MSEC = 1 * 60 * 1000; // 1 minute
class DummyDocWorkerMap implements IDocWorkerMap {
private _worker?: DocWorkerInfo;
private _available: boolean = false;
private _permits = new MapWithTTL<string, string>(PERMIT_TTL_MSEC);
private _elections = new MapWithTTL<string, string>(1); // default ttl never used
public async getDocWorker(docId: string) {
if (!this._worker) { throw new Error('no workers'); }
return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
}
public async assignDocWorker(docId: string) {
if (!this._worker || !this._available) { throw new Error('no workers'); }
return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
}
public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
if (!this._worker || !this._available) { throw new Error('no workers'); }
if (this._worker.id !== workerId) { throw new Error('worker not known'); }
return {docMD5: 'unknown', docWorker: this._worker, isActive: true};
}
public async updateDocStatus(docId: string, checksum: string) {
// nothing to do
}
public async addWorker(info: DocWorkerInfo): Promise<void> {
this._worker = info;
}
public async removeWorker(workerId: string): Promise<void> {
this._worker = undefined;
}
public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
this._available = available;
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
// nothing to do
}
public async getAssignments(workerId: string): Promise<string[]> {
return [];
}
public async setPermit(permit: Permit): Promise<string> {
const key = formatPermitKey(uuidv4());
this._permits.set(key, JSON.stringify(permit));
return key;
}
public async getPermit(key: string): Promise<Permit> {
const result = this._permits.get(key);
return result ? JSON.parse(result) : null;
}
public async removePermit(key: string): Promise<void> {
this._permits.delete(key);
}
public async close(): Promise<void> {
this._permits.clear();
this._elections.clear();
}
public async getElection(name: string, durationInMs: number): Promise<string|null> {
if (this._elections.get(name)) { return null; }
const key = uuidv4();
this._elections.setWithCustomTTL(name, key, durationInMs);
return key;
}
public async removeElection(name: string, electionKey: string): Promise<void> {
if (this._elections.get(name) === electionKey) {
this._elections.delete(name);
}
}
}
/**
* Manage the relationship between document and workers. Backed by Redis.
* Can also assign workers to "groups" for serving particular documents.
* Keys used:
* workers - the set of known active workers, identified by workerId
* workers-available - the set of workers available for assignment (a subset of the workers set)
* workers-available-{group} - the set of workers available for a given group
* worker-{workerId} - a hash of contact information for a worker
* worker-{workerId}-docs - a set of docs assigned to a worker, identified by docId
* worker-{workerId}-group - if set, marks the worker as serving a particular group
* doc-${docId} - a hash containing (JSON serialized) DocStatus fields, other than docMD5.
* doc-${docId}-checksum - the docs docMD5, or 'null' if docMD5 is null
* doc-${docId}-group - if set, marks the doc as to be served by workers in a given group
* workers-lock - a lock used when working with the list of workers
* groups - a hash from groupIds (arbitrary strings) to desired number of workers in group
* elections-${deployment} - a hash, from groupId to a (serialized json) list of worker ids
*
* Assignments of documents to workers can end abruptly at any time. Clients
* should be prepared to retry if a worker is not responding or denies that a document
* is assigned to it.
*
* If the groups key is set, workers assign themselves to groupIds to
* fill the counts specified in groups (in order of groupIds), and
* once those are exhausted, get assigned to the special group
* "default".
*/
export class DocWorkerMap implements IDocWorkerMap {
private _client: RedisClient;
private _redlock: Redlock;
// Optional deploymentKey argument supplies a key unique to the deployment (this is important
// for maintaining groups across redeployments only)
constructor(_clients?: RedisClient[], private _deploymentKey?: string, private _options?: {
permitMsec?: number
}) {
this._deploymentKey = this._deploymentKey || version.version;
_clients = _clients || [createClient(process.env.REDIS_URL)];
this._redlock = new Redlock(_clients);
this._client = _clients[0]!;
}
public async addWorker(info: DocWorkerInfo): Promise<void> {
log.info(`DocWorkerMap.addWorker ${info.id}`);
const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
try {
// Make a worker-{workerId} key with contact info, then add this worker to available set.
await this._client.hmsetAsync(`worker-${info.id}`, info);
await this._client.saddAsync('workers', info.id);
// Figure out if worker should belong to a group
const groups = await this._client.hgetallAsync('groups');
if (groups) {
const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`) || {};
for (const group of Object.keys(groups).sort()) {
const count = parseInt(groups[group], 10) || 0;
if (count < 1) { continue; }
const elected: string[] = JSON.parse(elections[group] || '[]');
if (elected.length >= count) { continue; }
elected.push(info.id);
await this._client.setAsync(`worker-${info.id}-group`, group);
await this._client.hsetAsync(`elections-${this._deploymentKey}`, group, JSON.stringify(elected));
break;
}
}
} finally {
await lock.unlock();
}
}
public async removeWorker(workerId: string): Promise<void> {
log.info(`DocWorkerMap.removeWorker ${workerId}`);
const lock = await this._redlock.lock('workers-lock', LOCK_TIMEOUT);
try {
// Drop out of available set first.
await this._client.sremAsync('workers-available', workerId);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
await this._client.sremAsync(`workers-available-${group}`, workerId);
// At this point, this worker should no longer be receiving new doc assignments, though
// clients may still be directed to the worker.
// If we were elected for anything, back out.
const elections = await this._client.hgetallAsync(`elections-${this._deploymentKey}`);
if (elections) {
if (group in elections) {
const elected: string[] = JSON.parse(elections[group]);
const newElected = elected.filter(worker => worker !== workerId);
if (elected.length !== newElected.length) {
if (newElected.length > 0) {
await this._client.hsetAsync(`elections-${this._deploymentKey}`, group,
JSON.stringify(newElected));
} else {
await this._client.hdelAsync(`elections-${this._deploymentKey}`, group);
delete elections[group];
}
}
// We're the last one involved in elections - remove the key entirely.
if (Object.keys(elected).length === 0) {
await this._client.delAsync(`elections-${this._deploymentKey}`);
}
}
}
// Now, we start removing the assignments.
const assignments = await this._client.smembersAsync(`worker-${workerId}-docs`);
if (assignments) {
const op = this._client.multi();
for (const doc of assignments) { op.del(`doc-${doc}`); }
await op.execAsync();
}
// Now remove worker-{workerId}* keys.
await this._client.delAsync(`worker-${workerId}-docs`);
await this._client.delAsync(`worker-${workerId}-group`);
await this._client.delAsync(`worker-${workerId}`);
// Forget about this worker completely.
await this._client.sremAsync('workers', workerId);
} finally {
await lock.unlock();
}
}
public async setWorkerAvailability(workerId: string, available: boolean): Promise<void> {
log.info(`DocWorkerMap.setWorkerAvailability ${workerId} ${available}`);
const group = await this._client.getAsync(`worker-${workerId}-group`) || 'default';
if (available) {
await this._client.saddAsync(`workers-available-${group}`, workerId);
await this._client.saddAsync('workers-available', workerId);
} else {
await this._client.sremAsync('workers-available', workerId);
await this._client.sremAsync(`workers-available-${group}`, workerId);
}
}
public async releaseAssignment(workerId: string, docId: string): Promise<void> {
const op = this._client.multi();
op.del(`doc-${docId}`);
op.srem(`worker-${workerId}-docs`, docId);
await op.execAsync();
}
public async getAssignments(workerId: string): Promise<string[]> {
return this._client.smembersAsync(`worker-${workerId}-docs`);
}
/**
* Defined by IDocWorkerMap.
*
* Looks up which DocWorker is responsible for this docId.
* Responsibility could change at any time after this call, so it
* should be treated as a hint, and clients should be prepared to be
* refused and need to retry.
*/
public async getDocWorker(docId: string): Promise<DocStatus|null> {
// Fetch the various elements that go into making a DocStatus
const props = await this._client.multi()
.hgetall(`doc-${docId}`)
.get(`doc-${docId}-checksum`)
.execAsync() as [{[key: string]: any}|null, string|null]|null;
if (!props) { return null; }
// If there is no worker, return null. An alternative would be to modify
// DocStatus so that it is possible for it to not have a worker assignment.
if (!props[0]) { return null; }
// Fields are JSON encoded since redis cannot store them directly.
const doc = mapValues(props[0], (val) => JSON.parse(val));
// Redis cannot store a null value, so we encode it as 'null', which does
// not match any possible MD5.
doc.docMD5 = props[1] === 'null' ? null : props[1];
// Ok, we have a valid DocStatus at this point.
return doc as DocStatus;
}
/**
*
* Defined by IDocWorkerMap.
*
* Assigns a DocWorker to this docId if one is not yet assigned.
* Note that the assignment could be unmade at any time after this
* call if the worker dies, is brought down, or for other potential
* reasons in the future such as migration of individual documents
* between workers.
*
* A preferred doc worker can be specified, which will be assigned
* if no assignment is already made.
*
*/
public async assignDocWorker(docId: string, workerId?: string): Promise<DocStatus> {
// Check if a DocWorker is already assigned; if so return result immediately
// without locking.
let docStatus = await this.getDocWorker(docId);
if (docStatus) { return docStatus; }
// No assignment yet, so let's lock and set an assignment up.
const lock = await this._redlock.lock(`workers-lock`, LOCK_TIMEOUT);
try {
// Now that we've locked, recheck that the worker hasn't been reassigned
// in the meantime. Return immediately if it has.
docStatus = await this.getDocWorker(docId);
if (docStatus) { return docStatus; }
if (!workerId) {
// Check if document has a preferred worker group set.
const group = await this._client.getAsync(`doc-${docId}-group`) || 'default';
// Let's start off by assigning documents to available workers randomly.
// TODO: use a smarter algorithm.
workerId = await this._client.srandmemberAsync(`workers-available-${group}`) || undefined;
if (!workerId) {
// No workers available in the desired worker group. Rather than refusing to
// open the document, we fall back on assigning a worker from any of the workers
// available, regardless of grouping.
// This limits the impact of operational misconfiguration (bad redis setup,
// or not starting enough workers). It has the downside of potentially disguising
// problems, so we log a warning.
log.warn(`DocWorkerMap.assignDocWorker ${docId} found no workers for group ${group}`);
workerId = await this._client.srandmemberAsync('workers-available') || undefined;
}
if (!workerId) { throw new Error('no doc workers available'); }
} else {
if (!await this._client.sismemberAsync('workers-available', workerId)) {
throw new Error(`worker ${workerId} not known or not available`);
}
}
// Look up how to contact the worker.
const docWorker = await this._client.hgetallAsync(`worker-${workerId}`) as DocWorkerInfo|null;
if (!docWorker) { throw new Error('no doc worker contact info available'); }
// We can now construct a DocStatus.
const newDocStatus = {docMD5: null, docWorker, isActive: true};
// We add the assignment to worker-{workerId}-docs and save doc-{docId}.
const result = await this._client.multi()
.sadd(`worker-${workerId}-docs`, docId)
.hmset(`doc-${docId}`, {
docWorker: JSON.stringify(docWorker), // redis can't store nested objects, strings only
isActive: JSON.stringify(true) // redis can't store booleans, strings only
})
.setex(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, 'null')
.execAsync();
if (!result) { throw new Error('failed to store new assignment'); }
return newDocStatus;
} finally {
await lock.unlock();
}
}
/**
*
* Defined by IDocWorkerMap.
*
* Assigns a specific DocWorker to this docId if one is not yet assigned.
*
*/
public async getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus> {
return this.assignDocWorker(docId, workerId);
}
public async updateDocStatus(docId: string, checksum: string): Promise<void> {
await this._client.setexAsync(`doc-${docId}-checksum`, CHECKSUM_TTL_MSEC / 1000.0, checksum);
}
public async setPermit(permit: Permit): Promise<string> {
const key = formatPermitKey(uuidv4());
const duration = (this._options && this._options.permitMsec) || PERMIT_TTL_MSEC;
// seems like only integer seconds are supported?
await this._client.setexAsync(key, Math.ceil(duration / 1000.0),
JSON.stringify(permit));
return key;
}
public async getPermit(key: string): Promise<Permit|null> {
if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
const result = await this._client.getAsync(key);
return result && JSON.parse(result);
}
public async removePermit(key: string): Promise<void> {
if (!checkPermitKey(key)) { throw new Error('permit could not be read'); }
await this._client.delAsync(key);
}
public async close(): Promise<void> {
// nothing to do
}
public async getElection(name: string, durationInMs: number): Promise<string|null> {
// Could use "set nx" for election, but redis docs don't encourage that any more,
// favoring redlock:
// https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode
const redisKey = `nomination-${name}`;
const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
try {
if (await this._client.getAsync(redisKey) !== null) { return null; }
const electionKey = uuidv4();
// seems like only integer seconds are supported?
await this._client.setexAsync(redisKey, Math.ceil(durationInMs / 1000.0), electionKey);
return electionKey;
} finally {
await lock.unlock();
}
}
public async removeElection(name: string, electionKey: string): Promise<void> {
const redisKey = `nomination-${name}`;
const lock = await this._redlock.lock(`${redisKey}-lock`, LOCK_TIMEOUT);
try {
const current = await this._client.getAsync(redisKey);
if (current === electionKey) {
await this._client.delAsync(redisKey);
} else if (current !== null) {
throw new Error('could not remove election');
}
} finally {
await lock.unlock();
}
}
}
// If we don't have redis available and use a DummyDocWorker, it should be a singleton.
let dummyDocWorkerMap: DummyDocWorkerMap|null = null;
export function getDocWorkerMap(): IDocWorkerMap {
if (process.env.REDIS_URL) {
return new DocWorkerMap();
} else {
dummyDocWorkerMap = dummyDocWorkerMap || new DummyDocWorkerMap();
return dummyDocWorkerMap;
}
}

File diff suppressed because it is too large Load Diff

@ -0,0 +1,21 @@
export enum Permissions {
NONE = 0x0,
// Note that the view permission bit provides view access ONLY to the resource to which
// the aclRule belongs - it does not allow listing that resource's children. A resource's
// children may only be listed if those children also have the view permission set.
VIEW = 0x1,
UPDATE = 0x2,
ADD = 0x4,
// Note that the remove permission bit provides remove access to a resource AND all of
// its child resources/ACLs
REMOVE = 0x8,
SCHEMA_EDIT = 0x10,
ACL_EDIT = 0x20,
EDITOR = VIEW | UPDATE | ADD | REMOVE, // tslint:disable-line:no-bitwise
ADMIN = EDITOR | SCHEMA_EDIT, // tslint:disable-line:no-bitwise
OWNER = ADMIN | ACL_EDIT, // tslint:disable-line:no-bitwise
// A virtual permission bit signifying that the general public has some access to
// the resource via ACLs involving the everyone@ user.
PUBLIC = 0x80
}

@ -0,0 +1,196 @@
// This contains two TypeORM patches.
// Patch 1:
// TypeORM Sqlite driver does not support using transactions in async code, if it is possible
// for two transactions to get called (one of the whole point of transactions). This
// patch adds support for that, based on a monkey patch published in:
// https://gist.github.com/keenondrums/556f8c61d752eff730841170cd2bc3f1
// Explanation at https://github.com/typeorm/typeorm/issues/1884#issuecomment-380767213
// Patch 2:
// TypeORM parameters are global, and collisions in setting them are not detected.
// We add a patch to throw an exception if a parameter value is ever set and then
// changed during construction of a query.
import * as sqlite3 from '@gristlabs/sqlite3';
import isEqual = require('lodash/isEqual');
import {EntityManager, QueryRunner} from 'typeorm';
import {SqliteDriver} from 'typeorm/driver/sqlite/SqliteDriver';
import {SqliteQueryRunner} from 'typeorm/driver/sqlite/SqliteQueryRunner';
import {
QueryRunnerProviderAlreadyReleasedError
} from 'typeorm/error/QueryRunnerProviderAlreadyReleasedError';
import {QueryBuilder} from 'typeorm/query-builder/QueryBuilder';
/**********************
* Patch 1
**********************/
type Releaser = () => void;
type Worker<T> = () => Promise<T>|T;
interface MutexInterface {
acquire(): Promise<Releaser>;
runExclusive<T>(callback: Worker<T>): Promise<T>;
isLocked(): boolean;
}
class Mutex implements MutexInterface {
private _queue: Array<(release: Releaser) => void> = [];
private _pending = false;
public isLocked(): boolean {
return this._pending;
}
public acquire(): Promise<Releaser> {
const ticket = new Promise<Releaser>(resolve => this._queue.push(resolve));
if (!this._pending) {
this._dispatchNext();
}
return ticket;
}
public runExclusive<T>(callback: Worker<T>): Promise<T> {
return this
.acquire()
.then(release => {
let result: T|Promise<T>;
try {
result = callback();
} catch (e) {
release();
throw(e);
}
return Promise
.resolve(result)
.then(
(x: T) => (release(), x),
e => {
release();
throw e;
}
);
}
);
}
private _dispatchNext(): void {
if (this._queue.length > 0) {
this._pending = true;
this._queue.shift()!(this._dispatchNext.bind(this));
} else {
this._pending = false;
}
}
}
// A singleton mutex for all sqlite transactions.
const mutex = new Mutex();
class SqliteQueryRunnerPatched extends SqliteQueryRunner {
private _releaseMutex: Releaser | null;
public async startTransaction(level?: any): Promise<void> {
this._releaseMutex = await mutex.acquire();
return super.startTransaction(level);
}
public async commitTransaction(): Promise<void> {
if (!this._releaseMutex) {
throw new Error('SqliteQueryRunnerPatched.commitTransaction -> mutex releaser unknown');
}
await super.commitTransaction();
this._releaseMutex();
this._releaseMutex = null;
}
public async rollbackTransaction(): Promise<void> {
if (!this._releaseMutex) {
throw new Error('SqliteQueryRunnerPatched.rollbackTransaction -> mutex releaser unknown');
}
await super.rollbackTransaction();
this._releaseMutex();
this._releaseMutex = null;
}
public async connect(): Promise<any> {
if (!this.isTransactionActive) {
const release = await mutex.acquire();
release();
}
return super.connect();
}
}
class SqliteDriverPatched extends SqliteDriver {
public createQueryRunner(): QueryRunner {
if (!this.queryRunner) {
this.queryRunner = new SqliteQueryRunnerPatched(this);
}
return this.queryRunner;
}
protected loadDependencies(): void {
// Use our own sqlite3 module, which is a fork of the original.
this.sqlite = sqlite3;
}
}
// Patch the underlying SqliteDriver, since it's impossible to convince typeorm to use only our
// patched classes. (Previously we patched DriverFactory and Connection, but those would still
// create an unpatched SqliteDriver and then overwrite it.)
SqliteDriver.prototype.createQueryRunner = SqliteDriverPatched.prototype.createQueryRunner;
(SqliteDriver.prototype as any).loadDependencies = (SqliteDriverPatched.prototype as any).loadDependencies;
export function applyPatch() {
// tslint: disable-next-line
EntityManager.prototype.transaction = async function <T>(arg1: any, arg2?: any): Promise<T> {
if (this.queryRunner && this.queryRunner.isReleased) {
throw new QueryRunnerProviderAlreadyReleasedError();
}
if (this.queryRunner && this.queryRunner.isTransactionActive) {
throw new Error(`Cannot start transaction because its already started`);
}
const queryRunner = this.connection.createQueryRunner();
const runInTransaction = typeof arg1 === "function" ? arg1 : arg2;
try {
await queryRunner.startTransaction();
const result = await runInTransaction(queryRunner.manager);
await queryRunner.commitTransaction();
return result;
} catch (err) {
try {
// we throw original error even if rollback thrown an error
await queryRunner.rollbackTransaction();
// tslint: disable-next-line
} catch (rollbackError) {
// tslint: disable-next-line
}
throw err;
} finally {
await queryRunner.release();
}
};
}
/**********************
* Patch 2
**********************/
abstract class QueryBuilderPatched<T> extends QueryBuilder<T> {
public setParameter(key: string, value: any): this {
const prev = this.expressionMap.parameters[key];
if (prev !== undefined && !isEqual(prev, value)) {
throw new Error(`TypeORM parameter collision for key '${key}' ('${prev}' vs '${value}')`);
}
this.expressionMap.parameters[key] = value;
return this;
}
}
(QueryBuilder.prototype as any).setParameter = (QueryBuilderPatched.prototype as any).setParameter;

@ -0,0 +1,62 @@
import {Document} from 'app/gen-server/entity/Document';
import {Organization} from 'app/gen-server/entity/Organization';
import {User} from 'app/gen-server/entity/User';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import * as log from 'app/server/lib/log';
// Frequency of logging usage information. Not something we need
// to track with much granularity.
const USAGE_PERIOD_MS = 1 * 60 * 60 * 1000; // log every 1 hour
/**
* Occasionally log usage information - number of users, orgs,
* docs, etc.
*/
export class Usage {
private _interval: NodeJS.Timeout;
public constructor(private _dbManager: HomeDBManager) {
this._interval = setInterval(() => this.apply().catch(log.warn.bind(log)), USAGE_PERIOD_MS);
// Log once at beginning, in case we roll over servers faster than
// the logging period for an extended length of time,
// and to raise the visibility of this logging step so if it gets
// slow devs notice.
this.apply().catch(log.warn.bind(log));
}
public close() {
clearInterval(this._interval);
}
public async apply() {
const manager = this._dbManager.connection.manager;
// raw count of users
const userCount = await manager.count(User);
// users who have logged in at least once
const userWithLoginCount = await manager.createQueryBuilder()
.from(User, 'users')
.where('first_login_at is not null')
.getCount();
// raw count of organizations (excluding personal orgs)
const orgCount = await manager.createQueryBuilder()
.from(Organization, 'orgs')
.where('owner_id is null')
.getCount();
// organizations with subscriptions that are in a non-terminated state
const orgInGoodStandingCount = await manager.createQueryBuilder()
.from(Organization, 'orgs')
.leftJoin('orgs.billingAccount', 'billing_accounts')
.where('owner_id is null')
.andWhere('billing_accounts.in_good_standing = true')
.getCount();
// raw count of documents
const docCount = await manager.count(Document);
log.rawInfo('activity', {
docCount,
orgCount,
orgInGoodStandingCount,
userCount,
userWithLoginCount,
});
}
}

@ -0,0 +1,209 @@
import {EntityManager} from "typeorm";
import * as roles from 'app/common/roles';
import {Document} from "app/gen-server/entity/Document";
import {Group} from "app/gen-server/entity/Group";
import {Organization} from "app/gen-server/entity/Organization";
import {Workspace} from "app/gen-server/entity/Workspace";
import pick = require('lodash/pick');
/**
*
* Remove the given user from the given org and every resource inside the org.
* If the user being removed is an owner of any resources in the org, the caller replaces
* them as the owner. This is to prevent complete loss of access to any resource.
*
* This method transforms ownership without regard to permissions. We all talked this
* over and decided this is what we wanted, but there's no denying it is funky and could
* be surprising.
* TODO: revisit user scrubbing when we can.
*
*/
export async function scrubUserFromOrg(
orgId: number,
removeUserId: number,
callerUserId: number,
manager: EntityManager
): Promise<void> {
await addMissingGuestMemberships(callerUserId, orgId, manager);
// This will be a list of all mentions of removeUser and callerUser in any resource
// within the org.
const mentions: Mention[] = [];
// Base query for all group_users related to these two users and this org.
const q = manager.createQueryBuilder()
.select('group_users.group_id, group_users.user_id')
.from('group_users', 'group_users')
.leftJoin(Group, 'groups', 'group_users.group_id = groups.id')
.addSelect('groups.name as name')
.leftJoin('groups.aclRule', 'acl_rules')
.where('(group_users.user_id = :removeUserId or group_users.user_id = :callerUserId)',
{removeUserId, callerUserId})
.andWhere('orgs.id = :orgId', {orgId});
// Pick out group_users related specifically to the org resource, in 'mentions' format
// (including resource id, a tag for the kind of resource, the group name, the user
// id, and the group id).
const orgs = q.clone()
.addSelect(`'org' as kind, orgs.id`)
.innerJoin(Organization, 'orgs', 'orgs.id = acl_rules.org_id');
mentions.push(...await orgs.getRawMany());
// Pick out mentions related to any workspace within the org.
const wss = q.clone()
.innerJoin(Workspace, 'workspaces', 'workspaces.id = acl_rules.workspace_id')
.addSelect(`'ws' as kind, workspaces.id`)
.innerJoin('workspaces.org', 'orgs');
mentions.push(...await wss.getRawMany());
// Pick out mentions related to any doc within the org.
const docs = q.clone()
.innerJoin(Document, 'docs', 'docs.id = acl_rules.doc_id')
.addSelect(`'doc' as kind, docs.id`)
.innerJoin('docs.workspace', 'workspaces')
.innerJoin('workspaces.org', 'orgs');
mentions.push(...await docs.getRawMany());
// Prepare to add and delete group_users.
const toDelete: Mention[] = [];
const toAdd: Mention[] = [];
// Now index the mentions by whether they are for the removeUser or the callerUser,
// and the resource they apply to.
const removeUserMentions = new Map<MentionKey, Mention>();
const callerUserMentions = new Map<MentionKey, Mention>();
for (const mention of mentions) {
const isGuest = mention.name === roles.GUEST;
if (mention.user_id === removeUserId) {
// We can safely remove any guest roles for the removeUser without any
// further inspection.
if (isGuest) { toDelete.push(mention); continue; }
removeUserMentions.set(getMentionKey(mention), mention);
} else {
if (isGuest) { continue; }
callerUserMentions.set(getMentionKey(mention), mention);
}
}
// Now iterate across the mentions of removeUser, and see what we need to do
// for each of them.
for (const [key, removeUserMention] of removeUserMentions) {
toDelete.push(removeUserMention);
if (removeUserMention.name !== roles.OWNER) {
// Nothing fancy needed for cases where the removeUser is not the owner.
// Just discard those.
continue;
}
// The removeUser was a direct owner on this resource, but the callerUser was
// not. We set the callerUser as a direct owner on this resource, to preserve
// access to it.
// TODO: the callerUser might inherit sufficient access, in which case this
// step is unnecessary and could be skipped. I believe it does no harm though.
const callerUserMention = callerUserMentions.get(key);
if (callerUserMention && callerUserMention.name === roles.OWNER) { continue; }
if (callerUserMention) { toDelete.push(callerUserMention); }
toAdd.push({...removeUserMention, user_id: callerUserId});
}
if (toDelete.length > 0) {
await manager.createQueryBuilder()
.delete()
.from('group_users')
.whereInIds(toDelete.map(m => pick(m, ['user_id', 'group_id'])))
.execute();
}
if (toAdd.length > 0) {
await manager.createQueryBuilder()
.insert()
.into('group_users')
.values(toAdd.map(m => pick(m, ['user_id', 'group_id'])))
.execute();
}
// TODO: At this point, we've removed removeUserId from every mention in group_users.
// The user may still be mentioned in billing_account_managers. If the billing_account
// is linked to just this single organization, perhaps it would make sense to remove
// the user there, if the callerUser is themselves a billing account manager?
await addMissingGuestMemberships(callerUserId, orgId, manager);
}
/**
* Adds specified user to any guest groups for the resources of an org where the
* user needs to be and is not already.
*/
export async function addMissingGuestMemberships(userId: number, orgId: number,
manager: EntityManager) {
// For workspaces:
// User should be in guest group if mentioned in a doc within that workspace.
let groupUsers = await manager.createQueryBuilder()
.select('workspace_groups.id as group_id, cast(:userId as int) as user_id')
.setParameter('userId', userId)
.from(Workspace, 'workspaces')
.where('workspaces.org_id = :orgId', {orgId})
.innerJoin('workspaces.docs', 'docs')
.innerJoin('docs.aclRules', 'doc_acl_rules')
.innerJoin('doc_acl_rules.group', 'doc_groups')
.innerJoin('doc_groups.memberUsers', 'doc_group_users')
.andWhere('doc_group_users.id = :userId', {userId})
.leftJoin('workspaces.aclRules', 'workspace_acl_rules')
.leftJoin('workspace_acl_rules.group', 'workspace_groups')
.leftJoin('group_users', 'workspace_group_users',
'workspace_group_users.group_id = workspace_groups.id and ' +
'workspace_group_users.user_id = :userId')
.andWhere('workspace_groups.name = :guestName', {guestName: roles.GUEST})
.groupBy('workspaces.id, workspace_groups.id, workspace_group_users.user_id')
.having('workspace_group_users.user_id is null')
.getRawMany();
if (groupUsers.length > 0) {
await manager.createQueryBuilder()
.insert()
.into('group_users')
.values(groupUsers)
.execute();
}
// For org:
// User should be in guest group if mentioned in a workspace within that org.
groupUsers = await manager.createQueryBuilder()
.select('org_groups.id as group_id, cast(:userId as int) as user_id')
.setParameter('userId', userId)
.from(Organization, 'orgs')
.where('orgs.id = :orgId', {orgId})
.innerJoin('orgs.workspaces', 'workspaces')
.innerJoin('workspaces.aclRules', 'workspaces_acl_rules')
.innerJoin('workspaces_acl_rules.group', 'workspace_groups')
.innerJoin('workspace_groups.memberUsers', 'workspace_group_users')
.andWhere('workspace_group_users.id = :userId', {userId})
.leftJoin('orgs.aclRules', 'org_acl_rules')
.leftJoin('org_acl_rules.group', 'org_groups')
.leftJoin('group_users', 'org_group_users',
'org_group_users.group_id = org_groups.id and ' +
'org_group_users.user_id = :userId')
.andWhere('org_groups.name = :guestName', {guestName: roles.GUEST})
.groupBy('org_groups.id, org_group_users.user_id')
.having('org_group_users.user_id is null')
.getRawMany();
if (groupUsers.length > 0) {
await manager.createQueryBuilder()
.insert()
.into('group_users')
.values(groupUsers)
.execute();
}
// For doc:
// Guest groups are not used.
}
interface Mention {
id: string|number; // id of resource
kind: 'org'|'ws'|'doc'; // type of resource
user_id: number; // id of user in group
group_id: number; // id of group
name: string; // name of group
}
type MentionKey = string;
function getMentionKey(mention: Mention): MentionKey {
return `${mention.kind} ${mention.id}`;
}

@ -0,0 +1,36 @@
/**
* This smoothes over some awkward differences between TypeORM treatment of
* booleans and json in sqlite and postgres. Booleans and json work fine
* with each db, but have different levels of driver-level support.
*/
export interface NativeValues {
// Json columns are handled natively by the postgres driver, but for
// sqlite requires a typeorm wrapper (simple-json).
jsonEntityType: 'json' | 'simple-json';
jsonType: 'json' | 'varchar';
booleanType: 'boolean' | 'integer';
dateTimeType: 'timestamp with time zone' | 'datetime';
trueValue: boolean | number;
falseValue: boolean | number;
}
const sqliteNativeValues: NativeValues = {
jsonEntityType: 'simple-json',
jsonType: 'varchar',
booleanType: 'integer',
dateTimeType: 'datetime',
trueValue: 1,
falseValue: 0
};
const postgresNativeValues: NativeValues = {
jsonEntityType: 'json',
jsonType: 'json',
booleanType: 'boolean',
dateTimeType: 'timestamp with time zone',
trueValue: true,
falseValue: false
};
export const nativeValues = (process.env.TYPEORM_TYPE === 'postgres') ? postgresNativeValues : sqliteNativeValues;

@ -0,0 +1,304 @@
import {MigrationInterface, QueryRunner, Table} from "typeorm";
export class Initial1536634251710 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
// TypeORM doesn't currently help with types of created tables:
// https://github.com/typeorm/typeorm/issues/305
// so we need to do a little smoothing over postgres and sqlite.
const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
const datetime = sqlite ? "datetime" : "timestamp with time zone";
const now = "now()";
await queryRunner.createTable(new Table({
name: "users",
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: "name",
type: "varchar",
},
{
name: "api_key",
type: "varchar",
isNullable: true,
isUnique: true
}
]
}), false);
await queryRunner.createTable(new Table({
name: "orgs",
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: "name",
type: "varchar",
},
{
name: "domain",
type: "varchar",
isNullable: true,
},
{
name: "created_at",
type: datetime,
default: now
},
{
name: "updated_at",
type: datetime,
default: now
},
{
name: "owner_id",
type: "integer",
isNullable: true,
isUnique: true
}
],
foreignKeys: [
{
columnNames: ["owner_id"],
referencedColumnNames: ["id"],
referencedTableName: "users"
}
]
}), false);
await queryRunner.createTable(new Table({
name: "workspaces",
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: "name",
type: "varchar",
},
{
name: "created_at",
type: datetime,
default: now
},
{
name: "updated_at",
type: datetime,
default: now
},
{
name: "org_id",
type: "integer",
isNullable: true
}
],
foreignKeys: [
{
columnNames: ["org_id"],
referencedColumnNames: ["id"],
referencedTableName: "orgs"
}
]
}), false);
await queryRunner.createTable(new Table({
name: "docs",
columns: [
{
name: "id",
type: "varchar",
isPrimary: true
},
{
name: "name",
type: "varchar",
},
{
name: "created_at",
type: datetime,
default: now
},
{
name: "updated_at",
type: datetime,
default: now
},
{
name: "workspace_id",
type: "integer",
isNullable: true
}
],
foreignKeys: [
{
columnNames: ["workspace_id"],
referencedColumnNames: ["id"],
referencedTableName: "workspaces"
}
]
}), false);
await queryRunner.createTable(new Table({
name: "groups",
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: "name",
type: "varchar",
}
]
}), false);
await queryRunner.createTable(new Table({
name: "acl_rules",
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: "permissions",
type: "integer"
},
{
name: "type",
type: "varchar"
},
{
name: "workspace_id",
type: "integer",
isNullable: true
},
{
name: "org_id",
type: "integer",
isNullable: true
},
{
name: "doc_id",
type: "varchar",
isNullable: true
},
{
name: "group_id",
type: "integer",
isNullable: true
}
],
foreignKeys: [
{
columnNames: ["workspace_id"],
referencedColumnNames: ["id"],
referencedTableName: "workspaces"
},
{
columnNames: ["org_id"],
referencedColumnNames: ["id"],
referencedTableName: "orgs"
},
{
columnNames: ["doc_id"],
referencedColumnNames: ["id"],
referencedTableName: "docs"
},
{
columnNames: ["group_id"],
referencedColumnNames: ["id"],
referencedTableName: "groups"
}
]
}), false);
await queryRunner.createTable(new Table({
name: "group_users",
columns: [
{
name: "group_id",
type: "integer",
isPrimary: true
},
{
name: "user_id",
type: "integer",
isPrimary: true
},
],
foreignKeys: [
{
columnNames: ["group_id"],
referencedColumnNames: ["id"],
referencedTableName: "groups"
},
{
columnNames: ["user_id"],
referencedColumnNames: ["id"],
referencedTableName: "users"
}
]
}), false);
await queryRunner.createTable(new Table({
name: "group_groups",
columns: [
{
name: "group_id",
type: "integer",
isPrimary: true
},
{
name: "subgroup_id",
type: "integer",
isPrimary: true
},
],
foreignKeys: [
{
columnNames: ["group_id"],
referencedColumnNames: ["id"],
referencedTableName: "groups"
},
{
columnNames: ["subgroup_id"],
referencedColumnNames: ["id"],
referencedTableName: "groups"
}
]
}), false);
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`DROP TABLE "group_groups"`);
await queryRunner.query(`DROP TABLE "group_users"`);
await queryRunner.query(`DROP TABLE "acl_rules"`);
await queryRunner.query(`DROP TABLE "groups"`);
await queryRunner.query(`DROP TABLE "docs"`);
await queryRunner.query(`DROP TABLE "workspaces"`);
await queryRunner.query(`DROP TABLE "orgs"`);
await queryRunner.query(`DROP TABLE "users"`);
}
}

@ -0,0 +1,39 @@
import {MigrationInterface, QueryRunner, Table} from "typeorm";
export class Login1539031763952 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.createTable(new Table({
name: 'logins',
columns: [
{
name: "id",
type: "integer",
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: 'user_id',
type: 'integer'
},
{
name: 'email',
type: 'varchar',
isUnique: true
}
],
foreignKeys: [
{
columnNames: ["user_id"],
referencedColumnNames: ["id"],
referencedTableName: "users"
}
]
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query('DROP TABLE logins');
}
}

@ -0,0 +1,17 @@
import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
export class PinDocs1549313797109 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
await queryRunner.addColumn('docs', new TableColumn({
name: 'is_pinned',
type: 'boolean',
default: sqlite ? 0 : false
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropColumn('docs', 'is_pinned');
}
}

@ -0,0 +1,16 @@
import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
export class UserPicture1549381727494 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.addColumn("users", new TableColumn({
name: "picture",
type: "varchar",
isNullable: true,
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropColumn("users", "picture");
}
}

@ -0,0 +1,16 @@
import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
export class LoginDisplayEmail1551805156919 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.addColumn('logins', new TableColumn({
name: 'display_email',
type: 'varchar',
isNullable: true,
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropColumn('logins', 'display_email');
}
}

@ -0,0 +1,32 @@
import {MigrationInterface, QueryRunner} from "typeorm";
export class LoginDisplayEmailNonNull1552416614755 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query('update logins set display_email = email where display_email is null');
// if our db will already heavily loaded, it might be better to add a check constraint
// rather than modifying the column properties. But for our case, this will be fast.
// To work correctly with RDS version of postgres, it is important to clone
// and change typeorm's settings for the column, rather than the settings specified
// in previous migrations. Otherwise typeorm will fall back on a brutal method of
// drop-and-recreate that doesn't work for non-null in any case.
//
// The pg command is very simple, just alter table logins alter column display_email set not null
// but sqlite migration is tedious since table needs to be rebuilt, so still just
// marginally worthwhile letting typeorm deal with it.
const logins = (await queryRunner.getTable('logins'))!;
const displayEmail = logins.findColumnByName('display_email')!;
const displayEmailNonNull = displayEmail.clone();
displayEmailNonNull.isNullable = false;
await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
}
public async down(queryRunner: QueryRunner): Promise<any> {
const logins = (await queryRunner.getTable('logins'))!;
const displayEmail = logins.findColumnByName('display_email')!;
const displayEmailNonNull = displayEmail.clone();
displayEmailNonNull.isNullable = true;
await queryRunner.changeColumn('logins', displayEmail, displayEmailNonNull);
}
}

@ -0,0 +1,66 @@
import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
export class Indexes1553016106336 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.createIndex("acl_rules", new TableIndex({
name: "acl_rules__org_id",
columnNames: ["org_id"]
}));
await queryRunner.createIndex("acl_rules", new TableIndex({
name: "acl_rules__workspace_id",
columnNames: ["workspace_id"]
}));
await queryRunner.createIndex("acl_rules", new TableIndex({
name: "acl_rules__doc_id",
columnNames: ["doc_id"]
}));
await queryRunner.createIndex("group_groups", new TableIndex({
name: "group_groups__group_id",
columnNames: ["group_id"]
}));
await queryRunner.createIndex("group_groups", new TableIndex({
name: "group_groups__subgroup_id",
columnNames: ["subgroup_id"]
}));
await queryRunner.createIndex("group_users", new TableIndex({
name: "group_users__group_id",
columnNames: ["group_id"]
}));
await queryRunner.createIndex("group_users", new TableIndex({
name: "group_users__user_id",
columnNames: ["user_id"]
}));
await queryRunner.createIndex("workspaces", new TableIndex({
name: "workspaces__org_id",
columnNames: ["org_id"]
}));
await queryRunner.createIndex("docs", new TableIndex({
name: "docs__workspace_id",
columnNames: ["workspace_id"]
}));
await queryRunner.createIndex("logins", new TableIndex({
name: "logins__user_id",
columnNames: ["user_id"]
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropIndex("acl_rules", "acl_rules__org_id");
await queryRunner.dropIndex("acl_rules", "acl_rules__workspace_id");
await queryRunner.dropIndex("acl_rules", "acl_rules__doc_id");
await queryRunner.dropIndex("group_groups", "group_groups__group_id");
await queryRunner.dropIndex("group_groups", "group_groups__subgroup_id");
await queryRunner.dropIndex("group_users", "group_users__group_id");
await queryRunner.dropIndex("group_users", "group_users__user_id");
await queryRunner.dropIndex("workspaces", "workspaces__org_id");
await queryRunner.dropIndex("docs", "docs__workspace_id");
await queryRunner.dropIndex("logins", "logins__user_id");
}
}

@ -0,0 +1,225 @@
import {MigrationInterface, QueryRunner, Table, TableColumn, TableForeignKey} from 'typeorm';
import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
import {BillingAccountManager} from 'app/gen-server/entity/BillingAccountManager';
import {Organization} from 'app/gen-server/entity/Organization';
import {Product} from 'app/gen-server/entity/Product';
import {nativeValues} from 'app/gen-server/lib/values';
export class Billing1556726945436 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
// Create table for products.
await queryRunner.createTable(new Table({
name: 'products',
columns: [
{
name: 'id',
type: 'integer',
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: 'name',
type: 'varchar'
},
{
name: 'stripe_product_id',
type: 'varchar',
isUnique: true,
isNullable: true
},
{
name: 'features',
type: nativeValues.jsonType
}
]
}));
// Create a basic free product that existing orgs can use.
const product = new Product();
product.name = 'Free';
product.features = {};
await queryRunner.manager.save(product);
// Create billing accounts and billing account managers.
await queryRunner.createTable(new Table({
name: 'billing_accounts',
columns: [
{
name: 'id',
type: 'integer',
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: 'product_id',
type: 'integer'
},
{
name: 'individual',
type: nativeValues.booleanType
},
{
name: 'in_good_standing',
type: nativeValues.booleanType,
default: nativeValues.trueValue
},
{
name: 'status',
type: nativeValues.jsonType,
isNullable: true
},
{
name: 'stripe_customer_id',
type: 'varchar',
isUnique: true,
isNullable: true
},
{
name: 'stripe_subscription_id',
type: 'varchar',
isUnique: true,
isNullable: true
},
{
name: 'stripe_plan_id',
type: 'varchar',
isNullable: true
}
],
foreignKeys: [
{
columnNames: ['product_id'],
referencedColumnNames: ['id'],
referencedTableName: 'products'
}
]
}));
await queryRunner.createTable(new Table({
name: 'billing_account_managers',
columns: [
{
name: 'id',
type: 'integer',
isGenerated: true,
generationStrategy: 'increment',
isPrimary: true
},
{
name: 'billing_account_id',
type: 'integer'
},
{
name: 'user_id',
type: 'integer'
}
],
foreignKeys: [
{
columnNames: ['billing_account_id'],
referencedColumnNames: ['id'],
referencedTableName: 'billing_accounts',
onDelete: 'CASCADE' // delete manager if referenced billing_account goes away
},
{
columnNames: ['user_id'],
referencedColumnNames: ['id'],
referencedTableName: 'users',
onDelete: 'CASCADE' // delete manager if referenced user goes away
}
]
}));
// Add a reference to billing accounts from orgs.
await queryRunner.addColumn('orgs', new TableColumn({
name: 'billing_account_id',
type: 'integer',
isNullable: true
}));
await queryRunner.createForeignKey('orgs', new TableForeignKey({
columnNames: ['billing_account_id'],
referencedColumnNames: ['id'],
referencedTableName: 'billing_accounts'
}));
// Let's add billing accounts to all existing orgs.
// Personal orgs are put on an individual billing account.
// Other orgs are put on a team billing account, with the
// list of payment managers seeded by owners of that account.
const query =
queryRunner.manager.createQueryBuilder()
.select('orgs.id')
.from(Organization, 'orgs')
.leftJoin('orgs.owner', 'owners')
.addSelect('orgs.owner.id')
.leftJoinAndSelect('orgs.aclRules', 'acl_rules')
.leftJoinAndSelect('acl_rules.group', 'groups')
.leftJoin('groups.memberUsers', 'users')
.addSelect('users.id')
.where('permissions & 8 = 8'); // seed managers with owners+editors, omitting guests+viewers
// (permission 8 is "Remove")
const orgs = await query.getMany();
for (const org of orgs) {
const individual = Boolean(org.owner);
const billingAccountInsert = await queryRunner.manager.createQueryBuilder()
.insert()
.into(BillingAccount)
.values([{product, individual}])
.execute();
const billingAccountId = billingAccountInsert.identifiers[0].id;
if (individual) {
await queryRunner.manager.createQueryBuilder()
.insert()
.into(BillingAccountManager)
.values([{billingAccountId, userId: org.owner.id}])
.execute();
} else {
for (const rule of org.aclRules) {
for (const user of rule.group.memberUsers) {
await queryRunner.manager.createQueryBuilder()
.insert()
.into(BillingAccountManager)
.values([{billingAccountId, userId: user.id}])
.execute();
}
}
}
await queryRunner.manager.createQueryBuilder()
.update(Organization)
.set({billingAccountId})
.where('id = :id', {id: org.id})
.execute();
}
// TODO: in a future migration, orgs.billing_account_id could be constrained
// to be non-null. All code deployments linked to a database that will be
// migrated must have code that sets orgs.billing_account_id by that time,
// otherwise they would fail to create orgs (and remember creating a user
// involves creating an org).
/*
// Now that all orgs have a billing account (and this migration is running within
// a transaction), we can constrain orgs.billing_account_id to be non-null.
const orgTable = (await queryRunner.getTable('orgs'))!;
const billingAccountId = orgTable.findColumnByName('billing_account_id')!;
const billingAccountIdNonNull = billingAccountId.clone();
billingAccountIdNonNull.isNullable = false;
await queryRunner.changeColumn('orgs', billingAccountId, billingAccountIdNonNull);
*/
}
public async down(queryRunner: QueryRunner): Promise<any> {
// this is a bit ugly, but is the documented way to remove a foreign key
const table = await queryRunner.getTable('orgs');
const foreignKey = table!.foreignKeys.find(fk => fk.columnNames.indexOf('billing_account_id') !== -1);
await queryRunner.dropForeignKey('orgs', foreignKey!);
await queryRunner.dropColumn('orgs', 'billing_account_id');
await queryRunner.dropTable('billing_account_managers');
await queryRunner.dropTable('billing_accounts');
await queryRunner.dropTable('products');
}
}

@ -0,0 +1,27 @@
import {MigrationInterface, QueryRunner} from "typeorm";
export class OrgDomainUnique1557157922339 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const logins = (await queryRunner.getTable('orgs'))!;
const domain = logins.findColumnByName('domain')!;
const domainUnique = domain.clone();
domainUnique.isUnique = true;
await queryRunner.changeColumn('orgs', domain, domainUnique);
// On postgres, all of the above amounts to:
// ALTER TABLE "orgs" ADD CONSTRAINT "..." UNIQUE ("domain")
// On sqlite, the table gets regenerated.
}
public async down(queryRunner: QueryRunner): Promise<any> {
const logins = (await queryRunner.getTable('orgs'))!;
const domain = logins.findColumnByName('domain')!;
const domainNonUnique = domain.clone();
domainNonUnique.isUnique = false;
await queryRunner.changeColumn('orgs', domain, domainNonUnique);
// On postgres, all of the above amount to:
// ALTER TABLE "orgs" DROP CONSTRAINT "..."
}
}

@ -0,0 +1,67 @@
import {MigrationInterface, QueryRunner, Table, TableColumn, TableIndex} from 'typeorm';
import {datetime, now} from 'app/gen-server/sqlUtils';
export class Aliases1561589211752 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const dbType = queryRunner.connection.driver.options.type;
// Make a table for document aliases.
await queryRunner.createTable(new Table({
name: 'aliases',
columns: [
{
name: 'url_id',
type: 'varchar',
isPrimary: true
},
{
name: 'org_id',
type: 'integer',
isPrimary: true
},
{
name: 'doc_id',
type: 'varchar',
isNullable: true // nullable in case in future we make aliases for other resources
},
{
name: "created_at",
type: datetime(dbType),
default: now(dbType)
}
],
foreignKeys: [
{
columnNames: ['doc_id'],
referencedColumnNames: ['id'],
referencedTableName: 'docs',
onDelete: 'CASCADE' // delete alias if doc goes away
},
{
columnNames: ['org_id'],
referencedColumnNames: ['id'],
referencedTableName: 'orgs'
// no CASCADE set - let deletions be triggered via docs
}
]
}));
// Add preferred alias to docs. Not quite a foreign key (we'd need org as well)
await queryRunner.addColumn('docs', new TableColumn({
name: 'url_id',
type: 'varchar',
isNullable: true
}));
await queryRunner.createIndex("docs", new TableIndex({
name: "docs__url_id",
columnNames: ["url_id"]
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropIndex('docs', 'docs__url_id');
await queryRunner.dropColumn('docs', 'url_id');
await queryRunner.dropTable('aliases');
}
}

@ -0,0 +1,56 @@
import {MigrationInterface, QueryRunner} from "typeorm";
import * as roles from "app/common/roles";
import {AclRuleOrg} from "app/gen-server/entity/AclRule";
import {Group} from "app/gen-server/entity/Group";
import {Organization} from "app/gen-server/entity/Organization";
import {Permissions} from "app/gen-server/lib/Permissions";
export class TeamMembers1568238234987 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
// Get all orgs and add a team member ACL (with group) to each.
const orgs = await queryRunner.manager.createQueryBuilder()
.select("orgs.id")
.from(Organization, "orgs")
.getMany();
for (const org of orgs) {
const groupInsert = await queryRunner.manager.createQueryBuilder()
.insert()
.into(Group)
.values([{name: roles.MEMBER}])
.execute();
const groupId = groupInsert.identifiers[0].id;
await queryRunner.manager.createQueryBuilder()
.insert()
.into(AclRuleOrg)
.values([{
permissions: Permissions.VIEW,
organization: {id: org.id},
group: groupId
}])
.execute();
}
}
public async down(queryRunner: QueryRunner): Promise<any> {
// Remove all team member groups and corresponding ACLs.
const groups = await queryRunner.manager.createQueryBuilder()
.select("groups")
.from(Group, "groups")
.where('name = :name', {name: roles.MEMBER})
.getMany();
for (const group of groups) {
await queryRunner.manager.createQueryBuilder()
.delete()
.from(AclRuleOrg)
.where("group_id = :id", {id: group.id})
.execute();
}
await queryRunner.manager.createQueryBuilder()
.delete()
.from(Group)
.where("name = :name", {name: roles.MEMBER})
.execute();
}
}

@ -0,0 +1,18 @@
import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
export class FirstLogin1569593726320 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
const datetime = sqlite ? "datetime" : "timestamp with time zone";
await queryRunner.addColumn('users', new TableColumn({
name: 'first_login_at',
type: datetime,
isNullable: true
}));
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.dropColumn('users', 'first_login_at');
}
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save