Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

(core) move home server into core

Browse Source 
Summary: This moves enough server material into core to run a home server.  The data engine is not yet incorporated (though in manual testing it works when ported).

Test Plan: existing tests pass

Reviewers: dsagal

Reviewed By: dsagal

Differential Revision: https://phab.getgrist.com/D2552
This commit is contained in:
Paul Fitzpatrick
2020-07-21 09:20:51 -04:00
parent c756f663ee
commit 5ef889addd
218 changed files with 33640 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

252
app/server/lib/ActionHistory.ts Normal file
View File

@@ -0,0 +1,252 @@
/**
* TODO For now, this is just a placeholder for an actual ActionHistory implementation that should
* replace today's ActionLog. It defines all the methods that are expected from it by Sharing.ts.
*
* In addition, it will need to support some methods to show action history to the user, which is
* the main purpose of ActionLog today. And it will need to allow querying a subset of history (at
* least by table or record).
*
* The main difference with today's ActionLog is that it needs to mark actions either with labels,
* or more likely with Git-like branches, so that we can distinguish shared, local-sent, and
* local-unsent actions. And it needs to work on LocalActionBundles, which include more
* information than what ActionLog stores. On the other hand, it can probably store actions as
* blobs, which can simplify the database storage.
*/
import {LocalActionBundle} from 'app/common/ActionBundle';
import {ActionGroup} from 'app/common/ActionGroup';
import {createEmptyActionSummary} from 'app/common/ActionSummary';
import {getSelectionDesc, UserAction} from 'app/common/DocActions';
import {DocState} from 'app/common/UserAPI';
import toPairs = require('lodash/toPairs');
import {summarizeAction} from './ActionSummary';
export abstract class ActionHistory {
/**
* Initialize the ActionLog by reading the database. No other methods may be used until the
* initialization completes. If used, their behavior is undefined.
*/
public abstract initialize(): Promise<void>;
public abstract isInitialized(): boolean;
/** Returns the actionNum of the next action we expect from the hub. */
public abstract getNextHubActionNum(): number;
/** Returns the actionNum of the next local action should have. */
public abstract getNextLocalActionNum(): number;
/**
* Act as if we have already seen actionNum. getNextHubActionNum will return 1 plus this.
* Only suitable for use if there are no unshared local actions.
*/
public abstract skipActionNum(actionNum: number): Promise<void>;
/** Returns whether we have local unsent actions. */
public abstract haveLocalUnsent(): boolean;
/** Returns whether we have any local actions that have been sent to the hub. */
public abstract haveLocalSent(): boolean;
/** Returns whether we have any locally-applied actions. */
public abstract haveLocalActions(): boolean;
/** Fetches and returns an array of all local unsent actions. */
public abstract fetchAllLocalUnsent(): Promise<LocalActionBundle[]>;
/** Fetches and returns an array of all local actions (sent and unsent). */
public abstract fetchAllLocal(): Promise<LocalActionBundle[]>;
/** Deletes all local-only actions, and resets the affected branch pointers. */
// TODO Should we actually delete, or be more git-like, only reset local branch pointer, and let
// cleanup of unreferenced actions happen in a separate step?
public abstract clearLocalActions(): Promise<void>;
/**
* Marks all actions returned from fetchAllLocalUnsent() as sent. Actions must be consecutive
* starting with the the first local unsent action.
*/
public abstract markAsSent(actions: LocalActionBundle[]): Promise<void>;
/**
* Matches the action from the hub against the first sent local action. If it's the same action,
* marks our action as "shared", i.e. accepted by the hub, and returns true. Else returns false.
* If actionHash is null, accepts unconditionally.
*/
public abstract acceptNextSharedAction(actionHash: string|null): Promise<boolean>;
/** Records a new local unsent action, after setting action.actionNum appropriately. */
public abstract recordNextLocalUnsent(action: LocalActionBundle): Promise<void>;
/** Records a new action received from the hub, after setting action.actionNum appropriately. */
public abstract recordNextShared(action: LocalActionBundle): Promise<void>;
/**
* Get the most recent actions from the history. Results are ordered by
* earliest actions first, later actions later. If `maxActions` is supplied,
* at most that number of actions are returned.
*/
public abstract getRecentActions(maxActions?: number): Promise<LocalActionBundle[]>;
/**
* Get the most recent states from the history. States are just
* actions without any content. Results are ordered by most recent
* states first (careful, this is the opposite to getRecentActions).
* If `maxStates` is supplied, at most that number of actions are
* returned.
*/
public abstract getRecentStates(maxStates?: number): Promise<DocState[]>;
/**
* Get a list of actions, identified by their actionNum. Any actions that could not be
* found are returned as undefined.
*/
public abstract getActions(actionNums: number[]): Promise<Array<LocalActionBundle|undefined>>;
/**
* Associates an action with a client. This association is expected to be transient, rather
* than persistent. It should survive a client-side reload but not a server-side restart.
*/
public abstract setActionClientId(actionHash: string, clientId: string): void;
/** Check for any client associated with an action, identified by checksum */
public abstract getActionClientId(actionHash: string): string | undefined;
}
/**
* Old helper to display the actionGroup in a human-readable way. Being maintained
* to avoid having to change too much at once.
*/
export function humanDescription(actions: UserAction[]): string {
const action = actions[0];
if (!action) { return ""; }
let output = '';
// Common names for various action parameters
const name = action[0];
const table = action[1];
const rows = action[2];
const colId = action[2];
const columns: any = action[3]; // TODO - better typing - but code may evaporate
switch (name) {
case 'UpdateRecord':
case 'BulkUpdateRecord':
case 'AddRecord':
case 'BulkAddRecord':
output = name + ' ' + getSelectionDesc(action, columns);
break;
case 'ApplyUndoActions':
// Currently cannot display information about what action was undone, as the action comes
// with the description of the "undo" message, which might be very different
// Also, cannot currently properly log redos as they are not distinguished from others in any way
// TODO: make an ApplyRedoActions type for redoing actions
output = 'Undo Previous Action';
break;
case 'InitNewDoc':
output = 'Initialized new Document';
break;
case 'AddColumn':
output = 'Added column ' + colId + ' to ' + table;
break;
case 'RemoveColumn':
output = 'Removed column ' + colId + ' from ' + table;
break;
case 'RemoveRecord':
case 'BulkRemoveRecord':
output = 'Removed record(s) ' + rows + ' from ' + table;
break;
case 'EvalCode':
output = 'Evaluated Code ' + action[1];
break;
case 'AddTable':
output = 'Added table ' + table;
break;
case 'RemoveTable':
output = 'Removed table ' + table;
break;
case 'ModifyColumn':
// TODO: The Action Log currently only logs user actions,
// But ModifyColumn/Rename Column are almost always triggered from the client
// through a meta-table UpdateRecord.
// so, this is a case where making use of explicit sandbox engine 'looged' actions
// may be useful
output = 'Modify column ' + colId + ", ";
for (const [col, val] of toPairs(columns)) {
output += col + ": " + val + ", ";
}
output += ' in table ' + table;
break;
case 'RenameColumn': {
const newColId = action[3];
output = 'Renamed Column ' + colId + ' to ' + newColId + ' in ' + table;
break;
}
default:
output = name + ' [No Description]';
}
// A period for good grammar
output += '.';
return output;
}
/**
* Convert an ActionBundle into an ActionGroup. ActionGroups are the representation of
* actions on the client.
* @param history: interface to action history
* @param act: action to convert
* @param options.summarize: if set, inspect the action in detail in order to include a summary of
* changes made within the action. Otherwise, the actionSummary returned is empty.
* @param options.client: the client for which the action group is being prepared, if known.
* @param options.retValues: values returned by the action, if known.
*/
export function asActionGroup(history: ActionHistory,
act: LocalActionBundle,
options: {
summarize?: boolean
client?: {clientId: string}|null,
retValues?: any[],
}): ActionGroup {
const {summarize, client, retValues} = options;
const info = act.info[1];
const fromSelf = (client && client.clientId && act.actionHash) ?
(history.getActionClientId(act.actionHash) === client.clientId) : false;
let rowIdHint = 0;
if (retValues) {
// A hint for cursor position. This logic used to live on the client, but now trying to
// limit how much the client looks at the internals of userActions.
// In case of AddRecord, the returned value is rowId, which is the best cursorPos for Redo.
for (let i = 0; i < act.userActions.length; i++) {
const name = act.userActions[i][0];
const retValue = retValues[i];
if (name === 'AddRecord') {
rowIdHint = retValue;
break;
} else if (name === 'BulkAddRecord') {
rowIdHint = retValue[0];
break;
}
}
}
const primaryAction: string = String((act.userActions[0] || [""])[0]);
const isUndo = primaryAction === 'ApplyUndoActions';
return {
actionNum: act.actionNum,
actionHash: act.actionHash || "",
desc: info.desc || humanDescription(act.userActions),
actionSummary: summarize ? summarizeAction(act) : createEmptyActionSummary(),
fromSelf,
linkId: info.linkId,
otherId: info.otherId,
time: info.time,
user: info.user,
rowIdHint,
primaryAction,
isUndo,
internal: act.actionNum === 0 // Mark lazy-loading calculated columns. In future,
// synchronizing fields to today's date and other
// changes from external values may count as internal.
};
}

673
app/server/lib/ActionHistoryImpl.ts Normal file
View File

@@ -0,0 +1,673 @@
/**
* Minimal ActionHistory implementation
*/
import {LocalActionBundle} from 'app/common/ActionBundle';
import * as marshaller from 'app/common/marshal';
import {DocState} from 'app/common/UserAPI';
import * as crypto from 'crypto';
import keyBy = require('lodash/keyBy');
import mapValues = require('lodash/mapValues');
import {ActionHistory} from './ActionHistory';
import {ISQLiteDB, ResultRow} from './SQLiteDB';
// History will from time to time be pruned back to within these limits
// on rows and the maximum total number of bytes in the "body" column.
// Pruning is done when the history has grown above these limits, to
// the specified factor.
const ACTION_HISTORY_MAX_ROWS = 1000;
const ACTION_HISTORY_MAX_BYTES = 1000 * 1000 * 1000; // 1 GB.
const ACTION_HISTORY_GRACE_FACTOR = 1.25; // allow growth to 1250 rows / 1.25 GB.
const ACTION_HISTORY_CHECK_PERIOD = 10; // number of actions between size checks.
/**
*
* Encode an action as a buffer.
*
*/
export function encodeAction(action: LocalActionBundle): Buffer {
const encoder = new marshaller.Marshaller({version: 2});
encoder.marshal(action);
return encoder.dumpAsBuffer();
}
/**
*
* Decode an action from a buffer. Throws an error if buffer doesn't look plausible.
*
*/
export function decodeAction(blob: Buffer | Uint8Array): LocalActionBundle {
return marshaller.loads(blob) as LocalActionBundle;
}
/**
*
* Decode an action from an ActionHistory row. Row must include body, actionNum, actionHash fields.
*
*/
function decodeActionFromRow(row: ResultRow): LocalActionBundle {
const body = decodeAction(row.body);
// Reset actionNum and actionHash, just to have one fewer thing to worry about.
body.actionNum = row.actionNum;
body.actionHash = row.actionHash;
return body;
}
/**
*
* Generate an action checksum from a LocalActionBundle
* Needs to be in sync with Hub/Sharing.
*
*/
export function computeActionHash(action: LocalActionBundle): string {
const shaSum = crypto.createHash('sha256');
const encoder = new marshaller.Marshaller({version: 2});
encoder.marshal(action.actionNum);
encoder.marshal(action.parentActionHash);
encoder.marshal(action.info);
encoder.marshal(action.stored);
const buf = encoder.dumpAsBuffer();
shaSum.update(buf);
return shaSum.digest('hex');
}
/** The important identifiers associated with an action */
interface ActionIdentifiers {
/**
*
* actionRef is the SQLite-allocated row id in the main ActionHistory table.
* See:
* https://www.sqlite.org/rowidtable.html
* https://sqlite.org/autoinc.html
* for background on how this works.
*
*/
actionRef: number|null;
/**
*
* actionHash is a checksum computed from salient parts of an ActionBundle.
*
*/
actionHash: string|null;
/**
*
* actionNum is the depth in history from the root, starting from 1 for the first
* action.
*
*/
actionNum: number|null;
/**
*
* The name of a branch where we found this action.
*
*/
branchName: string;
}
/** An organized view of the standard branches: shared, local_sent, local_unsent */
interface StandardBranches {
shared: ActionIdentifiers;
local_sent: ActionIdentifiers;
local_unsent: ActionIdentifiers;
}
/** Tweakable parameters for storing the action history */
interface ActionHistoryOptions {
maxRows: number; // maximum number of rows to aim for
maxBytes: number; // maximum total "body" bytes to aim for
graceFactor: number; // allow this amount of slop in limits
checkPeriod: number; // number of actions between checks
}
const defaultOptions: ActionHistoryOptions = {
maxRows: ACTION_HISTORY_MAX_ROWS,
maxBytes: ACTION_HISTORY_MAX_BYTES,
graceFactor: ACTION_HISTORY_GRACE_FACTOR,
checkPeriod: ACTION_HISTORY_CHECK_PERIOD,
};
/**
*
* An implementation of the ActionHistory interface, using SQLite tables.
*
* The history of Grist actions is essentially linear. We have a notion of
* action branches only to track certain "subhistories" of those actions,
* specifically:
* - those actions that have been "shared"
* - those actions that have been "sent" (but not yet declared "shared")
* The "shared" branch reaches from the beginning of history to the last known
* shared action. The "local_sent" branch reaches at least to that point, and
* potentially on to other actions that have been "sent" but not "shared".
* All remaining branches -- just one right now, called "local_unsent" --
* continue on from there. We may in the future permit multiple such
* branches. In this case, this part of the action history could actually
* form a tree and not be linear.
*
* For all branches, we track their "tip", the most recent action on
* that branch.
*
* TODO: links to parent actions stored in bundles are not currently
* updated in the database when those parent actions are deleted. If this
* is an issue, it might be best to remove such information from the bundles
* when stored and add it back as it is retrieved, or treat it separately.
*
*/
export class ActionHistoryImpl implements ActionHistory {
private _sharedActionNum: number = 1; // track depth in tree of shared actions
private _localActionNum: number = 1; // track depth in tree of local actions
private _haveLocalSent: boolean = false; // cache for this.haveLocalSent()
private _haveLocalUnsent: boolean = false; // cache for this.haveLocalUnsent()
private _initialized: boolean = false; // true when initialize() has completed
private _actionClient = new Map<string, string>(); // transient cache of who created actions
constructor(private _db: ISQLiteDB, private _options: ActionHistoryOptions = defaultOptions) {
}
/** remove any existing data from ActionHistory - useful during testing. */
public async wipe() {
await this._db.run("UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL");
await this._db.run("DELETE FROM _gristsys_ActionHistory");
this._actionClient.clear();
}
public async initialize(): Promise<void> {
const branches = await this._getBranches();
if (branches.shared.actionNum) {
this._sharedActionNum = branches.shared.actionNum + 1;
}
if (branches.local_unsent.actionNum) {
this._localActionNum = branches.local_unsent.actionNum + 1;
}
// Record whether we currently have local actions (sent or unsent).
const sharedActionNum = branches.shared.actionNum || -1;
const localSentActionNum = branches.local_sent.actionNum || -1;
const localUnsentActionNum = branches.local_unsent.actionNum || -1;
this._haveLocalUnsent = localUnsentActionNum > localSentActionNum;
this._haveLocalSent = localSentActionNum > sharedActionNum;
this._initialized = true;
// Apply any limits on action history size.
await this._pruneLargeHistory(sharedActionNum);
}
public isInitialized(): boolean {
return this._initialized;
}
public getNextHubActionNum(): number {
return this._sharedActionNum;
}
public getNextLocalActionNum(): number {
return this._localActionNum;
}
public async skipActionNum(actionNum: number): Promise<void> {
if (this._localActionNum !== this._sharedActionNum) {
throw new Error("Tried to skip to an actionNum with unshared local actions");
}
if (actionNum < this._sharedActionNum) {
if (actionNum === this._sharedActionNum - 1) {
// that was easy
return;
}
throw new Error("Tried to skip to an actionNum we've already passed");
}
// Force the actionNum to the desired value
this._localActionNum = this._sharedActionNum = actionNum;
// We store a row as we would for recordNextShared()
const action: LocalActionBundle = {
actionHash: null,
parentActionHash: null,
actionNum: this._sharedActionNum,
userActions: [],
undo: [],
envelopes: [],
info: [0, {time: 0, user: "grist", inst: "", desc: "root", otherId: 0, linkId: 0}],
stored: [],
calc: []
};
await this._db.execTransaction(async () => {
const branches = await this._getBranches();
if (branches.shared.actionRef !== branches.local_sent.actionRef ||
branches.shared.actionRef !== branches.local_unsent.actionRef) {
throw new Error("skipActionNum not defined when branches not in sync");
}
const actionRef = await this._addAction(action, branches.shared);
this._noteSharedAction(action.actionNum);
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name IN ('local_unsent', 'local_sent')`,
actionRef);
});
}
public haveLocalUnsent(): boolean {
return this._haveLocalUnsent;
}
public haveLocalSent(): boolean {
return this._haveLocalSent;
}
public haveLocalActions(): boolean {
return this._haveLocalSent || this._haveLocalUnsent;
}
public async fetchAllLocalUnsent(): Promise<LocalActionBundle[]> {
const branches = await this._getBranches();
return this._fetchActions(branches.local_sent, branches.local_unsent);
}
public async fetchAllLocal(): Promise<LocalActionBundle[]> {
const branches = await this._getBranches();
return this._fetchActions(branches.shared, branches.local_unsent);
}
public async clearLocalActions(): Promise<void> {
await this._db.execTransaction(async () => {
const branches = await this._getBranches();
const rows = await this._fetchParts(branches.shared, branches.local_unsent,
"_gristsys_ActionHistory.id, actionHash");
await this._deleteRows(rows);
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name IN ('local_unsent', 'local_sent')`,
branches.shared.actionRef);
this._haveLocalSent = false;
this._haveLocalUnsent = false;
this._localActionNum = this._sharedActionNum;
});
}
public async markAsSent(actions: LocalActionBundle[]): Promise<void> {
const branches = await this._getBranches();
const candidates = await this._fetchParts(branches.local_sent,
branches.local_unsent,
"_gristsys_ActionHistory.id, actionHash");
let tip: number|undefined;
try {
for (const act of actions) {
if (candidates.length === 0) {
throw new Error("markAsSent() called but nothing local and unsent");
}
const candidate = candidates[0];
// act and act2 must be one and the same
if (act.actionHash !== candidate.actionHash) {
throw new Error("markAsSent() got an unexpected action");
}
tip = candidate.id;
candidates.shift();
if (candidates.length === 0) {
this._haveLocalUnsent = false;
}
this._haveLocalSent = true;
}
} finally {
if (tip) {
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name = "local_sent"`,
tip);
}
}
}
public async acceptNextSharedAction(actionHash: string|null): Promise<boolean> {
const branches = await this._getBranches();
const candidates = await this._fetchParts(branches.shared,
branches.local_sent,
"_gristsys_ActionHistory.id, actionHash, actionNum",
2);
if (candidates.length === 0) {
return false;
}
const candidate = candidates[0];
if (actionHash != null) {
if (candidate.actionHash !== actionHash) {
return false;
}
}
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name = "shared"`,
candidate.id);
if (candidates.length === 1) {
this._haveLocalSent = false;
}
this._noteSharedAction(candidate.actionNum);
await this._pruneLargeHistory(candidate.actionNum);
return true;
}
/** This will populate action.actionHash and action.parentActionHash */
public async recordNextLocalUnsent(action: LocalActionBundle): Promise<void> {
const branches = await this._getBranches();
await this._addAction(action, branches.local_unsent);
this._noteLocalAction(action.actionNum);
this._haveLocalUnsent = true;
}
public async recordNextShared(action: LocalActionBundle): Promise<void> {
// I think, reading Sharing.ts, that these actions should be added to all
// the system branches - it is just a shortcut for getting to shared
await this._db.execTransaction(async () => {
const branches = await this._getBranches();
if (branches.shared.actionRef !== branches.local_sent.actionRef ||
branches.shared.actionRef !== branches.local_unsent.actionRef) {
throw new Error("recordNextShared not defined when branches not in sync");
}
const actionRef = await this._addAction(action, branches.shared);
this._noteSharedAction(action.actionNum);
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name IN ('local_unsent', 'local_sent')`,
actionRef);
});
await this._pruneLargeHistory(action.actionNum);
}
public async getRecentActions(maxActions?: number): Promise<LocalActionBundle[]> {
const branches = await this._getBranches();
const actions = await this._fetchParts(null,
branches.local_unsent,
"_gristsys_ActionHistory.id, actionNum, actionHash, body",
maxActions,
true);
const result = actions.map(decodeActionFromRow);
result.reverse(); // Implementation note: this could be optimized away when `maxActions`
// is not specified, by simply asking _fetchParts for ascending order.
return result;
}
public async getRecentStates(maxStates?: number): Promise<DocState[]> {
const branches = await this._getBranches();
const states = await this._fetchParts(null,
branches.local_unsent,
"_gristsys_ActionHistory.id, actionNum, actionHash",
maxStates,
true);
return states.map(row => ({n: row.actionNum, h: row.actionHash}));
}
public async getActions(actionNums: number[]): Promise<Array<LocalActionBundle|undefined>> {
const actions = await this._db.all(`SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory
where actionNum in (${actionNums.map(x => '?').join(',')})`,
actionNums);
const actionsByActionNum = keyBy(actions, 'actionNum');
return actionNums
.map(n => actionsByActionNum[n])
.map((row) => row ? decodeActionFromRow(row) : undefined);
}
/**
* Helper function to remove all stored actions except the last keepN and run the VACUUM command
* to reduce the size of the SQLite file.
*
* @param {Int} keepN - The number of most recent actions to keep. The value must be at least 1, and
* will default to 1 if not given.
* @returns {Promise} - A promise for the SQL execution.
*
* NOTE: Only keeps actions after maxActionNum - keepN, which might be less than keepN actions if
* actions are not sequential in the file.
*/
public async deleteActions(keepN: number): Promise<void> {
await this._db.execTransaction(async () => {
const branches = await this._getBranches();
const rows = await this._fetchParts(null,
branches.local_unsent,
"_gristsys_ActionHistory.id, actionHash",
keepN,
true);
const ids = await this._deleteRows(rows, true);
// By construction, we are removing all rows from the start of history to a certain point.
// So, if any of the removed actions are mentioned as the tip of a branch, that tip should
// now simply become null/empty.
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL WHERE actionRef NOT IN (${ids})`);
await this._db.requestVacuum();
});
}
public setActionClientId(actionHash: string, clientId: string): void {
this._actionClient.set(actionHash, clientId);
}
public getActionClientId(actionHash: string): string | undefined {
return this._actionClient.get(actionHash);
}
/** Check if we need to update the next shared actionNum */
private _noteSharedAction(actionNum: number): void {
if (actionNum >= this._sharedActionNum) {
this._sharedActionNum = actionNum + 1;
}
this._noteLocalAction(actionNum);
}
/** Check if we need to update the next local actionNum */
private _noteLocalAction(actionNum: number): void {
if (actionNum >= this._localActionNum) {
this._localActionNum = actionNum + 1;
}
}
/** Append an action to a branch. */
private async _addAction(action: LocalActionBundle,
branch: ActionIdentifiers): Promise<number> {
action.parentActionHash = branch.actionHash;
if (!action.actionHash) {
action.actionHash = computeActionHash(action);
}
const buf = encodeAction(action);
return this._db.execTransaction(async () => {
// Add the action. We let SQLite fill in the "id" column, which is an alias for
// the SQLite rowid in this case: https://www.sqlite.org/rowidtable.html
const id = await this._db.runAndGetId(`INSERT INTO _gristsys_ActionHistory
(actionHash, parentRef, actionNum, body)
VALUES (?, ?, ?, ?)`,
action.actionHash,
branch.actionRef,
action.actionNum,
buf);
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name = ?`,
id, branch.branchName);
return id;
});
}
/** Get the current status of the standard branches: shared, local_sent, and local_unsent */
private async _getBranches(): Promise<StandardBranches> {
const rows = await this._db.all(`SELECT name, actionNum, actionHash, Branch.actionRef
FROM _gristsys_ActionHistoryBranch as Branch
LEFT JOIN _gristsys_ActionHistory as History
ON History.id = Branch.actionRef
WHERE name in ("shared", "local_sent", "local_unsent")`);
const bits = mapValues(keyBy(rows, 'name'), this._asActionIdentifiers);
const missing = { actionHash: null, actionRef: null, actionNum: null } as ActionIdentifiers;
return {
shared: bits.shared || missing,
local_sent: bits.local_sent || missing,
local_unsent: bits.local_unsent || missing
};
}
/** Cast an sqlite result row into a structure with the IDs we care about */
private _asActionIdentifiers(row: ResultRow|null): ActionIdentifiers|null {
if (!row) {
return null;
}
return {
actionRef: row.actionRef,
actionHash: row.actionHash,
actionNum: row.actionNum,
branchName: row.name
};
}
/**
*
* Fetch selected parts of a range of actions. We do a recursive query
* working backwards from the action identified by `end`, following a
* chain of ancestors via `parentRef` links, until we reach the action
* identified by `start` or run out of ancestors. The action identified
* by `start` is NOT included in the results. Results are returned in
* ascending order of `actionNum` - in other words results closer to the
* beginning of history are returned first.
*
* @param start - identifiers of an action not to include in the results.
* @param end - identifiers of an action to include in the results
* @param selection - SQLite SELECT result-columns to return
* @param limit - optional cap on the number of results to return.
* @param desc - optional - if true, invert order of results, starting
* from highest `actionNum` rather than lowest.
*
* @return a list of ResultRows, containing whatever was requested in
* the `selection` parameter for each action found.
*
*/
private async _fetchParts(start: ActionIdentifiers|null,
end: ActionIdentifiers|null,
selection: string,
limit?: number,
desc?: boolean): Promise<ResultRow[]> {
if (!end) { return []; }
// Collect all actions, Starting at the branch tip, and working
// backwards until we hit a delimiting actionNum.
// See https://sqlite.org/lang_with.html for details of recursive CTEs.
const rows = await this._db.all(`WITH RECURSIVE
actions(id) AS (
VALUES(?)
UNION ALL
SELECT parentRef FROM _gristsys_ActionHistory, actions
WHERE _gristsys_ActionHistory.id = actions.id
AND parentRef IS NOT NULL
AND _gristsys_ActionHistory.id IS NOT ?)
SELECT ${selection} from actions
JOIN _gristsys_ActionHistory
ON actions.id = _gristsys_ActionHistory.id
WHERE _gristsys_ActionHistory.id IS NOT ?
ORDER BY actionNum ${desc ? "DESC " : ""}
${limit ? ("LIMIT " + limit) : ""}`,
end.actionRef,
start ? start.actionRef : null,
start ? start.actionRef : null);
return rows;
}
/**
*
* Fetch a range of actions as LocalActionBundles. We do a recursive query
* working backwards from the action identified by `end`, following a
* chain of ancestors via `parentRef` links, until we reach the action
* identified by `start` or run out of ancestors. The action identified
* by `start` is NOT included in the results. Results are returned in
* ascending order of `actionNum` - in other words results closer to the
* beginning of history are returned first.
*
* @param start - identifiers of an action not to include in the results.
* @param end - identifiers of an action to include in the results
*
* @return a list of LocalActionBundles.
*
*/
private async _fetchActions(start: ActionIdentifiers|null,
end: ActionIdentifiers|null): Promise<LocalActionBundle[]> {
const rows = await this._fetchParts(start, end, "body, actionNum, actionHash");
return rows.map(decodeActionFromRow);
}
/**
* Delete rows in the ActionHistory. Any client id association is also removed for
* the given rows. Branch information is not updated, it is the responsibility of
* the caller to keep that synchronized.
*
* @param rows: The rows to delete. Should have at least id and actionHash fields.
* @param invert: True if all but the listed rows should be deleted.
*
* Returns the list of ids of the supplied rows.
*/
private async _deleteRows(rows: ResultRow[], invert?: boolean): Promise<number[]> {
// There's no great solution for passing a long list of numbers to sqlite for a
// single query. Here, we concatenate them with comma separators and embed them
// in the SQL string.
// TODO: deal with limit on max length of sql statement https://www.sqlite.org/limits.html
const ids = rows.map(row => row.id);
const idList = ids.join(',');
await this._db.run(`DELETE FROM _gristsys_ActionHistory
WHERE id ${invert ? 'NOT' : ''} IN (${idList})`);
for (const row of rows) {
this._actionClient.delete(row.actionHash);
}
return ids;
}
/**
* Deletes rows in the ActionHistory if there are too many of them or they hold too
* much data.
*/
private async _pruneLargeHistory(actionNum: number): Promise<void> {
// We check history size occasionally, not on every single action. The check
// requires summing a blob length over up to roughly ACTION_HISTORY_MAX_ROWS rows.
// For a 2GB test db with 3 times this number of rows, the check takes < 10 ms.
// But there's no need to add that tax to every action.
if (actionNum % this._options.checkPeriod !== 0) {
return;
}
// Do a quick check on the history size. We work on the "shared" branch, to
// avoid the possibility of deleting history that has not yet been shared.
let branches = await this._getBranches();
const checks = (await this._fetchParts(null,
branches.shared,
"count(*) as count, sum(length(body)) as bytes",
undefined,
true))[0];
if (checks.count <= this._options.maxRows * this._options.graceFactor &&
checks.bytes <= this._options.maxBytes * this._options.graceFactor) {
return; // Nothing to do, size is ok.
}
// Too big! Check carefully what needs to be done.
await this._db.execTransaction(async () => {
// Make sure branches are up to date within this transaction.
branches = await this._getBranches();
const rows = await this._fetchParts(null,
branches.shared,
"_gristsys_ActionHistory.id, actionHash, actionNum, length(body) as bytes",
undefined,
true);
// Scan to find the first row that pushes us over a limit.
let count: number = 0;
let bytes: number = 0;
let first: number = -1;
for (let i = 0; i < rows.length; i++) {
const row = rows[i];
count++;
bytes += row.bytes;
if (count > 1 && (bytes > this._options.maxBytes || count > this._options.maxRows)) {
first = i;
break;
}
}
if (first === -1) { return; }
// Delete remaining rows - in batches because _deleteRows has limited capacity.
const batchLength: number = 100;
for (let i = first; i < rows.length; i += batchLength) {
const batch = rows.slice(i, i + batchLength);
const ids = await this._deleteRows(batch);
// We are removing all rows from the start of history to a certain point.
// So, if any of the removed actions are mentioned as the tip of a branch,
// that tip should now simply become null/empty.
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = NULL WHERE actionRef IN (${ids})`);
}
// At this point, to recover the maximum memory, we could VACUUM the document.
// But vacuuming is an unacceptably slow operation for large documents (e.g.
// 30 secs for a 2GB doc) so it is obnoxious to do that while the user is waiting.
// Without vacuuming, the document will grow due to fragmentation, but this should
// be at a lower rate than it would grow if we were simply retaining full history.
// TODO: occasionally VACUUM large documents while they are not being used.
});
}
}

434
app/server/lib/ActionSummary.ts Normal file
View File

@@ -0,0 +1,434 @@
import {getEnvContent, LocalActionBundle} from 'app/common/ActionBundle';
import {ActionSummary, ColumnDelta, createEmptyActionSummary,
createEmptyTableDelta, defunctTableName, LabelDelta, TableDelta} from 'app/common/ActionSummary';
import {DocAction} from 'app/common/DocActions';
import * as Action from 'app/common/DocActions';
import {arrayExtend} from 'app/common/gutil';
import {CellDelta} from 'app/common/TabularDiff';
import fromPairs = require('lodash/fromPairs');
import keyBy = require('lodash/keyBy');
import sortBy = require('lodash/sortBy');
import toPairs = require('lodash/toPairs');
import values = require('lodash/values');
/**
* The maximum number of rows in a single bulk change that will be recorded
* individually. Bulk changes that touch more than this number of rows
* will be summarized only by the number of rows touched.
*/
const MAXIMUM_INLINE_ROWS = 10;
/** helper function to access summary changes for a specific table by name */
function _forTable(summary: ActionSummary, tableId: string): TableDelta {
return summary.tableDeltas[tableId] || (summary.tableDeltas[tableId] = createEmptyTableDelta());
}
/** helper function to access summary changes for a specific cell by rowId and colId */
function _forCell(td: TableDelta, rowId: number, colId: string): CellDelta {
const cd = td.columnDeltas[colId] || (td.columnDeltas[colId] = {});
return cd[rowId] || (cd[rowId] = [null, null]);
}
/**
* helper function to store detailed cell changes for a single row.
* Direction parameter is 0 if values are prior values of cells, 1 if values are new values.
*/
function _addRow(td: TableDelta, rowId: number, colValues: Action.ColValues,
direction: 0|1) {
for (const [colId, colChanges] of toPairs(colValues)) {
const cell = _forCell(td, rowId, colId);
cell[direction] = [colChanges];
}
}
/** helper function to store detailed cell changes for a set of rows */
function _addRows(tableId: string, td: TableDelta, rowIds: number[],
colValues: Action.BulkColValues, direction: 0|1) {
let rows: Array<[number, number]>;
if (rowIds.length <= MAXIMUM_INLINE_ROWS || tableId.startsWith("_grist_")) {
rows = [...rowIds.entries()];
} else {
// if many rows, just take some from start and one from end as examples
rows = [...rowIds.slice(0, MAXIMUM_INLINE_ROWS - 1).entries()];
rows.push([rowIds.length - 1, rowIds[rowIds.length - 1]]);
}
for (const [colId, colChanges] of toPairs(colValues)) {
rows.forEach(([idx, rowId]) => {
const cell = _forCell(td, rowId, colId);
cell[direction] = [colChanges[idx]];
});
}
}
/** add a rename to a list, avoiding duplicates */
function _addRename(renames: LabelDelta[], rename: LabelDelta) {
if (renames.find(r => r[0] === rename[0] && r[1] === rename[1])) { return; }
renames.push(rename);
}
/** add information about an action based on the forward direction */
function _addForwardAction(summary: ActionSummary, act: DocAction) {
const tableId = act[1];
if (Action.isAddTable(act)) {
summary.tableRenames.push([null, tableId]);
for (const info of act[2]) {
_forTable(summary, tableId).columnRenames.push([null, info.id]);
}
} else if (Action.isRenameTable(act)) {
_addRename(summary.tableRenames, [tableId, act[2]]);
} else if (Action.isRenameColumn(act)) {
_addRename(_forTable(summary, tableId).columnRenames, [act[2], act[3]]);
} else if (Action.isAddColumn(act)) {
_forTable(summary, tableId).columnRenames.push([null, act[2]]);
} else if (Action.isRemoveColumn(act)) {
_forTable(summary, tableId).columnRenames.push([act[2], null]);
} else if (Action.isAddRecord(act)) {
const td = _forTable(summary, tableId);
td.addRows.push(act[2]);
_addRow(td, act[2], act[3], 1);
} else if (Action.isUpdateRecord(act)) {
const td = _forTable(summary, tableId);
td.updateRows.push(act[2]);
_addRow(td, act[2], act[3], 1);
} else if (Action.isBulkAddRecord(act)) {
const td = _forTable(summary, tableId);
arrayExtend(td.addRows, act[2]);
_addRows(tableId, td, act[2], act[3], 1);
} else if (Action.isBulkUpdateRecord(act)) {
const td = _forTable(summary, tableId);
arrayExtend(td.updateRows, act[2]);
_addRows(tableId, td, act[2], act[3], 1);
} else if (Action.isReplaceTableData(act)) {
const td = _forTable(summary, tableId);
arrayExtend(td.addRows, act[2]);
_addRows(tableId, td, act[2], act[3], 1);
}
}
/** add information about an action based on undo information */
function _addReverseAction(summary: ActionSummary, act: DocAction) {
const tableId = act[1];
if (Action.isAddTable(act)) { // undoing, so this is a table removal
summary.tableRenames.push([tableId, null]);
for (const info of act[2]) {
_forTable(summary, tableId).columnRenames.push([info.id, null]);
}
} else if (Action.isAddRecord(act)) { // undoing, so this is a record removal
const td = _forTable(summary, tableId);
td.removeRows.push(act[2]);
_addRow(td, act[2], act[3], 0);
} else if (Action.isUpdateRecord(act)) { // undoing, so this is reversal of a record update
const td = _forTable(summary, tableId);
_addRow(td, act[2], act[3], 0);
} else if (Action.isBulkAddRecord(act)) { // undoing, this may be reversing a table delete
const td = _forTable(summary, tableId);
arrayExtend(td.removeRows, act[2]);
_addRows(tableId, td, act[2], act[3], 0);
} else if (Action.isBulkUpdateRecord(act)) { // undoing, so this is reversal of a bulk record update
const td = _forTable(summary, tableId);
arrayExtend(td.updateRows, act[2]);
_addRows(tableId, td, act[2], act[3], 0);
} else if (Action.isRenameTable(act)) { // undoing - sometimes renames only in undo info
_addRename(summary.tableRenames, [act[2], tableId]);
} else if (Action.isRenameColumn(act)) { // undoing - sometimes renames only in undo info
_addRename(_forTable(summary, tableId).columnRenames, [act[3], act[2]]);
} else if (Action.isReplaceTableData(act)) { // undoing
const td = _forTable(summary, tableId);
arrayExtend(td.removeRows, act[2]);
_addRows(tableId, td, act[2], act[3], 0);
}
}
/**
* Summarize the tabular changes that a LocalActionBundle results in, in a form
* that will be suitable for composition.
*/
export function summarizeAction(body: LocalActionBundle): ActionSummary {
const summary = createEmptyActionSummary();
for (const act of getEnvContent(body.stored)) {
_addForwardAction(summary, act);
}
for (const act of Array.from(body.undo).reverse()) {
_addReverseAction(summary, act);
}
// Name tables consistently, by their ultimate name, now we know it.
for (const renames of summary.tableRenames) {
const pre = renames[0];
let post = renames[1];
if (pre === null) { continue; }
if (post === null) { post = defunctTableName(pre); }
if (summary.tableDeltas[pre]) {
summary.tableDeltas[post] = summary.tableDeltas[pre];
delete summary.tableDeltas[pre];
}
}
for (const td of values(summary.tableDeltas)) {
// Name columns consistently, by their ultimate name, now we know it.
for (const renames of td.columnRenames) {
const pre = renames[0];
let post = renames[1];
if (pre === null) { continue; }
if (post === null) { post = defunctTableName(pre); }
if (td.columnDeltas[pre]) {
td.columnDeltas[post] = td.columnDeltas[pre];
delete td.columnDeltas[pre];
}
}
// remove any duplicates that crept in
td.addRows = Array.from(new Set(td.addRows));
td.updateRows = Array.from(new Set(td.updateRows));
td.removeRows = Array.from(new Set(td.removeRows));
}
return summary;
}
/**
* Once we can produce an ActionSummary for each LocalActionBundle, it is useful to be able
* to compose them. Take the case of an ActionSummary pair, part 1 and part 2. NameMerge
* is an internal structure to help merging table/column name changes across two parts.
*/
interface NameMerge {
dead1: Set<string>; /** anything of this name in part 1 should be removed from merge */
dead2: Set<string>; /** anything of this name in part 2 should be removed from merge */
rename1: Map<string, string>; /** replace these names in part 1 */
rename2: Map<string, string>; /** replace these names in part 2 */
merge: LabelDelta[]; /** a merged list of adds/removes/renames for the result */
}
/**
* Looks at a pair of name change lists (could be tables or columns) and figures out what
* changes would need to be made to a data structure keyed on those names in order to key
* it consistently on final names.
*/
function planNameMerge(names1: LabelDelta[], names2: LabelDelta[]): NameMerge {
const result: NameMerge = {
dead1: new Set(),
dead2: new Set(),
rename1: new Map<string, string>(),
rename2: new Map<string, string>(),
merge: new Array<LabelDelta>(),
};
const names1ByFinalName: {[name: string]: LabelDelta} = keyBy(names1, p => p[1]!);
const names2ByInitialName: {[name: string]: LabelDelta} = keyBy(names2, p => p[0]!);
for (const [before1, after1] of names1) {
if (!after1) {
if (!before1) { throw new Error("invalid name change found"); }
// Table/column was deleted in part 1.
result.dead1.add(before1);
result.merge.push([before1, null]);
continue;
}
// At this point, we know the table/column existed at end of part 1.
const pair2 = names2ByInitialName[after1];
if (!pair2) {
// Table/column's name was stable in part 2, so only change was in part 1.
result.merge.push([before1, after1]);
continue;
}
const after2 = pair2[1];
if (!after2) {
// Table/column was deleted in part 2.
result.dead2.add(after1);
if (before1) {
// Table/column existed prior to part 1, so we need to expose its history.
result.dead1.add(before1);
result.merge.push([before1, null]);
} else {
// Table/column did not exist prior to part 1, so we erase it from history.
result.dead1.add(after1);
result.dead2.add(defunctTableName(after1));
}
continue;
}
// It we made it this far, our table/column exists after part 2. Any information
// keyed to its name in part 1 will need to be rekeyed to its final name.
result.rename1.set(after1, after2);
result.merge.push([before1, after2]);
}
// Look through part 2 for any changes not already covered. We won't need to do any
// renaming since table/column names at end of part 2 are just what we want.
for (const [before2, after2] of names2) {
if (!before2 && !after2) { throw new Error("invalid name change found"); }
if (before2 && names1ByFinalName[before2]) { continue; } // Already handled
result.merge.push([before2, after2]);
}
// For neatness, sort the merge order. Not essential.
result.merge = sortBy(result.merge, ([a, b]) => [a || "", b || ""]);
return result;
}
/**
* Re-key nested data to match name changes / removals. Needs to be done a little carefully
* since it is perfectly possible for names to be swapped or shuffled.
*
* Entries may be TableDeltas in the case of table renames or ColumnDeltas for column renames.
*
* @param entries: a dictionary of nested data - TableDeltas for tables, ColumnDeltas for columns.
* @param dead: a set of keys to remove from the dictionary.
* @param rename: changes of names to apply to the dictionary.
*/
function renameAndDelete<T>(entries: {[name: string]: T}, dead: Set<string>,
rename: Map<string, string>) {
// Remove all entries marked as dead.
for (const key of dead) { delete entries[key]; }
// Move all entries that are going to be renamed out to a cache temporarily.
const cache: {[name: string]: any} = {};
for (const key of rename.keys()) {
if (entries[key]) {
cache[key] = entries[key];
delete entries[key];
}
}
// Move all renamed entries back in with their new names.
for (const [key, val] of rename.entries()) { if (cache[key]) { entries[val] = cache[key]; } }
}
/**
* Apply planned name changes to a pair of entries, and return a merged entry encorporating
* their composition.
*
* @param names: the planned name changes as calculated by planNameMerge()
* @param entries1: the first dictionary of nested data keyed on the names
* @param entries2: test second dictionary of nested data keyed on the names
* @param mergeEntry: a function to apply any further corrections needed to the entries
*
*/
function mergeNames<T>(names: NameMerge,
entries1: {[name: string]: T},
entries2: {[name: string]: T},
mergeEntry: (e1: T, e2: T) => T): {[name: string]: T} {
// Update the keys of the entries1 and entries2 dictionaries to be consistent.
renameAndDelete(entries1, names.dead1, names.rename1);
renameAndDelete(entries2, names.dead2, names.rename2);
// Prepare the composition of the two dictionaries.
const entries = entries2; // Start with the second dictionary.
for (const key of Object.keys(entries1)) { // Add material from the first.
const e1 = entries1[key];
if (!entries[key]) { entries[key] = e1; continue; } // No overlap - just add and move on.
entries[key] = mergeEntry(e1, entries[key]); // Recursive merge if overlap.
}
return entries;
}
/**
* Track whether a specific row was added, removed or updated.
*/
interface RowChange {
added: boolean;
removed: boolean;
updated: boolean;
}
/** RowChange for each row in a table */
export interface RowChanges {
[rowId: number]: RowChange;
}
/**
* This is used when we hit a cell that we know has changed but don't know its
* value due to it being part of a bulk input. This produces a cell that
* represents the unknowns.
*/
function bulkCellFor(rc: RowChange|undefined): CellDelta|undefined {
if (!rc) { return undefined; }
const result: CellDelta = [null, null];
if (rc.removed || rc.updated) { result[0] = '?'; }
if (rc.added || rc.updated) { result[1] = '?'; }
return result;
}
/**
* Merge changes that apply to a particular column.
*
* @param present1: affected rows in part 1
* @param present2: affected rows in part 2
* @param e1: cached cell values for the column in part 1
* @param e2: cached cell values for the column in part 2
*/
function mergeColumn(present1: RowChanges, present2: RowChanges,
e1: ColumnDelta, e2: ColumnDelta): ColumnDelta {
for (const key of (Object.keys(present1) as unknown as number[])) {
let v1 = e1[key];
let v2 = e2[key];
if (!v1 && !v2) { continue; }
v1 = v1 || bulkCellFor(present1[key]);
v2 = v2 || bulkCellFor(present2[key]);
if (!v2) { e2[key] = e1[key]; continue; }
if (!v1[1]) { continue; } // Deleted row.
e2[key] = [v1[0], v2[1]]; // Change is from initial value in e1 to final value in e2.
}
return e2;
}
/** Put list of numbers in ascending order, with duplicates removed. */
function uniqueAndSorted(lst: number[]) {
return [...new Set(lst)].sort((a, b) => a - b);
}
/** For each row changed, figure out whether it was added/removed/updated */
/** TODO: need for this method suggests maybe a better core representation for this info */
function getRowChanges(e: TableDelta): RowChanges {
const all = new Set([...e.addRows, ...e.removeRows, ...e.updateRows]);
const added = new Set(e.addRows);
const removed = new Set(e.removeRows);
const updated = new Set(e.updateRows);
return fromPairs([...all].map(x => {
return [x, {added: added.has(x),
removed: removed.has(x),
updated: updated.has(x)}] as [number, RowChange];
}));
}
/**
* Merge changes that apply to a particular table. For updating addRows and removeRows, care is
* needed, since it is fine to remove and add the same rowId within a single summary -- this is just
* rowId reuse. It needs to be tracked so we know lifetime of rows though.
*/
function mergeTable(e1: TableDelta, e2: TableDelta): TableDelta {
// First, sort out any changes to names of columns.
const names = planNameMerge(e1.columnRenames, e2.columnRenames);
mergeNames(names, e1.columnDeltas, e2.columnDeltas,
mergeColumn.bind(null,
getRowChanges(e1),
getRowChanges(e2)));
e2.columnRenames = names.merge;
// All the columnar data is now merged. What remains is to merge the summary lists of rowIds
// that we maintain.
const addRows1 = new Set(e1.addRows); // Non-transient rows we have clearly added.
const removeRows2 = new Set(e2.removeRows); // Non-transient rows we have clearly removed.
const transients = e1.addRows.filter(x => removeRows2.has(x));
e2.addRows = uniqueAndSorted([...e2.addRows, ...e1.addRows.filter(x => !removeRows2.has(x))]);
e2.removeRows = uniqueAndSorted([...e2.removeRows.filter(x => !addRows1.has(x)), ...e1.removeRows]);
e2.updateRows = uniqueAndSorted([...e1.updateRows.filter(x => !removeRows2.has(x)),
...e2.updateRows.filter(x => !addRows1.has(x))]);
// Remove all traces of transients (rows that were created and destroyed) from history.
for (const cols of values(e2.columnDeltas)) {
for (const key of transients) { delete cols[key]; }
}
return e2;
}
/** Finally, merge a pair of summaries. */
export function concatenateSummaryPair(sum1: ActionSummary, sum2: ActionSummary): ActionSummary {
const names = planNameMerge(sum1.tableRenames, sum2.tableRenames);
const rowChanges = mergeNames(names, sum1.tableDeltas, sum2.tableDeltas, mergeTable);
const sum: ActionSummary = {
tableRenames: names.merge,
tableDeltas: rowChanges
};
return sum;
}
/** Generalize to merging a list of summaries. */
export function concatenateSummaries(sums: ActionSummary[]): ActionSummary {
if (sums.length === 0) { return createEmptyActionSummary(); }
let result = sums[0];
for (let i = 1; i < sums.length; i++) {
result = concatenateSummaryPair(result, sums[i]);
}
return result;
}

1132
app/server/lib/ActiveDoc.ts Normal file
View File

File diff suppressed because it is too large Load Diff

311
app/server/lib/ActiveDocImport.ts Normal file
View File

@@ -0,0 +1,311 @@
/* Helper file to separate ActiveDoc import functions and convert them to TypeScript. */
import * as path from 'path';
import * as _ from 'underscore';
import {DataSourceTransformed, ImportResult, ImportTableResult, TransformRuleMap} from 'app/common/ActiveDocAPI';
import {ApplyUAResult} from 'app/common/ActiveDocAPI';
import {ApiError} from 'app/common/ApiError';
import * as gutil from 'app/common/gutil';
import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
import {GristTable} from 'app/plugin/GristTable';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DocSession, OptDocSession} from 'app/server/lib/DocSession';
import * as log from 'app/server/lib/log';
import {globalUploadSet, moveUpload, UploadInfo} from 'app/server/lib/uploads';
/*
* AddTableRetValue contains return value of user actions 'AddTable'
*/
interface AddTableRetValue {
table_id: string;
id: number;
columns: string[];
views: object[];
}
interface ReferenceDescription {
// the table index
tableIndex: number;
// the column index
colIndex: number;
// the id of the table which is referenced
refTableId: string;
}
export class ActiveDocImport {
constructor(private _activeDoc: ActiveDoc) {}
/**
* Imports files, removes previously created temporary hidden tables and creates the new ones
*/
public async importFiles(docSession: DocSession, dataSource: DataSourceTransformed,
parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult> {
this._activeDoc.startBundleUserActions(docSession);
await this._removeHiddenTables(docSession, prevTableIds);
const userId = docSession.authorizer.getUserId();
const accessId = this._activeDoc.makeAccessId(userId);
const uploadInfo: UploadInfo = globalUploadSet.getUploadInfo(dataSource.uploadId, accessId);
return this._importFiles(docSession, uploadInfo, dataSource.transforms, parseOptions, true);
}
/**
* Finishes import files, removes temporary hidden tables, temporary uploaded files and creates
* the new tables
*/
public async finishImportFiles(docSession: DocSession, dataSource: DataSourceTransformed,
parseOptions: ParseOptions, prevTableIds: string[]): Promise<ImportResult> {
this._activeDoc.startBundleUserActions(docSession);
try {
await this._removeHiddenTables(docSession, prevTableIds);
const userId = docSession.authorizer.getUserId();
const accessId = this._activeDoc.makeAccessId(userId);
const uploadInfo: UploadInfo = globalUploadSet.getUploadInfo(dataSource.uploadId, accessId);
const importResult = await this._importFiles(docSession, uploadInfo, dataSource.transforms,
parseOptions, false);
await globalUploadSet.cleanup(dataSource.uploadId);
return importResult;
} finally {
this._activeDoc.stopBundleUserActions(docSession);
}
}
/**
* Cancels import files, removes temporary hidden tables and temporary uploaded files
*
* @param {ActiveDoc} activeDoc: Instance of ActiveDoc.
* @param {DataSourceTransformed} dataSource: an array of DataSource
* @param {Array<String>} prevTableIds: Array of tableIds as received from previous `importFiles`
* call when re-importing with changed `parseOptions`.
* @returns {Promise} Promise that's resolved when all actions are applied successfully.
*/
public async cancelImportFiles(docSession: DocSession,
dataSource: DataSourceTransformed,
prevTableIds: string[]): Promise<void> {
await this._removeHiddenTables(docSession, prevTableIds);
this._activeDoc.stopBundleUserActions(docSession);
await globalUploadSet.cleanup(dataSource.uploadId);
}
/**
* Import the given upload as new tables in one step. This does not give the user a chance to
* modify parse options or transforms. The caller is responsible for cleaning up the upload.
*/
public async oneStepImport(docSession: OptDocSession, uploadInfo: UploadInfo): Promise<ImportResult> {
this._activeDoc.startBundleUserActions(docSession);
try {
return this._importFiles(docSession, uploadInfo, [], {}, false);
} finally {
this._activeDoc.stopBundleUserActions(docSession);
}
}
/**
* Imports all files as new tables, using the given transform rules and parse options.
* The isHidden flag indicates whether to create temporary hidden tables, or final ones.
*/
private async _importFiles(docSession: OptDocSession, upload: UploadInfo, transforms: TransformRuleMap[],
parseOptions: ParseOptions, isHidden: boolean): Promise<ImportResult> {
// Check that upload size is within the configured limits.
const limit = (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || Infinity;
const totalSize = upload.files.reduce((acc, f) => acc + f.size, 0);
if (totalSize > limit) {
throw new ApiError(`Imported files must not exceed ${gutil.byteString(limit)}`, 413);
}
// The upload must be within the plugin-accessible directory. Once moved, subsequent calls to
// moveUpload() will return without having to do anything.
await moveUpload(upload, this._activeDoc.docPluginManager.tmpDir());
const importResult: ImportResult = {options: parseOptions, tables: []};
for (const [index, file] of upload.files.entries()) {
// If we have a better guess for the file's extension, replace it in origName, to ensure
// that DocPluginManager has access to it to guess the best parser type.
let origName: string = file.origName;
if (file.ext) {
origName = path.basename(origName, path.extname(origName)) + file.ext;
}
const res = await this._importFileAsNewTable(docSession, index, file.absPath, origName,
parseOptions, isHidden, transforms[index] || {});
if (index === 0) {
// Returned parse options from the first file should be used for all files in one upload.
importResult.options = parseOptions = res.options;
}
importResult.tables.push(...res.tables);
}
return importResult;
}
/**
* Imports the data stored at tmpPath.
*
* Currently it starts a python parser (that relies on the messytables library) as a child process
* outside the sandbox, and supports xls(x), csv, txt, and perhaps some other formats. It may
* result in the import of multiple tables, in case of e.g. Excel formats.
* @param {ActiveDoc} activeDoc: Instance of ActiveDoc.
* @param {Number} dataSourceIdx: Index of original dataSourse corresponding to current imported file.
* @param {String} tmpPath: The path from of the original file.
* @param {String} originalFilename: Suggested name of the import file. It is sometimes used as a
* suggested table name, e.g. for csv imports.
* @param {String} options: Containing parseOptions as serialized JSON to pass to the import plugin.
* @param {Boolean} isHidden: Flag to indicate whether table is temporary and hidden or regular.
* @param {TransformRuleMap} transformRuleMap: Containing transform rules for each table in file such as
* `destTableId`, `destCols`, `sourceCols`.
* @returns {Promise<ImportResult>} with `options` property containing parseOptions as serialized JSON as adjusted
* or guessed by the plugin, and `tables`, which is which is a list of objects with information about
* tables, such as `hiddenTableId`, `dataSourceIndex`, `origTableName`, `transformSectionRef`, `destTableId`.
*/
private async _importFileAsNewTable(docSession: OptDocSession, uploadFileIndex: number, tmpPath: string,
originalFilename: string,
options: ParseOptions, isHidden: boolean,
transformRuleMap: TransformRuleMap|undefined): Promise<ImportResult> {
log.info("ActiveDoc._importFileAsNewTable(%s, %s)", tmpPath, originalFilename);
const optionsAndData: ParseFileResult = await this._activeDoc.docPluginManager.parseFile(tmpPath,
originalFilename, options);
options = optionsAndData.parseOptions;
const parsedTables = optionsAndData.tables;
const references = this._encodeReferenceAsInt(parsedTables);
const tables: ImportTableResult[] = [];
const fixedColumnIdsByTable: { [tableId: string]: string[]; } = {};
for (const table of parsedTables) {
const ext = path.extname(originalFilename);
const basename = path.basename(originalFilename, ext).trim();
const hiddenTableName = 'GristHidden_import';
const origTableName = table.table_name ? table.table_name : '';
const transformRule = transformRuleMap && transformRuleMap.hasOwnProperty(origTableName) ?
transformRuleMap[origTableName] : null;
const result: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
[["AddTable", hiddenTableName, table.column_metadata]]);
const retValue: AddTableRetValue = result.retValues[0];
const hiddenTableId = retValue.table_id; // The sanitized version of the table name.
const hiddenTableColIds = retValue.columns; // The sanitized names of the columns.
// The table_data received from importFile is an array of columns of data, rather than a
// dictionary, so that it doesn't depend on column names. We instead construct the
// dictionary once we receive the sanitized column names from AddTable.
const dataLength = table.table_data[0] ? table.table_data[0].length : 0;
log.info("Importing table %s, %s rows, from %s", hiddenTableId, dataLength, table.table_name);
const rowIdColumn = _.range(1, dataLength + 1);
const columnValues = _.object(hiddenTableColIds, table.table_data);
const destTableId = transformRule ? transformRule.destTableId : null;
const ruleCanBeApplied = (transformRule != null) &&
_.difference(transformRule.sourceCols, hiddenTableColIds).length === 0;
await this._activeDoc.applyUserActions(docSession,
[["ReplaceTableData", hiddenTableId, rowIdColumn, columnValues]]);
// data parsed and put into hiddenTableId
// For preview_table (isHidden) do GenImporterView to make views and formulas and cols
// For final import, call TransformAndFinishImport, which imports file using a transform rule (or blank)
let createdTableId: string;
let transformSectionRef: number = -1; // TODO: we only have this if we genImporterView, is it necessary?
if (isHidden) {
// Generate formula columns, view sections, etc
const results: ApplyUAResult = await this._activeDoc.applyUserActions(docSession,
[['GenImporterView', hiddenTableId, destTableId, ruleCanBeApplied ? transformRule : null]]);
transformSectionRef = results.retValues[0];
createdTableId = hiddenTableId;
} else {
// Do final import
const intoNewTable: boolean = destTableId ? false : true;
const destTable = destTableId || table.table_name || basename;
const tableId = await this._activeDoc.applyUserActions(docSession,
[['TransformAndFinishImport',
hiddenTableId, destTable, intoNewTable,
ruleCanBeApplied ? transformRule : null]]);
createdTableId = tableId.retValues[0]; // this is garbage for now I think?
}
fixedColumnIdsByTable[createdTableId] = hiddenTableColIds;
tables.push({
hiddenTableId: createdTableId, // TODO: rename thing?
uploadFileIndex,
origTableName,
transformSectionRef, // TODO: this shouldnt always be needed, and we only get it if genimporttransform
destTableId
});
}
await this._fixReferences(docSession, parsedTables, tables, fixedColumnIdsByTable, references, isHidden);
return ({options, tables});
}
/**
* This function removes temporary hidden tables which were created during the import process
*
* @param {Array[String]} hiddenTableIds: Array of hidden table ids
* @returns {Promise} Promise that's resolved when all actions are applied successfully.
*/
private async _removeHiddenTables(docSession: DocSession, hiddenTableIds: string[]) {
if (hiddenTableIds.length !== 0) {
await this._activeDoc.applyUserActions(docSession, hiddenTableIds.map(t => ['RemoveTable', t]));
}
}
/**
* The methods changes every column of references into a column of integers in `parsedTables`. It
* returns `parsedTable` and a list of descriptors of all columns of references.
*/
private _encodeReferenceAsInt(parsedTables: GristTable[]): ReferenceDescription[] {
const references = [];
for (const [tableIndex, parsedTable] of parsedTables.entries()) {
for (const [colIndex, col] of parsedTable.column_metadata.entries()) {
const refTableId = gutil.removePrefix(col.type, "Ref:");
if (refTableId) {
references.push({refTableId, colIndex, tableIndex});
col.type = 'Int';
}
}
}
return references;
}
/**
* This function fix references that are broken by the change of table id.
*/
private async _fixReferences(docSession: OptDocSession,
parsedTables: GristTable[],
tables: ImportTableResult[],
fixedColumnIds: { [tableId: string]: string[]; },
references: ReferenceDescription[],
isHidden: boolean) {
// collect all new table ids
const tablesByOrigName = _.indexBy(tables, 'origTableName');
// gather all of the user actions
let userActions: any[] = references.map( ref => {
const fixedTableId = tables[ref.tableIndex].hiddenTableId;
return [
'ModifyColumn',
fixedTableId,
fixedColumnIds[fixedTableId][ref.colIndex],
{ type: `Ref:${tablesByOrigName[ref.refTableId].hiddenTableId}` }
];
});
if (isHidden) {
userActions = userActions.concat(userActions.map(([, tableId, columnId, colInfo]) => [
'ModifyColumn', tableId, 'gristHelper_Import_' + columnId, colInfo ]));
}
// apply user actions
if (userActions.length) {
await this._activeDoc.applyUserActions(docSession, userActions);
}
}
}

254
app/server/lib/AppEndpoint.ts Normal file
View File

@@ -0,0 +1,254 @@
/**
* AppServer serves up the main app.html file to the browser. It is the first point of contact of
* a browser with Grist. It handles sessions, redirect-to-login, and serving up a suitable version
* of the client-side code.
*/
import * as express from 'express';
import fetch, {RequestInit, Response as FetchResponse} from 'node-fetch';
import {ApiError} from 'app/common/ApiError';
import {getSlugIfNeeded, isOrgInPathOnly,
parseSubdomainStrictly} from 'app/common/gristUrls';
import {removeTrailingSlash} from 'app/common/gutil';
import {Document as APIDocument} from 'app/common/UserAPI';
import {Document} from "app/gen-server/entity/Document";
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, getTransitiveHeaders, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {expressWrap} from 'app/server/lib/expressWrap';
import {getAssignmentId} from 'app/server/lib/idUtils';
import * as log from 'app/server/lib/log';
import {adaptServerUrl, pruneAPIResult, trustOrigin} from 'app/server/lib/requestUtils';
import {ISendAppPageOptions} from 'app/server/lib/sendAppPage';
export interface AttachOptions {
app: express.Application; // Express app to which to add endpoints
middleware: express.RequestHandler[]; // Middleware to apply for all endpoints
docWorkerMap: IDocWorkerMap|null;
sendAppPage: (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => Promise<void>;
dbManager: HomeDBManager;
}
/**
* This method transforms a doc worker's public url as needed based on the request.
*
* For historic reasons, doc workers are assigned a public url at the time
* of creation. In production/staging, this is of the form:
* https://doc-worker-NNN-NNN-NNN-NNN.getgrist.com/v/VVVV/
* and in dev:
* http://localhost:NNNN/v/VVVV/
*
* Prior to support for different base domains, this was fine. Now that different
* base domains are supported, a wrinkle arises. When a web client communicates
* with a doc worker, it is important that it accesses the doc worker via a url
* containing the same base domain as the web page the client is on (for cookie
* purposes). Hence this method.
*
* If both the request and docWorkerUrl contain identifiable base domains (not localhost),
* then the base domain of docWorkerUrl is replaced with that of the request.
*
* But wait, there's another wrinkle: custom domains. In this case, we have a single
* domain available to serve a particular org from. This method will use the origin of req
* and include a /dw/doc-worker-NNN-NNN-NNN-NNN/
* (or /dw/local-NNNN/) prefix in all doc worker paths. Once this is in place, it
* will allow doc worker routing to be changed so it can be overlaid on a custom
* domain.
*
* TODO: doc worker registration could be redesigned to remove the assumption
* of a fixed base domain.
*/
function customizeDocWorkerUrl(docWorkerUrlSeed: string, req: express.Request) {
const docWorkerUrl = new URL(docWorkerUrlSeed);
const workerSubdomain = parseSubdomainStrictly(docWorkerUrl.hostname).org;
adaptServerUrl(docWorkerUrl, req);
// We wish to migrate to routing doc workers by path, so insert a doc worker identifier
// in the path (if not already present).
if (!docWorkerUrl.pathname.startsWith('/dw/')) {
// When doc worker is localhost, the port number is necessary and sufficient for routing.
// Let's add a /dw/... prefix just for consistency.
const workerIdent = workerSubdomain || `local-${docWorkerUrl.port}`;
docWorkerUrl.pathname = `/dw/${workerIdent}${docWorkerUrl.pathname}`;
}
return docWorkerUrl.href;
}
/**
*
* Gets the worker responsible for a given assignment, and fetches a url
* from the worker.
*
* If the fetch fails, we throw an exception, unless we see enough evidence
* to unassign the worker and try again.
*
* - If GRIST_MANAGED_WORKERS is set, we assume that we've arranged
* for unhealthy workers to be removed automatically, and that if a
* fetch returns a 404 with specific content, it is proof that the
* worker is no longer in existence. So if we see a 404 with that
* specific content, we can safely de-list the worker from redis,
* and repeat.
* - If GRIST_MANAGED_WORKERS is not set, we accept a broader set
* of failures as evidence of a missing worker.
*
* The specific content of a 404 that will be treated as evidence of
* a doc worker not being present is:
* - A json format body
* - With a key called "message"
* - With the value of "message" being "document worker not present"
* In production, this is provided by a special doc-worker-* load balancer
* rule.
*
*/
async function getWorker(docWorkerMap: IDocWorkerMap, assignmentId: string,
urlPath: string, config: RequestInit = {}) {
let docStatus: DocStatus|undefined;
const workersAreManaged = Boolean(process.env.GRIST_MANAGED_WORKERS);
for (;;) {
docStatus = await docWorkerMap.assignDocWorker(assignmentId);
const configWithTimeout = {timeout: 10000, ...config};
const fullUrl = removeTrailingSlash(docStatus.docWorker.internalUrl) + urlPath;
try {
const resp: FetchResponse = await fetch(fullUrl, configWithTimeout);
if (resp.ok) {
return {
resp,
docStatus,
};
}
if (resp.status === 403) {
throw new ApiError("You do not have access to this document.", resp.status);
}
if (resp.status !== 404) {
throw new ApiError(resp.statusText, resp.status);
}
let body: any;
try {
body = await resp.json();
} catch (e) {
throw new ApiError(resp.statusText, resp.status);
}
if (!(body && body.message && body.message === 'document worker not present')) {
throw new ApiError(resp.statusText, resp.status);
}
// This is a 404 with the expected content for a missing worker.
} catch (e) {
// If workers are managed, no errors merit continuing except a 404.
// Otherwise, we continue if we see a system error (e.g. ECONNREFUSED).
// We don't accept timeouts since there is too much potential to
// bring down a single-worker deployment that has a hiccup.
if (workersAreManaged || !(e.type === 'system')) {
throw e;
}
}
log.warn(`fetch from ${fullUrl} failed convincingly, removing that worker`);
await docWorkerMap.removeWorker(docStatus.docWorker.id);
docStatus = undefined;
}
}
export function attachAppEndpoint(options: AttachOptions): void {
const {app, middleware, docWorkerMap, sendAppPage, dbManager} = options;
// Per-workspace URLs open the same old Home page, and it's up to the client to notice and
// render the right workspace.
app.get(['/', '/ws/:wsId', '/p/:page'], ...middleware, expressWrap(async (req, res) =>
sendAppPage(req, res, {path: 'app.html', status: 200, config: {}, googleTagManager: 'anon'})));
app.get('/api/worker/:assignmentId([^/]+)/?*', expressWrap(async (req, res) => {
if (!trustOrigin(req, res)) { throw new Error('Unrecognized origin'); }
res.header("Access-Control-Allow-Credentials", "true");
if (!docWorkerMap) {
return res.status(500).json({error: 'no worker map'});
}
const assignmentId = getAssignmentId(docWorkerMap, req.params.assignmentId);
const {docStatus} = await getWorker(docWorkerMap, assignmentId, '/status');
if (!docStatus) {
return res.status(500).json({error: 'no worker'});
}
res.json({docWorkerUrl: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)});
}));
// Handler for serving the document landing pages. Expects the following parameters:
// urlId, slug (optional), remainder
// This handler is used for both "doc/urlId" and "urlId/slug" style endpoints.
const docHandler = expressWrap(async (req, res, next) => {
if (req.params.slug && req.params.slug === 'app.html') {
// This can happen on a single-port configuration, since "docId/app.html" matches
// the "urlId/slug" pattern. Luckily the "." character is not allowed in slugs.
return next();
}
if (!docWorkerMap) {
return await sendAppPage(req, res, {path: 'app.html', status: 200, config: {},
googleTagManager: 'anon'});
}
const mreq = req as RequestWithLogin;
const urlId = req.params.urlId;
let doc: Document|null = null;
try {
const userId = getUserId(mreq);
// Query DB for the doc metadata, to include in the page (as a pre-fetch of getDoc() call),
// and to get fresh (uncached) access info.
doc = await dbManager.getDoc({userId, org: mreq.org, urlId});
const slug = getSlugIfNeeded(doc);
const slugMismatch = (req.params.slug || null) !== (slug || null);
const preferredUrlId = doc.urlId || doc.id;
if (urlId !== preferredUrlId || slugMismatch) {
// Prepare to redirect to canonical url for document.
// Preserve org in url path if necessary.
const prefix = isOrgInPathOnly(req.hostname) ? `/o/${mreq.org}` : '';
// Preserve any query parameters or fragments.
const queryOrFragmentCheck = req.originalUrl.match(/([#?].*)/);
const queryOrFragment = (queryOrFragmentCheck && queryOrFragmentCheck[1]) || '';
if (slug) {
res.redirect(`${prefix}/${preferredUrlId}/${slug}${req.params.remainder}${queryOrFragment}`);
} else {
res.redirect(`${prefix}/doc/${preferredUrlId}${req.params.remainder}${queryOrFragment}`);
}
return;
}
// The docAuth value will be cached from the getDoc() above (or could be derived from doc).
const docAuth = await dbManager.getDocAuthCached({userId, org: mreq.org, urlId});
assertAccess('viewers', docAuth);
} catch (err) {
if (err.status === 404) {
log.info("/:urlId/app.html did not find doc", mreq.userId, urlId, doc && doc.access, mreq.org);
throw new ApiError('Document not found.', 404);
} else if (err.status === 403) {
log.info("/:urlId/app.html denied access", mreq.userId, urlId, doc && doc.access, mreq.org);
throw new ApiError('You do not have access to this document.', 403);
}
throw err;
}
// The reason to pass through app.html fetched from docWorker is in case it is a different
// version of Grist (could be newer or older).
// TODO: More must be done for correct version tagging of URLs: <base href> assumes all
// links and static resources come from the same host, but we'll have Home API, DocWorker,
// and static resources all at hostnames different from where this page is served.
// TODO docWorkerMain needs to serve app.html, perhaps with correct base-href already set.
const docId = doc.id;
const headers = {
Accept: 'application/json',
...getTransitiveHeaders(req),
};
const {docStatus, resp} = await getWorker(docWorkerMap, docId,
`/${docId}/app.html`, {headers});
const body = await resp.json();
await sendAppPage(req, res, {path: "", content: body.page, tag: body.tag, status: 200,
googleTagManager: 'anon', config: {
assignmentId: docId,
getWorker: {[docId]: customizeDocWorkerUrl(docStatus.docWorker.publicUrl, req)},
getDoc: {[docId]: pruneAPIResult(doc as unknown as APIDocument)},
}});
});
// The * is a wildcard in express 4, rather than a regex symbol.
// See https://expressjs.com/en/guide/routing.html
app.get('/doc/:urlId([^/]+):remainder(*)', ...middleware, docHandler);
app.get('/:urlId([^/]{12,})/:slug([^/]+):remainder(*)',
...middleware, docHandler);
}

429
app/server/lib/Authorizer.ts Normal file
View File

@@ -0,0 +1,429 @@
import {ApiError} from 'app/common/ApiError';
import {OpenDocMode} from 'app/common/DocListAPI';
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {canEdit, canView, getWeakestRole, Role} from 'app/common/roles';
import {Document} from 'app/gen-server/entity/Document';
import {User} from 'app/gen-server/entity/User';
import {DocAuthKey, DocAuthResult, HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {getSessionProfiles, getSessionUser, linkOrgWithEmail, SessionObj,
SessionUserObj} from 'app/server/lib/BrowserSession';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {COOKIE_MAX_AGE, getAllowedOrgForSessionID} from 'app/server/lib/gristSessions';
import * as log from 'app/server/lib/log';
import {IPermitStore, Permit} from 'app/server/lib/Permit';
import {allowHost} from 'app/server/lib/requestUtils';
import {NextFunction, Request, RequestHandler, Response} from 'express';
export interface RequestWithLogin extends Request {
sessionID: string;
session: SessionObj;
org?: string;
isCustomHost?: boolean; // when set, the request's domain is a recognized custom host linked
// with the specified org.
users?: UserProfile[];
userId?: number;
user?: User;
userIsAuthorized?: boolean; // If userId is for "anonymous", this will be false.
docAuth?: DocAuthResult; // For doc requests, the docId and the user's access level.
specialPermit?: Permit;
}
/**
* Extract the user id from a request, assuming we've added it via appropriate middleware.
* Throws ApiError with code 401 (unauthorized) if the user id is missing.
*/
export function getUserId(req: Request): number {
const userId = (req as RequestWithLogin).userId;
if (!userId) {
throw new ApiError("user not known", 401);
}
return userId;
}
/**
* Extract the user object from a request, assuming we've added it via appropriate middleware.
* Throws ApiError with code 401 (unauthorized) if the user is missing.
*/
export function getUser(req: Request): User {
const user = (req as RequestWithLogin).user;
if (!user) {
throw new ApiError("user not known", 401);
}
return user;
}
/**
* Extract the user profiles from a request, assuming we've added them via appropriate middleware.
* Throws ApiError with code 401 (unauthorized) if the profiles are missing.
*/
export function getUserProfiles(req: Request): UserProfile[] {
const users = (req as RequestWithLogin).users;
if (!users) {
throw new ApiError("user profile not found", 401);
}
return users;
}
// Extract the user id from a request, requiring it to be authorized (not an anonymous session).
export function getAuthorizedUserId(req: Request) {
const userId = getUserId(req);
if (isAnonymousUser(req)) {
throw new ApiError("user not authorized", 401);
}
return userId;
}
export function isAnonymousUser(req: Request) {
return !(req as RequestWithLogin).userIsAuthorized;
}
// True if Grist is configured for a single user without specific authorization
// (classic standalone/electron mode).
export function isSingleUserMode(): boolean {
return process.env.GRIST_SINGLE_USER === '1';
}
/**
* Returns the express request object with user information added, if it can be
* found based on passed in headers or the session. Specifically, sets:
* - req.userId: the id of the user in the database users table
* - req.userIsAuthorized: set if user has presented credentials that were accepted
* (the anonymous user has a userId but does not have userIsAuthorized set if,
* as would typically be the case, credentials were not presented)
* - req.users: set for org-and-session-based logins, with list of profiles in session
*/
export async function addRequestUser(dbManager: HomeDBManager, permitStore: IPermitStore,
fallbackEmail: string|null,
req: Request, res: Response, next: NextFunction) {
const mreq = req as RequestWithLogin;
let profile: UserProfile|undefined;
// First, check for an apiKey
if (mreq.headers && mreq.headers.authorization) {
// header needs to be of form "Bearer XXXXXXXXX" to apply
const parts = String(mreq.headers.authorization).split(' ');
if (parts[0] === "Bearer") {
const user = parts[1] ? await dbManager.getUserByKey(parts[1]) : undefined;
if (!user) {
return res.status(401).send('Bad request: invalid API key');
}
if (user.id === dbManager.getAnonymousUserId()) {
// We forbid the anonymous user to present an api key. That saves us
// having to think through the consequences of authorized access to the
// anonymous user's profile via the api (e.g. how should the api key be managed).
return res.status(401).send('Credentials cannot be presented for the anonymous user account via API key');
}
mreq.user = user;
mreq.userId = user.id;
mreq.userIsAuthorized = true;
}
}
// Special permission header for internal housekeeping tasks
if (mreq.headers && mreq.headers.permit) {
const permitKey = String(mreq.headers.permit);
try {
const permit = await permitStore.getPermit(permitKey);
if (!permit) { return res.status(401).send('Bad request: unknown permit'); }
mreq.user = dbManager.getAnonymousUser();
mreq.userId = mreq.user.id;
mreq.specialPermit = permit;
} catch (err) {
log.error(`problem reading permit: ${err}`);
return res.status(401).send('Bad request: permit could not be read');
}
}
// A bit of extra info we'll add to the "Auth" log message when this request passes the check
// for custom-host-specific sessionID.
let customHostSession = '';
// If we haven't selected a user by other means, and have profiles available in the
// session, then select a user based on those profiles.
const session = mreq.session;
if (!mreq.userId && session && session.users && session.users.length > 0 &&
mreq.org !== undefined) {
// Prevent using custom-domain sessionID to authorize to a different domain, since
// custom-domain owner could hijack such sessions.
const allowedOrg = getAllowedOrgForSessionID(mreq.sessionID);
if (allowedOrg) {
if (allowHost(req, allowedOrg.host)) {
customHostSession = ` custom-host-match ${allowedOrg.host}`;
} else {
// We need an exception for internal forwarding from home server to doc-workers. These use
// internal hostnames, so we can't expect a custom domain. These requests do include an
// Organization header, which we'll use to grant the exception, but security issues remain.
// TODO Issue 1: an attacker can use a custom-domain request to get an API key, which is an
// open door to all orgs accessible by this user.
// TODO Issue 2: Organization header is easy for an attacker (who has stolen a session
// cookie) to include too; it does nothing to prove that the request is internal.
const org = req.header('organization');
if (org && org === allowedOrg.org) {
customHostSession = ` custom-host-fwd ${org}`;
} else {
// Log error and fail.
log.warn("Auth[%s]: sessionID for host %s org %s; wrong for host %s org %s", mreq.method,
allowedOrg.host, allowedOrg.org, mreq.get('host'), mreq.org);
return res.status(403).send('Bad request: invalid session ID');
}
}
}
mreq.users = getSessionProfiles(session);
// If we haven't set a maxAge yet, set it now.
if (session && session.cookie && !session.cookie.maxAge) {
session.cookie.maxAge = COOKIE_MAX_AGE;
}
// See if we have a profile linked with the active organization already.
let sessionUser: SessionUserObj|null = getSessionUser(session, mreq.org);
if (!sessionUser) {
// No profile linked yet, so let's elect one.
// Choose a profile that is no worse than the others available.
const option = await dbManager.getBestUserForOrg(mreq.users, mreq.org);
if (option) {
// Modify request session object to link the current org with our choice of
// profile. Express-session will save this change.
sessionUser = linkOrgWithEmail(session, option.email, mreq.org);
// In this special case of initially linking a profile, we need to look up the user's info.
mreq.user = await dbManager.getUserByLogin(option.email);
mreq.userId = option.id;
mreq.userIsAuthorized = true;
} else {
// No profile has access to this org. We could choose to
// link no profile, in which case user will end up
// immediately presented with a sign-in page, or choose to
// link an arbitrary profile (say, the first one the user
// logged in as), in which case user will end up with a
// friendlier page explaining the situation and offering to
// add an account to resolve it. We go ahead and pick an
// arbitrary profile.
sessionUser = session.users[0];
if (!session.orgToUser) { throw new Error("Session misconfigured"); }
// Express-session will save this change.
session.orgToUser[mreq.org] = 0;
}
}
profile = sessionUser && sessionUser.profile || undefined;
// If we haven't computed a userId yet, check for one using an email address in the profile.
// A user record will be created automatically for emails we've never seen before.
if (profile && !mreq.userId) {
const user = await dbManager.getUserByLoginWithRetry(profile.email, profile);
if (user) {
mreq.user = user;
mreq.userId = user.id;
mreq.userIsAuthorized = true;
}
}
}
if (!mreq.userId && fallbackEmail) {
const user = await dbManager.getUserByLogin(fallbackEmail);
if (user) {
mreq.user = user;
mreq.userId = user.id;
mreq.userIsAuthorized = true;
const fullUser = dbManager.makeFullUser(user);
mreq.users = [fullUser];
profile = fullUser;
}
}
// If no userId has been found yet, fall back on anonymous.
if (!mreq.userId) {
const anon = dbManager.getAnonymousUser();
mreq.user = anon;
mreq.userId = anon.id;
mreq.userIsAuthorized = false;
mreq.users = [dbManager.makeFullUser(anon)];
}
log.debug("Auth[%s]: id %s email %s host %s path %s org %s%s", mreq.method,
mreq.userId, profile && profile.email, mreq.get('host'), mreq.path, mreq.org,
customHostSession);
return next();
}
/**
* Middleware to redirects user to a login page when the user is not
* logged in. If allowExceptions is set, then we make an exception
* for a team site allowing anonymous access, or a personal doc
* allowing anonymous access, or the merged org.
*/
export function redirectToLogin(
allowExceptions: boolean,
getLoginRedirectUrl: (redirectUrl: URL) => Promise<string>,
getSignUpRedirectUrl: (redirectUrl: URL) => Promise<string>,
dbManager: HomeDBManager
): RequestHandler {
return async (req: Request, resp: Response, next: NextFunction) => {
const mreq = req as RequestWithLogin;
mreq.session.alive = true; // This will ensure that express-session will set our cookie
// if it hasn't already - we'll need it if we redirect.
if (mreq.userIsAuthorized) { return next(); }
try {
// Otherwise it's an anonymous user. Proceed normally only if the org allows anon access.
if (mreq.userId && mreq.org && allowExceptions) {
// Anonymous user has qualified access to merged org.
if (dbManager.isMergedOrg(mreq.org)) { return next(); }
const result = await dbManager.getOrg({userId: mreq.userId}, mreq.org || null);
if (result.status === 200) { return next(); }
}
// In all other cases (including unknown org), redirect user to login or sign up.
// Redirect to sign up if it doesn't look like the user has ever logged in (on
// this browser) After logging in, `users` will be set in the session. Even after
// logging out again, `users` will still be set.
const signUp: boolean = (mreq.session.users === undefined);
log.debug(`Authorizer: redirecting to ${signUp ? 'sign up' : 'log in'}`);
const redirectUrl = new URL(req.protocol + '://' + req.get('host') + req.originalUrl);
if (signUp) {
return resp.redirect(await getSignUpRedirectUrl(redirectUrl));
} else {
return resp.redirect(await getLoginRedirectUrl(redirectUrl));
}
} catch (err) {
log.info("Authorizer failed to redirect", err.message);
return resp.status(401).send(err.message);
}
};
}
/**
* Sets mreq.docAuth if not yet set, and returns it.
*/
export async function getOrSetDocAuth(
mreq: RequestWithLogin, dbManager: HomeDBManager, urlId: string
): Promise<DocAuthResult> {
if (!mreq.docAuth) {
let effectiveUserId = getUserId(mreq);
if (mreq.specialPermit && mreq.userId === dbManager.getAnonymousUserId()) {
effectiveUserId = dbManager.getPreviewerUserId();
}
mreq.docAuth = await dbManager.getDocAuthCached({urlId, userId: effectiveUserId, org: mreq.org});
if (mreq.specialPermit && mreq.userId === dbManager.getAnonymousUserId() &&
mreq.specialPermit.docId === mreq.docAuth.docId) {
mreq.docAuth = {...mreq.docAuth, access: 'owners'};
}
}
return mreq.docAuth;
}
export interface ResourceSummary {
kind: 'doc';
id: string|number;
}
/**
*
* Handle authorization for a single resource accessed by a given user.
*
*/
export interface Authorizer {
// get the id of user, or null if no authorization in place.
getUserId(): number|null;
// Fetch the doc metadata from HomeDBManager.
getDoc(): Promise<Document>;
// Check access, throw error if the requested level of access isn't available.
assertAccess(role: 'viewers'|'editors'): Promise<void>;
}
/**
*
* Handle authorization for a single document and user.
*
*/
export class DocAuthorizer implements Authorizer {
constructor(
private _dbManager: HomeDBManager,
private _key: DocAuthKey,
public readonly openMode: OpenDocMode,
) {
}
public getUserId(): number {
return this._key.userId;
}
public async getDoc(): Promise<Document> {
return this._dbManager.getDoc(this._key);
}
public async assertAccess(role: 'viewers'|'editors'): Promise<void> {
const docAuth = await this._dbManager.getDocAuthCached(this._key);
assertAccess(role, docAuth, {openMode: this.openMode});
}
}
export class DummyAuthorizer implements Authorizer {
constructor(public role: Role|null) {}
public getUserId() { return null; }
public async getDoc(): Promise<Document> { throw new Error("Not supported in standalone"); }
public async assertAccess() { /* noop */ }
}
export function assertAccess(
role: 'viewers'|'editors', docAuth: DocAuthResult, options: {
openMode?: OpenDocMode,
allowRemoved?: boolean,
} = {}) {
const openMode = options.openMode || 'default';
const details = {status: 403, accessMode: openMode};
if (docAuth.error) {
if ([400, 401, 403].includes(docAuth.error.status)) {
// For these error codes, we know our access level - forbidden. Make errors more uniform.
throw new ErrorWithCode("AUTH_NO_VIEW", "No view access", details);
}
throw docAuth.error;
}
if (docAuth.removed && !options.allowRemoved) {
throw new ErrorWithCode("AUTH_NO_VIEW", "Document is deleted", {status: 404});
}
// If docAuth has no error, the doc is accessible, but we should still check the level (in case
// it's possible to access the doc with a level less than "viewer").
if (!canView(docAuth.access)) {
throw new ErrorWithCode("AUTH_NO_VIEW", "No view access", details);
}
if (role === 'editors') {
// If opening in a fork or view mode, treat user as viewer and deny write access.
const access = (openMode === 'fork' || openMode === 'view') ?
getWeakestRole('viewers', docAuth.access) : docAuth.access;
if (!canEdit(access)) {
throw new ErrorWithCode("AUTH_NO_EDIT", "No write access", details);
}
}
}
/**
* Pull out headers to pass along to a proxied service. Focussed primarily on
* authentication.
*/
export function getTransitiveHeaders(req: Request): {[key: string]: string} {
const Authorization = req.get('Authorization');
const Cookie = req.get('Cookie');
const PermitHeader = req.get('Permit');
const Organization = (req as RequestWithOrg).org;
return {
...(Authorization ? { Authorization } : undefined),
...(Cookie ? { Cookie } : undefined),
...(Organization ? { Organization } : undefined),
...(PermitHeader ? { Permit: PermitHeader } : undefined),
};
}

225
app/server/lib/BrowserSession.ts Normal file
View File

@@ -0,0 +1,225 @@
import {normalizeEmail} from 'app/common/emails';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {SessionStore} from 'app/server/lib/gristSessions';
import * as log from 'app/server/lib/log';
// Part of a session related to a single user.
export interface SessionUserObj {
// a grist-internal identify for the user, if known.
userId?: number;
// The user profile object. When updated, all clients get a message with the update.
profile?: UserProfile;
// Authentication provider string indicating the login method used.
authProvider?: string;
// Login ID token used to access AWS services.
idToken?: string;
// Login access token used to access other AWS services.
accessToken?: string;
// Login refresh token used to retrieve new ID and access tokens.
refreshToken?: string;
}
// Session state maintained for a particular browser. It is identified by a cookie. There may be
// several browser windows/tabs that share this cookie and this state.
export interface SessionObj {
// Session cookie.
// This is marked optional to reflect the reality of pre-existing code.
cookie?: any;
// A list of users we have logged in as.
// This is optional since the session may already exist.
users?: SessionUserObj[];
// map from org to an index into users[]
// This is optional since the session may already exist.
orgToUser?: {[org: string]: number};
// This gets set to encourage express-session to set a cookie.
alive?: boolean;
}
/**
* Extract the available user profiles from the session.
*
*/
export function getSessionProfiles(session: SessionObj): UserProfile[] {
if (!session.users) { return []; }
return session.users.filter(user => user && user.profile).map(user => user.profile!);
}
/**
*
* Gets user profile from the session for a given org, returning null if no profile is
* found specific to that org.
*
*/
export function getSessionUser(session: SessionObj, org: string): SessionUserObj|null {
if (!session.users) { return null; }
if (!session.users.length) { return null; }
if (session.orgToUser && session.orgToUser[org] !== undefined &&
session.users.length > session.orgToUser[org]) {
return session.users[session.orgToUser[org]] || null;
}
return null;
}
/**
*
* Record which user to use by default for a given org in future.
* This method mutates the session object passed to it. It does not save it,
* that is up to the caller.
*
*/
export function linkOrgWithEmail(session: SessionObj, email: string, org: string): SessionUserObj {
if (!session.users || !session.orgToUser) { throw new Error("Session not set up"); }
email = normalizeEmail(email);
for (let i = 0; i < session.users.length; i++) {
const iUser = session.users[i];
if (iUser && iUser.profile && normalizeEmail(iUser.profile.email) === email) {
session.orgToUser[org] = i;
return iUser;
}
}
throw new Error("Failed to link org with email");
}
/**
*
* This is a view of the session object, for a single organization (the "scope").
*
* Local caching is disabled in an enviroment where there is a home server (or we are
* the home server). In hosted Grist, per-instance caching would be a problem.
*
* We retain local caching for situations with a single server - especially electron.
*
*/
export class ScopedSession {
private _sessionCache?: SessionObj;
private _live: boolean; // if set, never cache session in memory.
/**
* Create an interface to the session identified by _sessionId, in the store identified
* by _sessionStore, for the organization identified by _scope.
*/
constructor(private _sessionId: string,
private _sessionStore: SessionStore,
private _org: string) {
// Assume we need to skip cache in a hosted environment. GRIST_HOST is always set there.
// TODO: find a cleaner way to configure this flag.
this._live = Boolean(process.env.GRIST_HOST || process.env.GRIST_HOSTED);
}
/**
* Get the user entry from the current session.
* @param prev: if supplied, this session object is used rather than querying the session again.
* @return the user entry
*/
public async getScopedSession(prev?: SessionObj): Promise<SessionUserObj> {
const session = prev || await this._getSession();
return getSessionUser(session, this._org) || {};
}
/**
*
* This performs an operation on the session object, limited to a single user entry. The state of that
* user entry before and after the operation are returned. LoginSession relies heavily on this method,
* to determine whether the change made by an operation merits certain follow-up work.
*
* @param op: Operation to perform. Given a single user entry, and should return a single user entry.
* It is fine to modify the supplied user entry in place.
*
* @return a pair [prev, current] with the state of the single user entry before and after the operation.
*
*/
public async operateOnScopedSession(op: (user: SessionUserObj) =>
Promise<SessionUserObj>): Promise<[SessionUserObj, SessionUserObj]> {
const session = await this._getSession();
const user = await this.getScopedSession(session);
const oldUser = JSON.parse(JSON.stringify(user)); // Old version to compare against.
const newUser = await op(JSON.parse(JSON.stringify(user))); // Modify a scratch version.
if (Object.keys(newUser).length === 0) {
await this.clearScopedSession(session);
} else {
await this._updateScopedSession(newUser, session);
}
return [oldUser, newUser];
}
/**
* This clears the current user entry from the session.
* @param prev: if supplied, this session object is used rather than querying the session again.
*/
public async clearScopedSession(prev?: SessionObj): Promise<void> {
const session = prev || await this._getSession();
this._clearUser(session);
await this._setSession(session);
}
/**
* Read the state of the session.
*/
private async _getSession(): Promise<SessionObj> {
if (this._sessionCache) { return this._sessionCache; }
const session = ((await this._sessionStore.getAsync(this._sessionId)) || {}) as SessionObj;
if (!this._live) { this._sessionCache = session; }
return session;
}
/**
* Set the session to the supplied object.
*/
private async _setSession(session: SessionObj): Promise<void> {
try {
await this._sessionStore.setAsync(this._sessionId, session);
if (!this._live) { this._sessionCache = session; }
} catch (e) {
// (I've copied this from old code, not sure if continuing after a session save error is
// something existing code depends on?)
// Report and keep going. This ensures that the session matches what's in the sessionStore.
log.error(`ScopedSession[${this._sessionId}]: Error updating sessionStore: ${e}`);
}
}
/**
* Update the session with the supplied user entry, replacing anything for that user already there.
* @param user: user entry to insert in session
* @param prev: if supplied, this session object is used rather than querying the session again.
*
*/
private async _updateScopedSession(user: SessionUserObj, prev?: SessionObj): Promise<void> {
const profile = user.profile;
if (!profile) {
throw new Error("No profile available");
}
// We used to also check profile.email_verified, but we no longer create UserProfile objects
// unless the email is verified, so this check is no longer needed.
if (!profile.email) {
throw new Error("Profile has no email address");
}
const session = prev || await this._getSession();
if (!session.users) { session.users = []; }
if (!session.orgToUser) { session.orgToUser = {}; }
let index = session.users.findIndex(u => Boolean(u.profile && u.profile.email === profile.email));
if (index < 0) { index = session.users.length; }
session.orgToUser[this._org] = index;
session.users[index] = user;
await this._setSession(session);
}
/**
* This clears all user logins (not just the current login).
* In future, we may want to be able to log in and out selectively, slack style,
* but right now it seems confusing.
*/
private _clearUser(session: SessionObj): void {
session.users = [];
session.orgToUser = {};
}
}

362
app/server/lib/Client.ts Normal file
View File

@@ -0,0 +1,362 @@
import {ApiError} from 'app/common/ApiError';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {getLoginState, LoginState} from 'app/common/LoginState';
import {User} from 'app/gen-server/entity/User';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {Authorizer} from 'app/server/lib/Authorizer';
import {DocSession} from 'app/server/lib/DocSession';
import * as log from 'app/server/lib/log';
import {ILoginSession} from 'app/server/lib/ILoginSession';
import {shortDesc} from 'app/server/lib/shortDesc';
import * as crypto from 'crypto';
import * as moment from 'moment';
/// How many messages to accumulate for a disconnected client before booting it.
const clientMaxMissedMessages = 100;
export type ClientMethod = (client: Client, ...args: any[]) => Promise<any>;
/**
* Generates and returns a random string to use as a clientId. This is better
* than numbering clients with consecutive integers; otherwise a reconnecting
* client presenting the previous clientId to a restarted (new) server may
* accidentally associate itself with a wrong session that happens to share the
* same clientId. In other words, we need clientIds to be unique across server
* restarts.
* @returns {String} - random string to use as a new clientId.
*/
function generateClientId(): string {
// Non-blocking version of randomBytes may fail if insufficient entropy is available without
// blocking. If we encounter that, we could either block, or maybe use less random values.
return crypto.randomBytes(8).toString('hex');
}
/**
* These are the types of messages that are allowed to be sent to the client even if the client is
* not authorized to use this instance (e.g. not a member of the team for this subdomain).
*/
const MESSAGE_TYPES_NO_AUTH = new Set([
'clientConnect',
'profileFetch',
'userSettings',
'clientLogout',
]);
// tslint:disable-next-line:no-unused-expression Silence "unused variable" warning.
void(MESSAGE_TYPES_NO_AUTH);
/**
* Class that encapsulates the information for a client. A Client may survive
* across multiple websocket reconnects.
* TODO: this could provide a cleaner interface.
*
* @param comm: parent Comm object
* @param websocket: websocket connection, promisified to have a sendAsync method
* @param methods: a mapping from method names to server methods (must return promises)
*/
export class Client {
public readonly clientId: string;
public session: ILoginSession|null = null;
public browserSettings: BrowserSettings = {};
// Maps docFDs to DocSession objects.
private _docFDs: Array<DocSession|null> = [];
private _missedMessages: any = [];
private _destroyTimer: NodeJS.Timer|null = null;
private _destroyed: boolean = false;
private _websocket: any;
private _loginState: LoginState|null = null;
private _org: string|null = null;
private _profile: UserProfile|null = null;
private _userId: number|null = null;
private _firstLoginAt: Date|null = null;
private _isAnonymous: boolean = false;
// Identifier for the current GristWSConnection object connected to this client.
private _counter: string|null = null;
constructor(
private _comm: any,
private _methods: any,
private _host: string
) {
this.clientId = generateClientId();
}
public toString() { return `Client ${this.clientId} #${this._counter}`; }
// Returns the LoginState object that's encoded and passed via login pages to login-connect.
public getLoginState(): LoginState|null { return this._loginState; }
public setCounter(counter: string) {
this._counter = counter;
}
public get host(): string {
return this._host;
}
public setConnection(websocket: any, reqHost: string, browserSettings: BrowserSettings) {
this._websocket = websocket;
// Set this._loginState, used by CognitoClient to construct login/logout URLs.
this._loginState = getLoginState(reqHost);
this.browserSettings = browserSettings;
}
/**
* Returns DocSession for the given docFD, or throws an exception if this doc is not open.
*/
public getDocSession(fd: number): DocSession {
const docSession = this._docFDs[fd];
if (!docSession) {
throw new Error(`Invalid docFD ${fd}`);
}
return docSession;
}
// Adds a new DocSession to this Client, and returns the new FD for it.
public addDocSession(activeDoc: ActiveDoc, authorizer: Authorizer): DocSession {
const fd = this._getNextDocFD();
const docSession = new DocSession(activeDoc, this, fd, authorizer);
this._docFDs[fd] = docSession;
return docSession;
}
// Removes a DocSession from this Client, called when a doc is closed.
public removeDocSession(fd: number): void {
this._docFDs[fd] = null;
}
// Check that client still has access to all documents. Used to determine whether
// a Comm client can be safely reused after a reconnect. Without this check, the client
// would be reused even if access to a document has been lost (although an error would be
// issued later, on first use of the document).
public async isAuthorized(): Promise<boolean> {
for (const docFD of this._docFDs) {
try {
if (docFD !== null) { await docFD.authorizer.assertAccess('viewers'); }
} catch (e) {
return false;
}
}
return true;
}
/**
* Closes all docs.
*/
public closeAllDocs() {
let count = 0;
for (let fd = 0; fd < this._docFDs.length; fd++) {
const docSession = this._docFDs[fd];
if (docSession && docSession.activeDoc) {
// Note that this indirectly calls to removeDocSession(docSession.fd)
docSession.activeDoc.closeDoc(docSession)
.catch((e) => { log.warn("%s: error closing docFD %d", this, fd); });
count++;
}
this._docFDs[fd] = null;
}
log.debug("%s: closeAllDocs() closed %d doc(s)", this, count);
}
public interruptConnection() {
if (this._websocket) {
this._websocket.removeAllListeners();
this._websocket.terminate(); // close() is inadequate when ws routed via loadbalancer
this._websocket = null;
}
}
/**
* Sends a message to the client, queuing it up on failure or if the client is disconnected.
*/
public async sendMessage(messageObj: any): Promise<void> {
if (this._destroyed) {
return;
}
const message: string = JSON.stringify(messageObj);
// Log something useful about the message being sent.
if (messageObj.type) {
log.info("%s: sending %s: %d bytes", this, messageObj.type, message.length);
} else if (messageObj.error) {
log.warn("%s: responding to #%d ERROR %s", this, messageObj.reqId, messageObj.error);
} else {
log.info("%s: responding to #%d OK: %d bytes", this, messageObj.reqId, message.length);
}
if (this._websocket) {
// If we have a websocket, send the message.
try {
await this._websocket.sendAsync(message);
} catch (err) {
// Sending failed. Presumably we should be getting onClose around now too.
// NOTE: if this handler is run after onClose, we could have messages end up out of order.
// Let's check to make sure. If this can happen, we need to refactor for correct ordering.
if (!this._websocket) {
log.error("%s sendMessage: UNEXPECTED ORDER OF CALLBACKS", this);
}
log.warn("%s sendMessage: queuing after send error: %s", this, err.toString());
this._missedMessages.push(message);
}
} else if (this._missedMessages.length < clientMaxMissedMessages) {
// Queue up the message.
this._missedMessages.push(message);
} else {
// Too many messages queued. Boot the client now, to make it reset when/if it reconnects.
log.error("%s sendMessage: too many messages queued; booting client", this);
if (this._destroyTimer) {
clearTimeout(this._destroyTimer);
this._destroyTimer = null;
}
this._comm._destroyClient(this);
}
}
// Assigns the client to the given login session and the session to the client.
public setSession(session: ILoginSession): void {
this.unsetSession();
this.session = session;
session.clients.add(this);
}
// Unsets the current login session and removes the client from it.
public unsetSession(): void {
if (this.session) { this.session.clients.delete(this); }
this.session = null;
}
public destroy() {
this.unsetSession();
this._destroyed = true;
}
/**
* Processes a request from a client. All requests from a client get a response, at least to
* indicate success or failure.
*/
public async onMessage(message: string): Promise<void> {
const clientId = this.clientId;
const request = JSON.parse(message);
if (request.beat) {
// this is a heart beat, to keep the websocket alive. No need to reply.
log.rawInfo('heartbeat', {clientId, counter: this._counter, url: request.url});
return;
} else {
log.info("%s: onMessage", this, shortDesc(message));
}
const response: any = {reqId: request.reqId};
const method = this._methods[request.method];
if (!method) {
response.error = `Unknown method ${request.method}`;
} else {
try {
response.data = await method(this, ...request.args);
} catch (error) {
const err: ErrorWithCode = error;
// Print the error stack, except for SandboxErrors, for which the JS stack isn't that useful.
// Also not helpful is the stack of AUTH_NO_VIEW|EDIT errors produced by the Authorizer.
const code: unknown = err.code;
const skipStack = (
!err.stack ||
err.stack.match(/^SandboxError:/) ||
(typeof code === 'string' && code.startsWith('AUTH_NO'))
);
log.warn("%s: Error %s %s", this, skipStack ? err : err.stack, code || '');
response.error = err.message;
if (err.code) {
response.errorCode = err.code;
}
if (typeof code === 'string' && code === 'AUTH_NO_EDIT' && err.accessMode === 'fork') {
response.shouldFork = true;
}
}
}
await this.sendMessage(response);
}
public setOrg(org: string): void {
this._org = org;
}
public getOrg(): string {
return this._org!;
}
public setProfile(profile: UserProfile|null): void {
this._profile = profile;
// Unset userId, so that we look it up again on demand. (Not that userId could change in
// practice via a change to profile, but let's not make any assumptions here.)
this._userId = null;
this._firstLoginAt = null;
this._isAnonymous = false;
}
public getProfile(): UserProfile|null {
return this._profile;
}
public getCachedUserId(): number|null {
return this._userId;
}
public isAnonymous(): boolean {
return this._isAnonymous;
}
// Returns the userId for profile.email, or null when profile is not set; with caching.
public async getUserId(dbManager: HomeDBManager): Promise<number|null> {
if (!this._userId) {
const user = await this._fetchUser(dbManager);
this._userId = (user && user.id) || null;
this._isAnonymous = this._userId && dbManager.getAnonymousUserId() === this._userId || false;
this._firstLoginAt = (user && user.firstLoginAt) || null;
}
return this._userId;
}
// Returns the userId for profile.email, or throws 403 error when profile is not set.
public async requireUserId(dbManager: HomeDBManager): Promise<number> {
const userId = await this.getUserId(dbManager);
if (userId) { return userId; }
throw new ApiError(this._profile ? `user not known: ${this._profile.email}` : 'user not set', 403);
}
public getLogMeta() {
const meta: {[key: string]: any} = {};
if (this._profile) { meta.email = this._profile.email; }
// We assume the _userId has already been cached, which will be true always (for all practical
// purposes) because it's set when the Authorizer checks this client.
if (this._userId) { meta.userId = this._userId; }
// Likewise for _firstLoginAt, which we learn along with _userId.
if (this._firstLoginAt) {
meta.age = Math.floor(moment.duration(moment().diff(this._firstLoginAt)).asDays());
}
if (this._org) { meta.org = this._org; }
meta.clientId = this.clientId; // identifies a client connection, essentially a websocket
meta.counter = this._counter; // identifies a GristWSConnection in the connected browser tab
return meta;
}
// Fetch the user database record from profile.email, or null when profile is not set.
private async _fetchUser(dbManager: HomeDBManager): Promise<User|undefined> {
return this._profile && this._profile.email ?
await dbManager.getUserByLogin(this._profile.email) :
undefined;
}
// Returns the next unused docFD number.
private _getNextDocFD(): number {
let fd = 0;
while (this._docFDs[fd]) { fd++; }
return fd;
}
}

395
app/server/lib/Comm.js Normal file
View File

@@ -0,0 +1,395 @@
/**
* The server's Comm object implements communication with the client.
*
* The server receives requests, to which it sends a response (or an error). The server can
* also send asynchronous messages to the client. Available methods should be provided via
* comm.registerMethods().
*
* To send async messages, you may call broadcastMessage() or sendDocMessage().
*
* In practice, requests which modify the document are done via UserActions.js, and result in an
* asynchronous message updating the document (which is sent to all clients who have the document
* open), and the response could return some useful value, but does not have to.
*
* See app/client/components/Comm.js for other details of the communication protocol.
*
*
* Currently, this module also implements the concept of a "Client". A Client corresponds to a
* browser window, and should persist across brief disconnects. A Client has a 'clientId'
* property, which uniquely identifies a client within the currently running server. Method
* registered with Comm always receive a Client object as the first argument.
*
* In the future, we may want to have a separate Client.js file with documentation of the various
* properties that may be associated with a client.
*
* Note that users of this module should never use the websocket of a Client, since that's an
* implementation detail of Comm.js.
*/
/**
* Event for DocList changes.
* @event docListAction Emitted when the document list changes in any way.
* @property {Array[String]} [addDocs] Array of names of documents to add to the docList.
* @property {Array[String]} [removeDocs] Array of names of documents that got removed.
* @property {Array[String]} [renameDocs] Array of [oldName, newName] pairs for renamed docs.
* @property {Array[String]} [addInvites] Array of document invite names to add.
* @property {Array[String]} [removeInvites] Array of documents invite names to remove.
*/
var events = require('events');
var url = require('url');
var util = require('util');
var ws = require('ws');
var Promise = require('bluebird');
var log = require('./log');
var gutil = require('app/common/gutil');
const {parseFirstUrlPart} = require('app/common/gristUrls');
const version = require('app/common/version');
const {Client} = require('./Client');
// Bluebird promisification, to be able to use e.g. websocket.sendAsync method.
Promise.promisifyAll(ws.prototype);
/// How long the client state persists after a disconnect.
var clientRemovalTimeoutMsDefault = 300 * 1000; // 300s = 5 minutes.
var clientRemovalTimeoutMs = clientRemovalTimeoutMsDefault;
/**
* Constructs a Comm object.
* @param {Object} server - The HTTP server.
* @param {Object} options.sessions - A collection of sessions
* @param {Object} options.settings - The config object containing instance settings
* including features.
* @param {Object} options.instanceManager - Instance manager, giving access to InstanceStore
* and per-instance objects. If null, HubUserClient will not be created.
* @param {Object} options.hosts - Hosts object from extractOrg.ts. if set, we use
* hosts.getOrgInfo(req) to extract an organization from a (possibly versioned) url.
*/
function Comm(server, options) {
events.EventEmitter.call(this);
this._server = server;
this._httpsServer = options.httpsServer;
this.wss = this._startServer();
// Maps client IDs to websocket objects.
this._clients = {}; // Maps clientIds to Client objects.
this.clientList = []; // List of all active Clients, ordered by clientId.
// Maps sessionIds to LoginSession objects.
this.sessions = options.sessions;
this._settings = options.settings;
this._instanceManager = options.instanceManager;
this._hosts = options.hosts;
// This maps method names to their implementation.
this.methods = {};
// For testing, we need a way to override the server version reported.
// For upgrading, we use this to set the server version for a defunct server
// to "dead" so that a client will know that it needs to periodically recheck
// for a valid server.
this._serverVersion = null;
}
util.inherits(Comm, events.EventEmitter);
/**
* Registers server methods.
* @param {Object[String:Function]} Mapping of method name to their implementations. All methods
* receive the client as the first argument, and the arguments from the request.
*/
Comm.prototype.registerMethods = function(serverMethods) {
// Wrap methods to translate return values and exceptions to promises.
for (var methodName in serverMethods) {
this.methods[methodName] = Promise.method(serverMethods[methodName]);
}
};
/**
* Returns the Client object associated with the given clientId, or throws an Error if not found.
*/
Comm.prototype.getClient = function(clientId) {
const client = this._clients[clientId];
if (!client) { throw new Error('Unrecognized clientId'); }
return client;
};
/**
* Returns a LoginSession object with the given session id from the list of sessions,
* or adds a new one and returns that.
*/
Comm.prototype.getOrCreateSession = function(sid, req) {
// LoginSessions are specific to a session id / org combination.
const org = req.org || "";
return this.sessions.getOrCreateLoginSession(sid, org, this, this._instanceManager);
};
/**
* Returns the sessionId from the signed grist cookie.
*/
Comm.prototype.getSessionIdFromCookie = function(gristCookie) {
return this.sessions.getSessionIdFromCookie(gristCookie);
};
/**
* Broadcasts an app-level message to all clients.
* @param {String} type - Type of message, e.g. 'docListAction'.
* @param {Object} messageData - The data for this type of message.
*/
Comm.prototype.broadcastMessage = function(type, messageData) {
return this._broadcastMessage(type, messageData, this.clientList);
};
Comm.prototype._broadcastMessage = function(type, data, clients) {
clients.forEach(client => client.sendMessage({type, data}));
};
/**
* Sends a per-doc message to the given client.
* @param {Object} client - The client object, as passed to all per-doc methods.
* @param {Number} docFD - The document's file descriptor in the given client.
* @param {String} type - The type of the message, e.g. 'docUserAction'.
* @param {Object} messageData - The data for this type of message.
* @param {Boolean} fromSelf - Whether `client` is the originator of this message.
*/
Comm.sendDocMessage = function(client, docFD, type, data, fromSelf = undefined) {
client.sendMessage({type, docFD, data, fromSelf});
};
/**
* Processes a new websocket connection.
* TODO: Currently it always creates a new client, but in the future the creation of a client
* should possibly be delayed until some hello message, so that a previous client may reconnect
* without losing state.
*/
Comm.prototype._onWebSocketConnection = async function(websocket, req) {
log.info("Comm: Got WebSocket connection: %s", req.url);
if (this._hosts) {
// DocWorker ID (/dw/) and version tag (/v/) may be present in this request but are not
// needed. addOrgInfo assumes req.url starts with /o/ if present.
req.url = parseFirstUrlPart('dw', req.url).path;
req.url = parseFirstUrlPart('v', req.url).path;
await this._hosts.addOrgInfo(req);
}
websocket.on('error', this.onError.bind(this, websocket));
websocket.on('close', this.onClose.bind(this, websocket));
// message handler is added later, after we create a Client but before any async operations
// Parse the cookie in the request to get the sessionId.
var sessionId = this.sessions.getSessionIdFromRequest(req);
var urlObj = url.parse(req.url, true);
var existingClientId = urlObj.query.clientId;
var browserSettings = urlObj.query.browserSettings ? JSON.parse(urlObj.query.browserSettings) : {};
var newClient = (parseInt(urlObj.query.newClient, 10) === 1);
const counter = urlObj.query.counter;
// Associate an ID with each websocket, reusing the supplied one if it's valid.
var client;
if (existingClientId && this._clients.hasOwnProperty(existingClientId) &&
!this._clients[existingClientId]._websocket &&
await this._clients[existingClientId].isAuthorized()) {
client = this._clients[existingClientId];
client.setCounter(counter);
log.info("Comm %s: existing client reconnected (%d missed messages)", client,
client._missedMessages.length);
if (client._destroyTimer) {
log.warn("Comm %s: clearing scheduled destruction", client);
clearTimeout(client._destroyTimer);
client._destroyTimer = null;
}
if (newClient) {
// If this isn't a reconnect, then we assume that the browser client lost its state (e.g.
// reloaded the page), so we treat it as a disconnect followed by a new connection to the
// same state. At the moment, this only means that we close all docs.
if (client._missedMessages.length) {
log.warn("Comm %s: clearing missed messages for new client", client);
}
client._missedMessages.length = 0;
client.closeAllDocs();
}
client.setConnection(websocket, req.headers.host, browserSettings);
} else {
client = new Client(this, this.methods, req.headers.host);
client.setCounter(counter);
client.setConnection(websocket, req.headers.host, browserSettings);
this._clients[client.clientId] = client;
this.clientList.push(client);
log.info("Comm %s: new client", client);
}
websocket._commClient = client;
websocket.clientId = client.clientId;
// Add a Session object to the client.
log.info(`Comm ${client}: using session ${sessionId}`);
const loginSession = this.getOrCreateSession(sessionId, req);
client.setSession(loginSession);
// Delegate message handling to the client
websocket.on('message', client.onMessage.bind(client));
loginSession.getSessionProfile()
.then((profile) => {
log.debug(`Comm ${client}: sending clientConnect with ` +
`${client._missedMessages.length} missed messages`);
// Don't use sendMessage here, since we don't want to queue up this message on failure.
client.setOrg(req.org || "");
client.setProfile(profile);
const clientConnectMsg = {
type: 'clientConnect',
clientId: client.clientId,
serverVersion: this._serverVersion || version.gitcommit,
missedMessages: client._missedMessages.slice(0),
settings: this._settings,
profile,
};
// If reconnecting a client with missed messages, clear them now.
client._missedMessages.length = 0;
return websocket.sendAsync(JSON.stringify(clientConnectMsg))
// A heavy-handed fix to T396, since 'clientConnect' is sometimes not seen in the browser,
// (seemingly when the 'message' event is triggered before 'open' on the native WebSocket.)
// See also my report at https://stackoverflow.com/a/48411315/328565
.delay(250).then(() => {
if (client._destroyed) { return; } // object is already closed - don't show messages
if (websocket.readyState === websocket.OPEN) {
return websocket.sendAsync(JSON.stringify(Object.assign(clientConnectMsg, {dup: true})));
} else {
log.debug(`Comm ${client}: websocket closed right after clientConnect`);
}
});
})
.then(() => {
if (!client._destroyed) { log.debug(`Comm ${client}: clientConnect sent successfully`); }
})
.catch(err => {
log.error(`Comm ${client}: failed to prepare or send clientConnect:`, err);
});
};
/**
* Processes an error on the websocket.
*/
Comm.prototype.onError = function(websocket, err) {
log.warn("Comm cid %s: onError", websocket.clientId, err);
// TODO Make sure that this is followed by onClose when the connection is lost.
};
/**
* Processes the closing of a websocket.
*/
Comm.prototype.onClose = function(websocket) {
log.info("Comm cid %s: onClose", websocket.clientId);
websocket.removeAllListeners();
var client = websocket._commClient;
if (client) {
// Remove all references to the websocket.
client._websocket = null;
// Schedule the client to be destroyed after a timeout. The timer gets cleared if the same
// client reconnects in the interim.
if (client._destroyTimer) {
log.warn("Comm cid %s: clearing previously scheduled destruction", websocket.clientId);
clearTimeout(client._destroyTimer);
}
log.warn("Comm cid %s: will discard client in %s sec",
websocket.clientId, clientRemovalTimeoutMs / 1000);
client._destroyTimer = setTimeout(this._destroyClient.bind(this, client),
clientRemovalTimeoutMs);
}
};
Comm.prototype._startServer = function() {
const servers = [this._server];
if (this._httpsServer) { servers.push(this._httpsServer); }
const wss = [];
for (const server of servers) {
const wssi = new ws.Server({server});
wssi.on('connection', async (websocket, req) => {
try {
await this._onWebSocketConnection(websocket, req);
} catch (e) {
log.error("Comm connection for %s threw exception: %s", req.url, e.message);
websocket.removeAllListeners();
websocket.terminate(); // close() is inadequate when ws routed via loadbalancer
}
});
wss.push(wssi);
}
return wss;
};
Comm.prototype.testServerShutdown = async function() {
if (this.wss) {
for (const wssi of this.wss) {
await Promise.fromCallback((cb) => wssi.close(cb));
}
this.wss = null;
}
};
Comm.prototype.testServerRestart = async function() {
await this.testServerShutdown();
this.wss = this._startServer();
};
/**
* Destroy all clients, forcing reconnections.
*/
Comm.prototype.destroyAllClients = function() {
// Iterate over all clients. Take a copy of the list of clients since it will be changing
// during the loop as we remove them one by one.
for (const client of this.clientList.slice()) {
client.interruptConnection();
this._destroyClient(client);
}
};
/**
* Destroys a client. If the same browser window reconnects later, it will get a new Client
* object and clientId.
*/
Comm.prototype._destroyClient = function(client) {
log.info("Comm %s: client gone", client);
client.closeAllDocs();
if (client._destroyTimer) {
clearTimeout(client._destroyTimer);
}
delete this._clients[client.clientId];
gutil.arrayRemove(this.clientList, client);
client.destroy();
};
/**
* Override the version string Comm will report to clients.
* Call with null to reset the override.
*
*/
Comm.prototype.setServerVersion = function (serverVersion) {
this._serverVersion = serverVersion;
};
/**
* Mark the server as active or inactive. If inactive, any client that manages to
* connect to it will read a server version of "dead".
*/
Comm.prototype.setServerActivation = function (active) {
this._serverVersion = active ? null : 'dead';
};
/**
* Set how long clients persist on the server after disconnection. Call with
* 0 to return to the default.
*/
Comm.prototype.testSetClientPersistence = function (ttlMs) {
clientRemovalTimeoutMs = ttlMs || clientRemovalTimeoutMsDefault;
}
module.exports = Comm;

560
app/server/lib/DocApi.ts Normal file
View File

@@ -0,0 +1,560 @@
import { Application, NextFunction, Request, RequestHandler, Response } from "express";
import { ApiError } from 'app/common/ApiError';
import { BrowserSettings } from "app/common/BrowserSettings";
import { fromTableDataAction, TableColValues } from 'app/common/DocActions';
import { arrayRepeat } from "app/common/gutil";
import { SortFunc } from 'app/common/SortFunc';
import { DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import { HomeDBManager, makeDocAuthResult } from 'app/gen-server/lib/HomeDBManager';
import { ActiveDoc } from "app/server/lib/ActiveDoc";
import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, getUserId, isAnonymousUser,
RequestWithLogin } from 'app/server/lib/Authorizer';
import { DocManager } from "app/server/lib/DocManager";
import { DocWorker } from "app/server/lib/DocWorker";
import { expressWrap } from 'app/server/lib/expressWrap';
import { GristServer } from 'app/server/lib/GristServer';
import { makeForkIds } from "app/server/lib/idUtils";
import { getDocId, getDocScope, integerParam, isParameterOn, optStringParam,
sendOkReply, sendReply } from 'app/server/lib/requestUtils';
import { SandboxError } from "app/server/lib/sandboxUtil";
import { handleOptionalUpload, handleUpload } from "app/server/lib/uploads";
import * as contentDisposition from 'content-disposition';
import fetch from 'node-fetch';
import * as path from 'path';
// Cap on the number of requests that can be outstanding on a single document via the
// rest doc api. When this limit is exceeded, incoming requests receive an immediate
// reply with status 429.
const MAX_PARALLEL_REQUESTS_PER_DOC = 10;
type WithDocHandler = (activeDoc: ActiveDoc, req: RequestWithLogin, resp: Response) => Promise<void>;
/**
* Middleware to track the number of requests outstanding on each document, and to
* throw an exception when the maximum number of requests are already outstanding.
* Access to a document must already have been authorized.
*/
function apiThrottle(usage: Map<string, number>,
callback: (req: RequestWithLogin,
resp: Response,
next: NextFunction) => Promise<void>): RequestHandler {
return async (req, res, next) => {
const docId = getDocId(req);
try {
const count = usage.get(docId) || 0;
usage.set(docId, count + 1);
if (count + 1 > MAX_PARALLEL_REQUESTS_PER_DOC) {
throw new ApiError(`Too many backlogged requests for document ${docId} - ` +
`try again later?`, 429);
}
await callback(req as RequestWithLogin, res, next);
} catch (err) {
next(err);
} finally {
const count = usage.get(docId);
if (count) {
if (count === 1) {
usage.delete(docId);
} else {
usage.set(docId, count - 1);
}
}
}
};
}
export class DocWorkerApi {
constructor(private _app: Application, private _docWorker: DocWorker, private _docManager: DocManager,
private _dbManager: HomeDBManager, private _grist: GristServer) {}
/**
* Adds endpoints for the doc api.
*
* Note that it expects bodyParser, userId, and jsonErrorHandler middleware to be set up outside
* to apply to these routes.
*/
public addEndpoints() {
// check document exists (not soft deleted) and user can view it
const canView = expressWrap(this._assertAccess.bind(this, 'viewers', false));
// check document exists (not soft deleted) and user can edit it
const canEdit = expressWrap(this._assertAccess.bind(this, 'editors', false));
// check user can edit document, with soft-deleted documents being acceptable
const canEditMaybeRemoved = expressWrap(this._assertAccess.bind(this, 'editors', true));
// Middleware to limit number of outstanding requests per document. Will also
// handle errors like expressWrap would.
const throttled = apiThrottle.bind(null, new Map());
const withDoc = (callback: WithDocHandler) => throttled(this._requireActiveDoc(callback));
// Apply user actions to a document.
this._app.post('/api/docs/:docId/apply', canEdit, withDoc(async (activeDoc, req, res) => {
res.json(await activeDoc.applyUserActions({ client: null, req }, req.body));
}));
// Get the specified table.
this._app.get('/api/docs/:docId/tables/:tableId/data', canView, withDoc(async (activeDoc, req, res) => {
const filters = req.query.filter ? JSON.parse(String(req.query.filter)) : {};
if (!Object.keys(filters).every(col => Array.isArray(filters[col]))) {
throw new ApiError("Invalid query: filter values must be arrays", 400);
}
const tableId = req.params.tableId;
const tableData = await handleSandboxError(tableId, [], activeDoc.fetchQuery(null, {tableId, filters}, true));
// Apply sort/limit parameters, if set. TODO: move sorting/limiting into data engine
// and sql.
const params = getQueryParameters(req);
res.json(applyQueryParameters(fromTableDataAction(tableData), params));
}));
// The upload should be a multipart post with an 'upload' field containing one or more files.
// Returns the list of rowIds for the rows created in the _grist_Attachments table.
this._app.post('/api/docs/:docId/attachments', canEdit, withDoc(async (activeDoc, req, res) => {
const uploadResult = await handleUpload(req, res);
res.json(await activeDoc.addAttachments(req, uploadResult.uploadId));
}));
// Returns the metadata for a given attachment ID (i.e. a rowId in _grist_Attachments table).
this._app.get('/api/docs/:docId/attachments/:attId', canView, withDoc(async (activeDoc, req, res) => {
const attRecord = activeDoc.getAttachmentMetadata(req.params.attId as string);
const {fileName, fileSize, timeUploaded: t} = attRecord;
const timeUploaded = (typeof t === 'number') ? new Date(t).toISOString() : undefined;
res.json({fileName, fileSize, timeUploaded});
}));
// Responds with attachment contents, with suitable Content-Type and Content-Disposition.
this._app.get('/api/docs/:docId/attachments/:attId/download', canView, withDoc(async (activeDoc, req, res) => {
const attRecord = activeDoc.getAttachmentMetadata(req.params.attId as string);
const fileIdent = attRecord.fileIdent as string;
const ext = path.extname(fileIdent);
const origName = attRecord.fileName as string;
const fileName = ext ? path.basename(origName, path.extname(origName)) + ext : origName;
const fileData = await activeDoc.getAttachmentData(null, fileIdent);
res.status(200)
.type(ext)
// Construct a content-disposition header of the form 'attachment; filename="NAME"'
.set('Content-Disposition', contentDisposition(fileName, {type: 'attachment'}))
.set('Cache-Control', 'private, max-age=3600')
.send(fileData);
}));
// Adds records.
this._app.post('/api/docs/:docId/tables/:tableId/data', canEdit, withDoc(async (activeDoc, req, res) => {
const tableId = req.params.tableId;
const columnValues = req.body;
const colNames = Object.keys(columnValues);
// user actions expect [null, ...] as row ids, first let's figure the number of items to add by
// looking at the length of a column
const count = columnValues[colNames[0]].length;
// then, let's create [null, ...]
const rowIds = arrayRepeat(count, null);
const sandboxRes = await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req},
[['BulkAddRecord', tableId, rowIds, columnValues]]));
res.json(sandboxRes.retValues[0]);
}));
this._app.post('/api/docs/:docId/tables/:tableId/data/delete', canEdit, withDoc(async (activeDoc, req, res) => {
const tableId = req.params.tableId;
const rowIds = req.body;
const sandboxRes = await handleSandboxError(tableId, [], activeDoc.applyUserActions({client: null, req},
[['BulkRemoveRecord', tableId, rowIds]]));
res.json(sandboxRes.retValues[0]);
}));
// Download full document
// TODO: look at download behavior if ActiveDoc is shutdown during call (cannot
// use withDoc wrapper)
this._app.get('/api/docs/:docId/download', canView, throttled(async (req, res) => {
try {
// We carefully avoid creating an ActiveDoc for the document being downloaded,
// in case it is broken in some way. It is convenient to be able to download
// broken files for diagnosis/recovery.
return await this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
} catch (e) {
if (e.message && e.message.match(/does not exist yet/)) {
// The document has never been seen on file system / s3. It may be new, so
// we try again after having created an ActiveDoc for the document.
await this._getActiveDoc(req);
return this._docWorker.downloadDoc(req, res, this._docManager.storageManager);
} else {
throw e;
}
}
}));
// Update records. The records to update are identified by their id column. Any invalid id fails
// the request and returns a 400 error code.
this._app.patch('/api/docs/:docId/tables/:tableId/data', canEdit, withDoc(async (activeDoc, req, res) => {
const tableId = req.params.tableId;
const columnValues = req.body;
const colNames = Object.keys(columnValues);
const rowIds = columnValues.id;
// sandbox expects no id column
delete columnValues.id;
await handleSandboxError(tableId, colNames, activeDoc.applyUserActions({client: null, req},
[['BulkUpdateRecord', tableId, rowIds, columnValues]]));
res.json(null);
}));
// Reload a document forcibly (in fact this closes the doc, it will be automatically
// reopened on use).
this._app.post('/api/docs/:docId/force-reload', canEdit, withDoc(async (activeDoc, req, res) => {
await activeDoc.reloadDoc();
res.json(null);
}));
// DELETE /api/docs/:docId
// Delete the specified doc.
this._app.delete('/api/docs/:docId', canEditMaybeRemoved, throttled(async (req, res) => {
await this._removeDoc(req, res, true);
}));
// POST /api/docs/:docId/remove
// Soft-delete the specified doc. If query parameter "permanent" is set,
// delete permanently.
this._app.post('/api/docs/:docId/remove', canEditMaybeRemoved, throttled(async (req, res) => {
await this._removeDoc(req, res, isParameterOn(req.query.permanent));
}));
this._app.get('/api/docs/:docId/snapshots', canView, withDoc(async (activeDoc, req, res) => {
const {snapshots} = await activeDoc.getSnapshots();
res.json({snapshots});
}));
this._app.post('/api/docs/:docId/flush', canEdit, throttled(async (req, res) => {
const activeDocPromise = this._getActiveDocIfAvailable(req);
if (!activeDocPromise) {
// Only need to flush if doc is actually open.
res.json(false);
return;
}
const activeDoc = await activeDocPromise;
await activeDoc.flushDoc();
res.json(true);
}));
// This endpoint cannot use withDoc since it is expected behavior for the ActiveDoc it
// starts with to become muted.
this._app.post('/api/docs/:docId/replace', canEdit, throttled(async (req, res) => {
const activeDoc = await this._getActiveDoc(req);
const options: DocReplacementOptions = {};
if (req.body.sourceDocId) {
options.sourceDocId = await this._confirmDocIdForRead(req, String(req.body.sourceDocId));
// We should make sure the source document has flushed recently.
// It may not be served by the same worker, so work through the api.
await fetch(this._grist.getHomeUrl(req, `/api/docs/${options.sourceDocId}/flush`), {
method: 'POST',
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
}
if (req.body.snapshotId) {
options.snapshotId = String(req.body.snapshotId);
}
await activeDoc.replace(options);
res.json(null);
}));
this._app.get('/api/docs/:docId/states', canView, withDoc(async (activeDoc, req, res) => {
res.json(await this._getStates(activeDoc));
}));
this._app.get('/api/docs/:docId/compare/:docId2', canView, withDoc(async (activeDoc, req, res) => {
const {states} = await this._getStates(activeDoc);
const ref = await fetch(this._grist.getHomeUrl(req, `/api/docs/${req.params.docId2}/states`), {
headers: {
...getTransitiveHeaders(req),
'Content-Type': 'application/json',
}
});
const states2: DocState[] = (await ref.json()).states;
const left = states[0];
const right = states2[0];
if (!left || !right) {
// This should not arise unless there's a bug.
throw new Error('document with no history');
}
const rightHashes = new Set(states2.map(state => state.h));
const parent = states.find(state => rightHashes.has(state.h )) || null;
const leftChanged = parent && parent.h !== left.h;
const rightChanged = parent && parent.h !== right.h;
const summary = leftChanged ? (rightChanged ? 'both' : 'left') :
(rightChanged ? 'right' : (parent ? 'same' : 'unrelated'));
const comparison: DocStateComparison = {
left, right, parent, summary
};
res.json(comparison);
}));
// Do an import targeted at a specific workspace. Although the URL fits ApiServer, this
// endpoint is handled only by DocWorker, so is handled here. (Note: this does not handle
// actual file uploads, so no worries here about large request bodies.)
this._app.post('/api/workspaces/:wid/import', expressWrap(async (req, res) => {
const userId = getUserId(req);
const wsId = integerParam(req.params.wid);
const uploadId = integerParam(req.body.uploadId);
const result = await this._docManager.importDocToWorkspace(userId, uploadId, wsId, req.body.browserSettings);
res.json(result);
}));
// Create a document. When an upload is included, it is imported as the initial
// state of the document. Otherwise a fresh empty document is created.
// A "timezone" option can be supplied.
// Documents are created "unsaved".
// TODO: support workspaceId option for creating regular documents, at which point
// existing import endpoint and doc creation endpoint can share implementation
// with this.
// Returns the id of the created document.
this._app.post('/api/docs', expressWrap(async (req, res) => {
const userId = getUserId(req);
let uploadId: number|undefined;
let parameters: {[key: string]: any};
if (req.is('multipart/form-data')) {
const formResult = await handleOptionalUpload(req, res);
if (formResult.upload) {
uploadId = formResult.upload.uploadId;
}
parameters = formResult.parameters || {};
} else {
parameters = req.body;
}
if (parameters.workspaceId) { throw new Error('workspaceId not supported'); }
const browserSettings: BrowserSettings = {};
if (parameters.timezone) { browserSettings.timezone = parameters.timezone; }
if (uploadId !== undefined) {
const result = await this._docManager.importDocToWorkspace(userId, uploadId, null,
browserSettings);
return res.json(result.id);
}
const isAnonymous = isAnonymousUser(req);
const {docId} = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE,
trunkUrlId: NEW_DOCUMENT_CODE});
await this._docManager.fetchDoc({ client: null, req: req as RequestWithLogin, browserSettings }, docId);
return res.status(200).json(docId);
}));
}
/**
* Check for read access to the given document, and return its
* canonical docId. Throws error if read access not available.
* This method is used for documents that are not the main document
* associated with the request, but are rather an extra source to be
* read from, so the access information is not cached in the
* request.
*/
private async _confirmDocIdForRead(req: Request, urlId: string): Promise<string> {
const userId = getUserId(req);
const org = (req as RequestWithLogin).org;
const docAuth = await makeDocAuthResult(this._dbManager.getDoc({urlId, userId, org}));
if (docAuth.error) { throw docAuth.error; }
assertAccess('viewers', docAuth);
return docAuth.docId!;
}
private _getActiveDoc(req: RequestWithLogin): Promise<ActiveDoc> {
return this._docManager.fetchDoc({ client: null, req }, getDocId(req));
}
private _getActiveDocIfAvailable(req: RequestWithLogin): Promise<ActiveDoc>|undefined {
return this._docManager.getActiveDoc(getDocId(req));
}
private async _assertAccess(role: 'viewers'|'editors', allowRemoved: boolean,
req: Request, res: Response, next: NextFunction) {
const scope = getDocScope(req);
allowRemoved = scope.showAll || scope.showRemoved || allowRemoved;
const docAuth = await getOrSetDocAuth(req as RequestWithLogin, this._dbManager, scope.urlId);
assertAccess(role, docAuth, {allowRemoved});
next();
}
// Helper to generate a 503 if the ActiveDoc has been muted.
private _checkForMute(activeDoc: ActiveDoc|undefined) {
if (activeDoc && activeDoc.muted) {
throw new ApiError('Document in flux - try again later', 503);
}
}
/**
* Throws an error if, during processing, the ActiveDoc becomes "muted". Also replaces any
* other error that may have occurred if the ActiveDoc becomes "muted", since the document
* shutting down during processing may have caused a variety of errors.
*
* Expects to be called within a handler that catches exceptions.
*/
private _requireActiveDoc(callback: WithDocHandler): RequestHandler {
return async (req, res) => {
let activeDoc: ActiveDoc|undefined;
try {
activeDoc = await this._getActiveDoc(req as RequestWithLogin);
await callback(activeDoc, req as RequestWithLogin, res);
if (!res.headersSent) { this._checkForMute(activeDoc); }
} catch (err) {
this._checkForMute(activeDoc);
throw err;
}
};
}
private async _getStates(activeDoc: ActiveDoc): Promise<DocStates> {
const states = await activeDoc.getRecentStates();
return {
states,
};
}
private async _removeDoc(req: Request, res: Response, permanent: boolean) {
const scope = getDocScope(req);
const docId = getDocId(req);
if (permanent) {
const query = await this._dbManager.deleteDocument(scope);
this._dbManager.checkQueryResult(query); // fail immediately if deletion denied.
await this._docManager.deleteDoc(null, docId, true);
await sendReply(req, res, query);
} else {
await this._dbManager.softDeleteDocument(scope);
await sendOkReply(req, res);
}
await this._dbManager.flushSingleDocAuthCache(scope, docId);
await this._docManager.interruptDocClients(docId);
}
}
export function addDocApiRoutes(
app: Application, docWorker: DocWorker, docManager: DocManager, dbManager: HomeDBManager,
grist: GristServer
) {
const api = new DocWorkerApi(app, docWorker, docManager, dbManager, grist);
api.addEndpoints();
}
/**
* Catches the errors thrown by the sandbox, and converts to more descriptive ones (such as for
* invalid table names, columns, or rowIds) with better status codes. Accepts the table name, a
* list of column names in that table, and a promise for the result of the sandbox call.
*/
async function handleSandboxError<T>(tableId: string, colNames: string[], p: Promise<T>): Promise<T> {
try {
return await p;
} catch (e) {
if (e instanceof SandboxError) {
let match = e.message.match(/non-existent record #([0-9]+)/);
if (match) {
throw new ApiError(`Invalid row id ${match[1]}`, 400);
}
match = e.message.match(/\[Sandbox\] KeyError '(.*?)'/);
if (match) {
if (match[1] === tableId) {
throw new ApiError(`Table not found "${tableId}"`, 404);
} else if (colNames.includes(match[1])) {
throw new ApiError(`Invalid column "${match[1]}"`, 400);
}
}
throw new ApiError(`Error doing API call: ${e.message}`, 400);
}
throw e;
}
}
/**
* Options for returning results from a query about document data.
* Currently these option don't affect the query itself, only the
* results returned to the user.
*/
export interface QueryParameters {
sort?: string[]; // Columns to sort by (ascending order by default,
// prepend "-" for descending order).
limit?: number; // Limit on number of rows to return.
}
/**
* Extract a sort parameter from a request, if present. Follows
* https://jsonapi.org/format/#fetching-sorting for want of a better
* standard - comma separated, defaulting to ascending order, keys
* prefixed by "-" for descending order.
*
* The sort parameter can either be given as a query parameter, or
* as a header.
*/
function getSortParameter(req: Request): string[]|undefined {
const sortString: string|undefined = optStringParam(req.query.sort) || req.get('X-Sort');
if (!sortString) { return undefined; }
return sortString.split(',');
}
/**
* Extract a limit parameter from a request, if present. Should be a
* simple integer. The limit parameter can either be given as a query
* parameter, or as a header.
*/
function getLimitParameter(req: Request): number|undefined {
const limitString: string|undefined = optStringParam(req.query.limit) || req.get('X-Limit');
if (!limitString) { return undefined; }
const limit = parseInt(limitString, 10);
if (isNaN(limit)) { throw new Error('limit is not a number'); }
return limit;
}
/**
* Extract sort and limit parameters from request, if they are present.
*/
function getQueryParameters(req: Request): QueryParameters {
return {
sort: getSortParameter(req),
limit: getLimitParameter(req),
};
}
/**
* Sort table contents being returned. Sort keys with a '-' prefix
* are sorted in descending order, otherwise ascending. Contents are
* modified in place.
*/
function applySort(values: TableColValues, sort: string[]) {
if (!sort) { return values; }
const sortKeys = sort.map(key => key.replace(/^-/, ''));
const iteratees = sortKeys.map(key => {
if (!(key in values)) {
throw new Error(`unknown key ${key}`);
}
const col = values[key];
return (i: number) => col[i];
});
const sortSpec = sort.map((key, i) => (key.startsWith('-') ? -i - 1 : i + 1));
const index = values.id.map((_, i) => i);
const sortFunc = new SortFunc({
getColGetter(i) { return iteratees[i - 1]; },
getManualSortGetter() { return null; }
});
sortFunc.updateSpec(sortSpec);
index.sort(sortFunc.compare.bind(sortFunc));
for (const key of Object.keys(values)) {
const col = values[key];
values[key] = index.map(i => col[i]);
}
return values;
}
/**
* Truncate columns to the first N values. Columns are modified in place.
*/
function applyLimit(values: TableColValues, limit: number) {
// for no limit, or 0 limit, do not apply any restriction
if (!limit) { return values; }
for (const key of Object.keys(values)) {
values[key].splice(limit);
}
return values;
}
/**
* Apply query parameters to table contents. Contents are modified in place.
*/
export function applyQueryParameters(values: TableColValues, params: QueryParameters): TableColValues {
if (params.sort) { applySort(values, params.sort); }
if (params.limit) { applyLimit(values, params.limit); }
return values;
}

90
app/server/lib/DocClients.ts Normal file
View File

@@ -0,0 +1,90 @@
/**
* Module to manage the clients of an ActiveDoc. It keeps track of how many clients have the doc
* open, and what FD they are using.
*/
import {arrayRemove} from 'app/common/gutil';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {Authorizer} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
import {sendDocMessage} from 'app/server/lib/Comm';
import {DocSession} from 'app/server/lib/DocSession';
import * as log from 'app/server/lib/log';
export class DocClients {
private _docSessions: DocSession[] = [];
constructor(
public readonly activeDoc: ActiveDoc
) {}
/**
* Returns the number of connected clients.
*/
public clientCount(): number {
return this._docSessions.length;
}
/**
* Adds a client's open file to the list of connected clients.
*/
public addClient(client: Client, authorizer: Authorizer): DocSession {
const docSession = client.addDocSession(this.activeDoc, authorizer);
this._docSessions.push(docSession);
log.debug("DocClients (%s) now has %d clients; new client is %s (fd %s)", this.activeDoc.docName,
this._docSessions.length, client.clientId, docSession.fd);
return docSession;
}
/**
* Removes a client from the list of connected clients for this document. In other words, closes
* this DocSession.
*/
public removeClient(docSession: DocSession): void {
log.debug("DocClients.removeClient", docSession.client.clientId);
docSession.client.removeDocSession(docSession.fd);
if (arrayRemove(this._docSessions, docSession)) {
log.debug("DocClients (%s) now has %d clients", this.activeDoc.docName, this._docSessions.length);
}
}
/**
* Removes all active clients from this document, i.e. closes all DocSessions.
*/
public removeAllClients(): void {
log.debug("DocClients.removeAllClients() removing %s docSessions", this._docSessions.length);
const docSessions = this._docSessions.splice(0);
for (const docSession of docSessions) {
docSession.client.removeDocSession(docSession.fd);
}
}
public interruptAllClients() {
log.debug("DocClients.interruptAllClients() interrupting %s docSessions", this._docSessions.length);
for (const docSession of this._docSessions) {
docSession.client.interruptConnection();
}
}
/**
* Broadcasts a message to all clients of this document using Comm.sendDocMessage. Also sends all
* docAction to active doc's plugin manager.
* @param {Object} client: Originating client used to set the `fromSelf` flag in the message.
* @param {String} type: The type of the message, e.g. 'docUserAction'.
* @param {Object} messageData: The data for this type of message.
*/
public broadcastDocMessage(client: Client|null, type: string, messageData: any): void {
for (let i = 0, len = this._docSessions.length; i < len; i++) {
const curr = this._docSessions[i];
const fromSelf = (curr.client === client);
sendDocMessage(curr.client, curr.fd, type, messageData, fromSelf);
}
if (type === "docUserAction" && messageData.docActions) {
for (const action of messageData.docActions) {
this.activeDoc.docPluginManager.receiveAction(action);
}
}
}
}

481
app/server/lib/DocManager.ts Normal file
View File

@@ -0,0 +1,481 @@
import * as pidusage from '@gristlabs/pidusage';
import * as bluebird from 'bluebird';
import {EventEmitter} from 'events';
import noop = require('lodash/noop');
import * as path from 'path';
import {ApiError} from 'app/common/ApiError';
import {mapSetOrClear} from 'app/common/AsyncCreate';
import {BrowserSettings} from 'app/common/BrowserSettings';
import {DocCreationInfo, DocEntry, DocListAPI, OpenDocMode, OpenLocalDocResult} from 'app/common/DocListAPI';
import {EncActionBundleFromHub} from 'app/common/EncActionBundle';
import {Invite} from 'app/common/sharing';
import {tbind} from 'app/common/tbind';
import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer,
isSingleUserMode} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
import {makeOptDocSession, OptDocSession} from 'app/server/lib/DocSession';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {makeForkIds, makeId} from 'app/server/lib/idUtils';
import * as log from 'app/server/lib/log';
import * as ServerMetrics from 'app/server/lib/ServerMetrics';
import {ActiveDoc} from './ActiveDoc';
import {PluginManager} from './PluginManager';
import {getFileUploadInfo, globalUploadSet, makeAccessId, UploadInfo} from './uploads';
// A TTL in milliseconds to use for material that can easily be recomputed / refetched
// but is a bit of a burden under heavy traffic.
export const DEFAULT_CACHE_TTL = 10000;
/**
* DocManager keeps track of "active" Grist documents, i.e. those loaded
* in-memory, with clients connected to them.
*/
export class DocManager extends EventEmitter {
// Maps docName to promise for ActiveDoc object. Most of the time the promise
// will be long since resolved, with the resulting document cached.
private _activeDocs: Map<string, Promise<ActiveDoc>> = new Map();
constructor(
public readonly storageManager: IDocStorageManager,
public readonly pluginManager: PluginManager,
private _homeDbManager: HomeDBManager|null,
public gristServer: GristServer
) {
super();
}
// attach a home database to the DocManager. During some tests, it
// is awkward to have this set up at the point of construction.
public testSetHomeDbManager(dbManager: HomeDBManager) {
this._homeDbManager = dbManager;
}
/**
* Returns an implementation of the DocListAPI for the given Client object.
*/
public getDocListAPIImpl(client: Client): DocListAPI {
return {
getDocList: tbind(this.listDocs, this, client),
createNewDoc: tbind(this.createNewDoc, this, client),
importSampleDoc: tbind(this.importSampleDoc, this, client),
importDoc: tbind(this.importDoc, this, client),
deleteDoc: tbind(this.deleteDoc, this, client),
renameDoc: tbind(this.renameDoc, this, client),
openDoc: tbind(this.openDoc, this, client),
};
}
/**
* Returns the number of currently open docs.
*/
public numOpenDocs(): number {
return this._activeDocs.size;
}
/**
* Returns a Map from docId to number of connected clients for each doc.
*/
public async getDocClientCounts(): Promise<Map<string, number>> {
const values = await Promise.all(Array.from(this._activeDocs.values(), async (adocPromise) => {
const adoc = await adocPromise;
return [adoc.docName, adoc.docClients.clientCount()] as [string, number];
}));
return new Map(values);
}
/**
* Returns a promise for all known Grist documents and document invites to show in the doc list.
*/
public async listDocs(client: Client): Promise<{docs: DocEntry[], docInvites: DocEntry[]}> {
const docs = await this.storageManager.listDocs();
return {docs, docInvites: []};
}
/**
* Returns a promise for invites to docs which have not been downloaded.
*/
public async getLocalInvites(client: Client): Promise<Invite[]> {
return [];
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @returns {Promise:String} The name of the new document.
*/
public async createNewDoc(client: Client): Promise<string> {
log.debug('DocManager.createNewDoc');
const activeDoc: ActiveDoc = await this.createNewEmptyDoc(makeOptDocSession(client), 'Untitled');
await activeDoc.addInitialTable();
return activeDoc.docName;
}
/**
* Download a shared doc by creating a new doc and applying to it the shared doc snapshot actions.
* Also marks the invite to the doc as ignored, since it has already been accepted.
* @returns {Promise:String} The name of the new document.
*/
public async downloadSharedDoc(client: Client, docId: string, docName: string): Promise<string> {
throw new Error('downloadSharedDoc not implemented');
}
/**
* Creates a new document, fetches it, and adds a table to it.
* @param {String} sampleDocName: Doc name of a sample document.
* @returns {Promise:String} The name of the new document.
*/
public async importSampleDoc(client: Client, sampleDocName: string): Promise<string> {
const sourcePath = this.storageManager.getSampleDocPath(sampleDocName);
if (!sourcePath) {
throw new Error(`no path available to sample ${sampleDocName}`);
}
log.info('DocManager.importSampleDoc importing', sourcePath);
const basenameHint = path.basename(sampleDocName);
const targetName = await docUtils.createNumbered(basenameHint, '-',
(name: string) => docUtils.createExclusive(this.storageManager.getPath(name)));
const targetPath = this.storageManager.getPath(targetName);
log.info('DocManager.importSampleDoc saving as', targetPath);
await docUtils.copyFile(sourcePath, targetPath);
return targetName;
}
/**
* Processes an upload, containing possibly multiple files, to create a single new document, and
* returns the new document's name/id.
*/
public async importDoc(client: Client, uploadId: number): Promise<string> {
const userId = this._homeDbManager ? await client.requireUserId(this._homeDbManager) : null;
const result = await this._doImportDoc(makeOptDocSession(client),
globalUploadSet.getUploadInfo(uploadId, this.makeAccessId(userId)), {naming: 'classic'});
return result.id;
}
// Import a document, assigning it a unique id distinct from its title. Cleans up uploadId.
public importDocWithFreshId(docSession: OptDocSession, userId: number, uploadId: number): Promise<DocCreationInfo> {
const accessId = this.makeAccessId(userId);
return this._doImportDoc(docSession, globalUploadSet.getUploadInfo(uploadId, accessId),
{naming: 'saved'});
}
// Do an import targeted at a specific workspace. Cleans up uploadId.
// UserId should correspond to the user making the request.
// A workspaceId of null results in an import to an unsaved doc, not
// associated with a specific workspace.
public async importDocToWorkspace(
userId: number, uploadId: number, workspaceId: number|null, browserSettings?: BrowserSettings,
): Promise<DocCreationInfo> {
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const accessId = this.makeAccessId(userId);
const result = await this._doImportDoc(makeOptDocSession(null, browserSettings),
globalUploadSet.getUploadInfo(uploadId, accessId), {
naming: workspaceId ? 'saved' : 'unsaved',
userId,
});
if (workspaceId) {
const queryResult = await this._homeDbManager.addDocument({userId}, workspaceId,
{name: result.title}, result.id);
if (queryResult.status !== 200) {
// TODO The ready-to-add document is not yet in storageManager, but is in the filesystem. It
// should get cleaned up in case of error here.
throw new ApiError(queryResult.errMessage || 'unable to add imported document', queryResult.status);
}
}
// Ship the import to S3, since it isn't associated with any particular worker at this time.
// We could associate it with the current worker, but that is not necessarily desirable.
await this.storageManager.addToStorage(result.id);
return result;
}
/**
* Imports file at filepath into the app by creating a new document and adding the file to
* the documents directory.
* @param {String} filepath - Path to the current location of the file on the server.
* @returns {Promise:String} The name of the new document.
*/
public async importNewDoc(filepath: string): Promise<DocCreationInfo> {
const uploadId = globalUploadSet.registerUpload([await getFileUploadInfo(filepath)], null, noop, null);
return await this._doImportDoc(makeOptDocSession(null), globalUploadSet.getUploadInfo(uploadId, null),
{naming: 'classic'});
}
/**
* Deletes the Grist files and directories for a given document name.
* @param {String} docName - The name of the Grist document to be deleted.
* @returns {Promise:String} The name of the deleted Grist document.
*
*/
public async deleteDoc(client: Client|null, docName: string, deletePermanently: boolean): Promise<string> {
log.debug('DocManager.deleteDoc starting for %s', docName);
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
// Call activeDoc's shutdown method first, to remove the doc from internal structures.
const doc: ActiveDoc = await docPromise;
await doc.shutdown();
}
await this.storageManager.deleteDoc(docName, deletePermanently);
return docName;
}
/**
* Interrupt all clients, forcing them to reconnect. Handy when a document has changed
* status in some major way that affects access rights, such as being deleted.
*/
public async interruptDocClients(docName: string) {
const docPromise = this._activeDocs.get(docName);
if (docPromise) {
const doc: ActiveDoc = await docPromise;
doc.docClients.interruptAllClients();
}
}
/**
* Opens a document. Adds the client as a subscriber to the document, and fetches and returns the
* document's metadata.
* @returns {Promise:Object} An object with properties:
* `docFD` - the descriptor to use in further methods and messages about this document,
* `doc` - the object with metadata tables.
*/
public async openDoc(client: Client, docId: string,
mode: OpenDocMode = 'default'): Promise<OpenLocalDocResult> {
let auth: Authorizer;
const dbManager = this._homeDbManager;
if (!isSingleUserMode()) {
if (!dbManager) { throw new Error("HomeDbManager not available"); }
// Sets up authorization of the document.
const org = client.getOrg();
if (!org) { throw new Error('Documents can only be opened in the context of a specific organization'); }
const userId = await client.getUserId(dbManager) || dbManager.getAnonymousUserId();
// We use docId in the key, and disallow urlId, so we can be sure that we are looking at the
// right doc when we re-query the DB over the life of the websocket.
const key = {urlId: docId, userId, org};
log.debug("DocManager.openDoc Authorizer key", key);
const docAuth = await dbManager.getDocAuthCached(key);
assertAccess('viewers', docAuth);
if (docAuth.docId !== docId) {
// The only plausible way to end up here is if we called openDoc with a urlId rather
// than a docId.
throw new Error(`openDoc expected docId ${docAuth.docId} not urlId ${docId}`);
}
auth = new DocAuthorizer(dbManager, key, mode);
} else {
log.debug(`DocManager.openDoc not using authorization for ${docId} because GRIST_SINGLE_USER`);
auth = new DummyAuthorizer('owners');
}
// Fetch the document, and continue when we have the ActiveDoc (which may be immediately).
const activeDoc: ActiveDoc = await this.fetchDoc(makeOptDocSession(client), docId);
if (activeDoc.muted) {
log.debug('DocManager.openDoc interrupting, called for a muted doc', docId);
client.interruptConnection();
throw new Error(`document ${docId} cannot be opened right now`);
}
const docSession = activeDoc.addClient(client, auth);
const [metaTables, recentActions] = await Promise.all([
activeDoc.fetchMetaTables(client),
activeDoc.getRecentActions(client, false)
]);
this.emit('open-doc', this.storageManager.getPath(activeDoc.docName));
ServerMetrics.get('docs.num_open').set(this._activeDocs.size);
ServerMetrics.get('app.have_doc_open').set(true);
ServerMetrics.get('app.doc_open_span').start();
return {
docFD: docSession.fd,
clientId: docSession.client.clientId,
doc: metaTables,
log: recentActions,
plugins: activeDoc.docPluginManager.getPlugins()
};
}
/**
* Shut down all open docs. This is called, in particular, on server shutdown.
*/
public async shutdownAll() {
await Promise.all(Array.from(this._activeDocs.values(),
adocPromise => adocPromise.then(adoc => adoc.shutdown())));
try {
await this.storageManager.closeStorage();
} catch (err) {
log.error('DocManager had problem shutting down storage: %s', err.message);
}
// Clear the setInterval that the pidusage module sets up internally.
pidusage.clear();
}
// Access a document by name.
public getActiveDoc(docName: string): Promise<ActiveDoc>|undefined {
return this._activeDocs.get(docName);
}
public async removeActiveDoc(activeDoc: ActiveDoc): Promise<void> {
this._activeDocs.delete(activeDoc.docName);
ServerMetrics.get('docs.num_open').set(this._activeDocs.size);
ServerMetrics.get('app.have_doc_open').set(this._activeDocs.size > 0);
ServerMetrics.get('app.doc_open_span').setRunning(this._activeDocs.size > 0);
}
public async renameDoc(client: Client, oldName: string, newName: string): Promise<void> {
log.debug('DocManager.renameDoc %s -> %s', oldName, newName);
const docPromise = this._activeDocs.get(oldName);
if (docPromise) {
const adoc: ActiveDoc = await docPromise;
await adoc.renameDocTo(client, newName);
this._activeDocs.set(newName, docPromise);
this._activeDocs.delete(oldName);
} else {
await this.storageManager.renameDoc(oldName, newName);
}
}
public markAsChanged(activeDoc: ActiveDoc) {
if (!activeDoc.muted) {
this.storageManager.markAsChanged(activeDoc.docName);
}
}
public markAsEdited(activeDoc: ActiveDoc) {
if (!activeDoc.muted) {
this.storageManager.markAsEdited(activeDoc.docName);
}
}
/**
* Helper function for creating a new empty document that also emits an event.
* @param docSession The client session.
* @param basenameHint Suggested base name to use (no directory, no extension).
*/
public async createNewEmptyDoc(docSession: OptDocSession, basenameHint: string): Promise<ActiveDoc> {
const docName = await this._createNewDoc(basenameHint);
return mapSetOrClear(this._activeDocs, docName,
this.gristServer.create.ActiveDoc(this, docName).createDoc(docSession));
}
/**
* Fetches an ActiveDoc object. Used by openDoc.
*/
public async fetchDoc(docSession: OptDocSession, docName: string): Promise<ActiveDoc> {
log.debug('DocManager.fetchDoc', docName);
// Repeat until we acquire an ActiveDoc that is not muted (shutting down).
for (;;) {
if (!this._activeDocs.has(docName)) {
const newDoc = this.gristServer.create.ActiveDoc(this, docName);
// Propagate backupMade events from newly opened activeDocs (consolidate all to DocMan)
newDoc.on('backupMade', (bakPath: string) => {
this.emit('backupMade', bakPath);
});
return mapSetOrClear(this._activeDocs, docName, newDoc.loadDoc(docSession));
}
const activeDoc = await this._activeDocs.get(docName)!;
if (!activeDoc.muted) { return activeDoc; }
log.debug('DocManager.fetchDoc waiting because doc is muted', docName);
await bluebird.delay(1000);
}
}
public makeAccessId(userId: number|null): string|null {
return makeAccessId(this.gristServer, userId);
}
/**
* Helper function for creating a new shared document given the doc snapshot bundles received
* from the sharing hub.
* @param {String} basenameHint: Suggested base name to use (no directory, no extension).
* @param {String} docId: The docId of the doc received from the hub.
* @param {String} instanceId: The user instanceId creating the doc.
* @param {EncActionBundleFromHub[]} encBundles: The action bundles making up the doc snapshot.
* @returns {Promise:ActiveDoc} ActiveDoc for the newly created document.
*/
protected async _createNewSharedDoc(basenameHint: string, docId: string, instanceId: string,
encBundles: EncActionBundleFromHub[]): Promise<ActiveDoc> {
const docName = await this._createNewDoc(basenameHint);
return mapSetOrClear(this._activeDocs, docName,
this.gristServer.create.ActiveDoc(this, docName).downloadSharedDoc(docId, instanceId, encBundles));
}
/**
* Helper that implements doing the actual import of an uploaded set of files to create a new
* document.
*/
private async _doImportDoc(docSession: OptDocSession, uploadInfo: UploadInfo,
options: {
naming: 'classic'|'saved'|'unsaved',
userId?: number,
}): Promise<DocCreationInfo> {
try {
const fileCount = uploadInfo.files.length;
const hasGristDoc = Boolean(uploadInfo.files.find(f => path.extname(f.origName) === '.grist'));
if (hasGristDoc && fileCount > 1) {
throw new Error('Grist docs must be uploaded individually');
}
const first = uploadInfo.files[0].origName;
const ext = path.extname(first);
const basename = path.basename(first, ext).trim() || "Untitled upload";
let id: string;
switch (options.naming) {
case 'saved':
id = makeId();
break;
case 'unsaved': {
const {userId} = options;
if (!userId) { throw new Error('unsaved import requires userId'); }
if (!this._homeDbManager) { throw new Error("HomeDbManager not available"); }
const isAnonymous = userId === this._homeDbManager.getAnonymousUserId();
id = makeForkIds({userId, isAnonymous, trunkDocId: NEW_DOCUMENT_CODE,
trunkUrlId: NEW_DOCUMENT_CODE}).docId;
break;
}
case 'classic':
id = basename;
break;
default:
throw new Error('naming mode not recognized');
}
if (ext === '.grist') {
// If the import is a grist file, copy it to the docs directory.
// TODO: We should be skeptical of the upload file to close a possible
// security vulnerability. See https://phab.getgrist.com/T457.
const docName = await this._createNewDoc(id);
const docPath = await this.storageManager.getPath(docName);
await docUtils.copyFile(uploadInfo.files[0].absPath, docPath);
return {title: basename, id: docName};
} else {
const doc = await this.createNewEmptyDoc(docSession, id);
await doc.oneStepImport(docSession, uploadInfo);
return {title: basename, id: doc.docName};
}
} catch (err) {
throw new ApiError(err.message, err.status || 400, {
tips: [{action: 'ask-for-help', message: 'Ask for help'}]
});
} finally {
await globalUploadSet.cleanup(uploadInfo.uploadId);
}
}
// Returns the name for a new doc, based on basenameHint.
private async _createNewDoc(basenameHint: string): Promise<string> {
const docName: string = await docUtils.createNumbered(basenameHint, '-', async (name: string) => {
if (this._activeDocs.has(name)) {
throw new Error("Existing entry in active docs for: " + name);
}
return docUtils.createExclusive(this.storageManager.getPath(name));
});
log.debug('DocManager._createNewDoc picked name', docName);
await this.pluginManager.pluginsLoaded;
return docName;
}
}

32
app/server/lib/DocPluginData.ts Normal file
View File

@@ -0,0 +1,32 @@
import {Promisified} from 'app/common/tpromisified';
import {Storage} from 'app/plugin/StorageAPI';
import {DocStorage} from 'app/server/lib/DocStorage';
/**
* DocPluginData implements a document's `Storage` for plugin.
*/
export class DocPluginData implements Promisified<Storage> {
constructor(private _docStorage: DocStorage, private _pluginId: string) {
// nothing to do here
}
public async getItem(key: string): Promise<any> {
const res = await this._docStorage.getPluginDataItem(this._pluginId, key);
if (typeof res === 'string') {
return JSON.parse(res);
}
return res;
}
public hasItem(key: string): Promise<boolean> {
return this._docStorage.hasPluginDataItem(this._pluginId, key);
}
public setItem(key: string, value: any): Promise<void> {
return this._docStorage.setPluginDataItem(this._pluginId, key, JSON.stringify(value));
}
public removeItem(key: string): Promise<void> {
return this._docStorage.removePluginDataItem(this._pluginId, key);
}
public clear(): Promise<void> {
return this._docStorage.clearPluginDataItem(this._pluginId);
}
}

218
app/server/lib/DocPluginManager.ts Normal file
View File

@@ -0,0 +1,218 @@
import {ApplyUAResult} from 'app/common/ActiveDocAPI';
import {fromTableDataAction, TableColValues} from 'app/common/DocActions';
import * as gutil from 'app/common/gutil';
import {LocalPlugin} from 'app/common/plugin';
import {createRpcLogger, PluginInstance} from 'app/common/PluginInstance';
import {Promisified} from 'app/common/tpromisified';
import {ParseFileResult, ParseOptions} from 'app/plugin/FileParserAPI';
import {checkers, GristTable} from "app/plugin/grist-plugin-api";
import {GristDocAPI} from "app/plugin/GristAPI";
import {Storage} from 'app/plugin/StorageAPI';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {DocPluginData} from 'app/server/lib/DocPluginData';
import {FileParserElement} from 'app/server/lib/FileParserElement';
import {GristServer} from 'app/server/lib/GristServer';
import * as log from 'app/server/lib/log';
import { SafePythonComponent } from 'app/server/lib/SafePythonComponent';
import { UnsafeNodeComponent } from 'app/server/lib/UnsafeNodeComponent';
import { promisifyAll } from 'bluebird';
import * as fse from 'fs-extra';
import * as path from 'path';
import * as tmp from 'tmp';
promisifyAll(tmp);
/**
* Implements GristDocAPI interface.
*/
class GristDocAPIImpl implements GristDocAPI {
constructor(private _activeDoc: ActiveDoc) {}
public async getDocName() { return this._activeDoc.docName; }
public async listTables(): Promise<string[]> {
const table = this._activeDoc.docData!.getTable('_grist_Tables')!;
return (table.getColValues('tableId') as string[])
.filter(id => !id.startsWith("GristSummary_")).sort();
}
public async fetchTable(tableId: string): Promise<TableColValues> {
return fromTableDataAction(await this._activeDoc.fetchTable(null, tableId));
}
public applyUserActions(actions: any[][]): Promise<ApplyUAResult> {
return this._activeDoc.applyUserActions({client: null}, actions);
}
}
/**
* DocPluginManager manages plugins for a document.
*
* DocPluginManager instanciates asynchronously. Wait for the `ready` to resolve before using any
* plugin.
*
*/
export class DocPluginManager {
public readonly plugins: {[s: string]: PluginInstance} = {};
public readonly ready: Promise<any>;
public readonly gristDocAPI: GristDocAPI;
private _tmpDir: string;
private _pluginInstances: PluginInstance[];
constructor(private _localPlugins: LocalPlugin[], private _appRoot: string, private _activeDoc: ActiveDoc, private _server: GristServer) {
this.gristDocAPI = new GristDocAPIImpl(_activeDoc);
this._pluginInstances = [];
this.ready = this._initialize();
}
public tmpDir(): string {
return this._tmpDir;
}
/**
* To be moved in ActiveDoc.js as a new implementation for ActiveDoc.importFile.
* Throws if no importers can parse the file.
*/
public async parseFile(filePath: string, fileName: string, parseOptions: ParseOptions): Promise<ParseFileResult> {
// Support an existing grist json format directly for files with a "jgrist"
// extension.
if (path.extname(fileName) === '.jgrist') {
try {
const result = JSON.parse(await fse.readFile(filePath, 'utf8')) as ParseFileResult;
result.parseOptions = {};
// The parseOptions component isn't checked here, since it seems free-form.
checkers.ParseFileResult.check(result);
checkReferences(result.tables);
return result;
} catch (err) {
throw new Error('Grist json format could not be parsed: ' + err);
}
}
const matchingFileParsers: FileParserElement[] = FileParserElement.getMatching(this._pluginInstances, fileName);
if (!this._tmpDir) {
throw new Error("DocPluginManager: initialization has not completed");
}
// TODO: PluginManager shouldn't patch path here. Instead it should expose a method to create
// dataSources, that would move the file to under _tmpDir and return an object with the relative
// path.
filePath = path.relative(this._tmpDir, filePath);
log.debug(`parseFile: found ${matchingFileParsers.length} fileParser with matching file extensions`);
const messages = [];
for (const {plugin, parseFileStub} of matchingFileParsers) {
const name = plugin.definition.id;
try {
log.info(`DocPluginManager.parseFile: calling to ${name} with ${filePath}`);
const result = await parseFileStub.parseFile({path: filePath, origName: fileName}, parseOptions);
checkers.ParseFileResult.check(result);
checkReferences(result.tables);
return result;
} catch (err) {
const cleanerMessage = err.message.replace(/^\[Sandbox\] (Exception)?/, '').trim();
messages.push(cleanerMessage);
log.warn(`DocPluginManager.parseFile: ${name} Failed parseFile `, err.message);
continue;
}
}
const details = messages.length ? ": " + messages.join("; ") : "";
throw new Error(`Cannot parse this data${details}`);
}
/**
* Returns a promise which resolves with the list of plugins definitions.
*/
public getPlugins(): LocalPlugin[] {
return this._localPlugins;
}
/**
* Shut down all plugins for this document.
*/
public async shutdown(): Promise<void> {
const names = Object.keys(this.plugins);
log.debug("DocPluginManager.shutdown cleaning up %s plugins", names.length);
await Promise.all(names.map(name => this.plugins[name].shutdown()));
if (this._tmpDir) {
log.debug("DocPluginManager.shutdown removing tmpDir %s", this._tmpDir);
await fse.remove(this._tmpDir);
}
}
/**
* Reload plugins: shutdown all plugins, clear list of plugins and load new ones. Returns a
* promise that resolves when initialisation is done.
*/
public async reload(plugins: LocalPlugin[]): Promise<void> {
await this.shutdown();
this._pluginInstances = [];
this._localPlugins = plugins;
await this._initialize();
}
public receiveAction(action: any[]): void {
for (const plugin of this._pluginInstances) {
const unsafeNode = plugin.unsafeNode as UnsafeNodeComponent;
if (unsafeNode) {
unsafeNode.receiveAction(action);
}
}
}
private async _initialize(): Promise<void> {
this._tmpDir = await tmp.dirAsync({prefix: 'grist-tmp-', unsafeCleanup: true});
for (const plugin of this._localPlugins) {
try {
// todo: once Comm has been replaced by grain-rpc, pluginInstance.rpc should forward '*' to client
const pluginInstance = new PluginInstance(plugin, createRpcLogger(log, `PLUGIN ${plugin.id}:`));
pluginInstance.rpc.registerForwarder('grist', pluginInstance.rpc, '');
pluginInstance.rpc.registerImpl<GristDocAPI>("GristDocAPI", this.gristDocAPI, checkers.GristDocAPI);
pluginInstance.rpc.registerImpl<Promisified<Storage>>("DocStorage",
new DocPluginData(this._activeDoc.docStorage, plugin.id), checkers.Storage);
const components = plugin.manifest.components;
if (components) {
const {safePython, unsafeNode} = components;
if (safePython) {
const comp = pluginInstance.safePython = new SafePythonComponent(plugin, safePython, this._tmpDir,
this._activeDoc.docName, this._server);
pluginInstance.rpc.registerForwarder(safePython, comp);
}
if (unsafeNode) {
const gristDocPath = this._activeDoc.docStorage.docPath;
const comp = pluginInstance.unsafeNode = new UnsafeNodeComponent(plugin, pluginInstance.rpc, unsafeNode,
this._appRoot, gristDocPath);
pluginInstance.rpc.registerForwarder(unsafeNode, comp);
}
}
this._pluginInstances.push(pluginInstance);
} catch (err) {
log.info(`DocPluginInstance: failed to create instance ${plugin.id}: ${err.message}`);
}
}
for (const instance of this._pluginInstances) {
this.plugins[instance.definition.id] = instance;
}
}
}
/**
* Checks that tables include all the tables referenced by tables columns. Throws an exception
* otherwise.
*/
function checkReferences(tables: GristTable[]) {
const tableIds = tables.map(table => table.table_name);
for (const table of tables) {
for (const col of table.column_metadata) {
const refTableId = gutil.removePrefix(col.type, "Ref:");
if (refTableId && !tableIds.includes(refTableId)) {
throw new Error(`Column type: ${col.type}, references an unknown table`);
}
}
}
}

49
app/server/lib/DocSession.ts Normal file
View File

@@ -0,0 +1,49 @@
import {BrowserSettings} from 'app/common/BrowserSettings';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
/**
* OptDocSession allows for certain ActiveDoc operations to work with or without an open document.
* It is useful in particular for actions when importing a file to create a new document.
*/
export interface OptDocSession {
client: Client|null;
shouldBundleActions?: boolean;
linkId?: number;
browserSettings?: BrowserSettings;
req?: RequestWithLogin;
}
export function makeOptDocSession(client: Client|null, browserSettings?: BrowserSettings): OptDocSession {
if (client && !browserSettings) { browserSettings = client.browserSettings; }
return {client, browserSettings};
}
/**
* DocSession objects maintain information for a single session<->doc instance.
*/
export class DocSession implements OptDocSession {
/**
* Flag to indicate that user actions 'bundle' process is started and in progress (`true`),
* otherwise it's `false`
*/
public shouldBundleActions?: boolean;
/**
* Indicates the actionNum of the previously applied action
* to which the first action in actions should be linked.
* Linked actions appear as one action and can be undone/redone in a single step.
*/
public linkId?: number;
constructor(
public readonly activeDoc: ActiveDoc,
public readonly client: Client,
public readonly fd: number,
public readonly authorizer: Authorizer
) {}
// Browser settings (like timezone) obtained from the Client.
public get browserSettings(): BrowserSettings { return this.client.browserSettings; }
}

151
app/server/lib/DocSnapshots.ts Normal file
View File

@@ -0,0 +1,151 @@
import { KeyedOps } from 'app/common/KeyedOps';
import { ExternalStorage } from 'app/server/lib/ExternalStorage';
import * as log from 'app/server/lib/log';
import * as moment from 'moment';
/**
* Metadata about a single document version.
*/
export interface ObjSnapshot {
lastModified: Date;
snapshotId: string;
}
/**
* Information about a single document snapshot in S3, including a Grist docId.
* Similar to a type in app/common/UserAPI, but with lastModified as a Date
* rather than a string.
*/
export interface DocSnapshot extends ObjSnapshot {
docId: string;
}
/**
* A collection of document snapshots. Most recent snapshots first.
*/
export interface DocSnapshots {
snapshots: DocSnapshot[];
}
/**
* A utility for pruning snapshots, so the number of snapshots doesn't get out of hand.
*/
export class DocSnapshotPruner {
private _closing: boolean = false; // when set, should ignore prune requests
private _prunes: KeyedOps;
// Specify store to be pruned, and delay before pruning.
constructor(private _ext: ExternalStorage, _options: {
delayBeforeOperationMs?: number,
minDelayBetweenOperationsMs?: number
} = {}) {
this._prunes = new KeyedOps((key) => this.prune(key), {
..._options,
retry: false,
logError: (key, failureCount, err) => log.error(`Pruning document ${key} gave error ${err}`)
});
}
// Shut down. Prunes scheduled for the future are run immediately.
// Can be called repeated safely.
public async close() {
this._closing = true;
this._prunes.expediteOperations();
await this.wait();
}
// Wait for all in-progress prunes to finish up in an orderly fashion.
public async wait() {
await this._prunes.wait(() => 'waiting for pruning to finish');
}
// Note that a document has changed, and should be pruned (or repruned). Pruning operation
// done as a background operation.
public requestPrune(key: string) {
// If closing down, do not accept any prune requests.
if (this._closing) { return; }
// Mark the key as needing work.
this._prunes.addOperation(key);
}
// Get all snapshots for a document, and whether they should be kept or pruned.
public async classify(key: string): Promise<Array<{snapshot: ObjSnapshot, keep: boolean}>> {
const versions = await this._ext.versions(key);
return shouldKeepSnapshots(versions).map((keep, index) => ({keep, snapshot: versions[index]}));
}
// Prune the specified document immediately.
public async prune(key: string) {
const versions = await this.classify(key);
const redundant = versions.filter(v => !v.keep);
await this._ext.remove(key, redundant.map(r => r.snapshot.snapshotId));
log.info(`Pruned ${redundant.length} versions of ${versions.length} for document ${key}`);
}
}
/**
* Calculate which snapshots to keep. Expects most recent snapshots to be first.
* We keep:
* - The five most recent versions (including the current version)
* - The most recent version in every hour, for up to 25 hours before the current version
* - The most recent version in every day, for up to 32 days before the current version
* - The most recent version in every week, for up to 12 weeks before the current version
* - The most recent version in every month, for up to 36 months before the current version
* - The most recent version in every year, for up to 1000 years before the current version
* Calculations done in UTC, Gregorian calendar, ISO weeks (week starts with Monday).
*/
export function shouldKeepSnapshots(snapshots: ObjSnapshot[]): boolean[] {
// Get current version
const current = snapshots[0];
if (!current) { return []; }
// Get time of current version
const start = moment.utc(current.lastModified);
// Track saved version per hour, day, week, month, year, and number of times a version
// has been saved based on a corresponding rule.
const buckets: TimeBucket[] = [
{range: 'hour', prev: start, usage: 0, cap: 25},
{range: 'day', prev: start, usage: 0, cap: 32},
{range: 'isoWeek', prev: start, usage: 0, cap: 12},
{range: 'month', prev: start, usage: 0, cap: 36},
{range: 'year', prev: start, usage: 0, cap: 1000}
];
// For each snapshot starting with newest, check if it is worth saving by comparing
// it with the last saved snapshot based on hour, day, week, month, year
return snapshots.map((snapshot, index) => {
let keep = index < 5; // Keep 5 most recent versions
const date = moment.utc(snapshot.lastModified);
for (const bucket of buckets) {
if (updateAndCheckRange(date, bucket)) { keep = true; }
}
return keep;
});
}
/**
* Check whether time `t` is in the same time-bucket as the time
* stored in `prev` for that time-bucket, and the time-bucket has not
* been used to its limit to justify saving versions.
*
* If all is good, we return true, store `t` in the appropriate
* time-bucket in `prev`, and increment the usage count. Note keeping
* a single version can increment usage on several buckets. This is
* easy to change, but other variations have results that feel
* counter-intuitive.
*/
function updateAndCheckRange(t: moment.Moment, bucket: TimeBucket) {
if (bucket.usage < bucket.cap && !t.isSame(bucket.prev, bucket.range)) {
bucket.prev = t;
bucket.usage++;
return true;
}
return false;
}
interface TimeBucket {
range: 'hour' | 'day' | 'isoWeek' | 'month' | 'year',
prev: moment.Moment; // last time stored in this bucket
usage: number; // number of times this bucket justified saving a snapshot
cap: number; // maximum number of usages permitted
}

1353
app/server/lib/DocStorage.ts Normal file
View File

File diff suppressed because it is too large Load Diff

352
app/server/lib/DocStorageManager.ts Normal file
View File

@@ -0,0 +1,352 @@
import * as bluebird from 'bluebird';
import * as chokidar from 'chokidar';
import * as fse from 'fs-extra';
import * as moment from 'moment';
import * as path from 'path';
import {DocEntry, DocEntryTag} from 'app/common/DocListAPI';
import * as gutil from 'app/common/gutil';
import * as Comm from 'app/server/lib/Comm';
import {OptDocSession} from 'app/server/lib/DocSession';
import {DocSnapshots} from 'app/server/lib/DocSnapshots';
import * as docUtils from 'app/server/lib/docUtils';
import {GristServer} from 'app/server/lib/GristServer';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import {IShell} from 'app/server/lib/IShell';
import * as log from 'app/server/lib/log';
import * as uuidv4 from "uuid/v4";
/**
* DocStorageManager manages Grist documents. This implementation deals with files in the file
* system. An alternative implementation could provide the same public methods to implement
* storage management for the hosted version of Grist.
*
* This file-based DocStorageManager uses file path as the docName identifying a document, with
* one exception. For files in the docsRoot directory, the basename of the document is used
* instead, with .grist extension stripped; primarily to maintain previous behavior and keep
* clean-looking URLs. In all other cases, the realpath of the file (including .grist extension)
* is the canonical docName.
*
*/
export class DocStorageManager implements IDocStorageManager {
private _watcher: any; // chokidar filesystem watcher
private _shell: IShell;
/**
* Initialize with the given root directory, which should be a fully-resolved path (i.e. using
* fs.realpath or docUtils.realPath).
* The file watcher is created if the optComm argument is given.
*/
constructor(private _docsRoot: string, private _samplesRoot?: string,
private _comm?: Comm, gristServer?: GristServer) {
// If we have a way to communicate with clients, watch the docsRoot for changes.
this._watcher = null;
this._shell = (gristServer && gristServer.create.Shell()) || {
moveItemToTrash() { throw new Error('Unable to move document to trash'); },
showItemInFolder() { throw new Error('Unable to show item in folder'); }
};
if (_comm) {
this._initFileWatcher();
}
}
/**
* Returns the path to the given document. This is used by DocStorage.js, and is specific to the
* file-based storage implementation.
* @param {String} docName: The canonical docName.
* @returns {String} path: Filesystem path.
*/
public getPath(docName: string): string {
docName += (path.extname(docName) === '.grist' ? '' : '.grist');
return path.resolve(this._docsRoot, docName);
}
/**
* Returns the path to the given sample document.
*/
public getSampleDocPath(sampleDocName: string): string|null {
return this._samplesRoot ? this.getPath(path.resolve(this._samplesRoot, sampleDocName)) : null;
}
/**
* Translates a possibly non-canonical docName to a canonical one (e.g. adds .grist to a path
* without .grist extension, and canonicalizes the path). All other functions deal with
* canonical docNames.
* @param {String} altDocName: docName which may not be the canonical one.
* @returns {Promise:String} Promise for the canonical docName.
*/
public async getCanonicalDocName(altDocName: string): Promise<string> {
const p = await docUtils.realPath(this.getPath(altDocName));
return path.dirname(p) === this._docsRoot ? path.basename(p, '.grist') : p;
}
/**
* Prepares a document for use locally. Returns whether the document is new (needs to be
* created). This is a no-op in the local DocStorageManager case.
*/
public async prepareLocalDoc(docName: string, docSession: OptDocSession): Promise<boolean> { return false; }
/**
* Returns a promise for the list of docNames to show in the doc list. For the file-based
* storage, this will include all .grist files under the docsRoot.
* @returns {Promise:Array<DocEntry>} Promise for an array of objects with `name`, `size`,
* and `mtime`.
*/
public listDocs(): Promise<DocEntry[]> {
return bluebird.Promise.all([
this._listDocs(this._docsRoot, ""),
this._samplesRoot ? this._listDocs(this._samplesRoot, "sample") : [],
])
.spread((docsEntries: DocEntry[], samplesEntries: DocEntry[]) => {
return [...docsEntries, ...samplesEntries];
});
}
/**
* Deletes a document.
* @param {String} docName: docName of the document to delete.
* @returns {Promise} Resolved on success.
*/
public deleteDoc(docName: string, deletePermanently?: boolean): Promise<void> {
const docPath = this.getPath(docName);
// Keep this check, to protect against wiping out the whole disk or the user's home.
if (path.extname(docPath) !== '.grist') {
return Promise.reject(new Error("Refusing to delete path which does not end in .grist"));
} else if (deletePermanently) {
return fse.remove(docPath);
} else {
this._shell.moveItemToTrash(docPath); // this is a synchronous action
return Promise.resolve();
}
}
/**
* Renames a closed document. In the file-system case, moves files to reflect the new paths. For
* a document already open, use `docStorageInstance.renameDocTo()` instead.
* @param {String} oldName: original docName.
* @param {String} newName: new docName.
* @returns {Promise} Resolved on success.
*/
public renameDoc(oldName: string, newName: string): Promise<void> {
const oldPath = this.getPath(oldName);
const newPath = this.getPath(newName);
return docUtils.createExclusive(newPath)
.catch(async (e: any) => {
if (e.code !== 'EEXIST') { throw e; }
const isSame = await docUtils.isSameFile(oldPath, newPath);
if (!isSame) { throw e; }
})
.then(() => fse.rename(oldPath, newPath))
// Send 'renameDocs' event immediately after the rename. Previously, this used to be sent by
// DocManager after reopening the renamed doc. The extra delay caused issue T407, where
// chokidar.watch() triggered 'removeDocs' before 'renameDocs'.
.then(() => { this._sendDocListAction('renameDocs', oldPath, [oldName, newName]); })
.catch((err: Error) => {
log.warn("DocStorageManager: rename %s -> %s failed: %s", oldPath, newPath, err.message);
throw err;
});
}
/**
* Should create a backup of the file
* @param {String} docName - docName to backup
* @param {String} backupTag - string to identify backup, like foo.grist.$DATE.$TAG.bak
* @returns {Promise} Resolved on success, returns path to backup (to show user)
*/
public makeBackup(docName: string, backupTag: string): Promise<string> {
// this need to persist between calling createNumbered and
// getting it's return value, to re-add the extension again (._.)
let ext: string;
let finalBakPath: string; // holds final value of path, with numbering
return bluebird.Promise.try(() => this._generateBackupFilePath(docName, backupTag))
.then((bakPath: string) => { // make a numbered migration if necessary
log.debug(`DocStorageManager: trying to make backup at ${bakPath}`);
// create a file at bakPath, adding numbers if necessary
ext = path.extname(bakPath); // persists to makeBackup closure
const bakPathPrefix = bakPath.slice(0, -ext.length);
return docUtils.createNumbered(bakPathPrefix, '-',
(pathPrefix: string) => docUtils.createExclusive(pathPrefix + ext)
);
}).tap((numberedBakPathPrefix: string) => { // do the copying, but return bakPath anyway
finalBakPath = numberedBakPathPrefix + ext;
const docPath = this.getPath(docName);
log.info(`Backing up ${docName} to ${finalBakPath}`);
return docUtils.copyFile(docPath, finalBakPath);
}).then(() => {
log.debug("DocStorageManager: Backup made successfully at: %s", finalBakPath);
return finalBakPath;
}).catch((err: Error) => {
log.error("DocStorageManager: Backup %s %s failed: %s", docName, err.message);
throw err;
});
}
/**
* Electron version only. Shows the given doc in the file explorer.
*/
public async showItemInFolder(docName: string): Promise<void> {
this._shell.showItemInFolder(await this.getPath(docName));
}
public async closeStorage() {
// nothing to do
}
public async closeDocument(docName: string) {
// nothing to do
}
public markAsChanged(docName: string): void {
// nothing to do
}
public markAsEdited(docName: string): void {
// nothing to do
}
public testReopenStorage(): void {
// nothing to do
}
public addToStorage(id: string): void {
// nothing to do
}
public prepareToCloseStorage(): void {
// nothing to do
}
public async flushDoc(docName: string): Promise<void> {
// nothing to do
}
public async getCopy(docName: string): Promise<string> {
const srcPath = this.getPath(docName);
const postfix = uuidv4();
const tmpPath = `${srcPath}-${postfix}`;
await docUtils.copyFile(srcPath, tmpPath);
return tmpPath;
}
public async getSnapshots(docName: string): Promise<DocSnapshots> {
throw new Error('getSnapshots not implemented');
}
public async replace(docName: string, options: any): Promise<void> {
throw new Error('replacement not implemented');
}
/**
* Returns a promise for the list of docNames for all docs in the given directory.
* @returns {Promise:Array<Object>} Promise for an array of objects with `name`, `size`,
* and `mtime`.
*/
private _listDocs(dirPath: string, tag: DocEntryTag): Promise<any[]> {
return fse.readdir(dirPath)
// Filter out for .grist files, and strip the .grist extension.
.then(entries => Promise.all(
entries.filter(e => (path.extname(e) === '.grist'))
.map(e => {
const docPath = path.resolve(dirPath, e);
return fse.stat(docPath)
.then(stat => getDocListFileInfo(docPath, stat, tag));
})
))
// Sort case-insensitively.
.then(entries => entries.sort((a, b) => a.name.toLowerCase().localeCompare(b.name.toLowerCase())))
// If the root directory is missing, just return an empty array.
.catch(err => {
if (err.cause && err.cause.code === 'ENOENT') { return []; }
throw err;
});
}
/**
* Generates the filename for the given document backup
* Backup names should look roughly like:
* ${basefilename}.grist.${YYYY-MM-DD}.${tag}.bak
*
* @returns {Promise} backup filepath (might need to createNumbered)
*/
private _generateBackupFilePath(docName: string, backupTag: string): Promise<string> {
const dateString = moment().format("YYYY-MM-DD");
return docUtils.realPath(this.getPath(docName))
.then((filePath: string) => {
const fileName = path.basename(filePath);
const fileDir = path.dirname(filePath);
const bakName = `${fileName}.${dateString}.${backupTag}.bak`;
return path.join(fileDir, bakName);
});
}
/**
* Creates the file watcher and begins monitoring the docsRoot. Returns the created watcher.
*/
private _initFileWatcher(): void {
// NOTE: The chokidar watcher reports file renames as unlink then add events.
this._watcher = chokidar.watch(this._docsRoot, {
ignoreInitial: true, // Prevent messages for initial adds of all docs when watching begins
depth: 0, // Ignore changes in subdirectories of docPath
alwaysStat: true, // Tells the watcher to always include the stats arg
// Waits for a file to remain constant for a short time after changing before triggering
// an action. Prevents reporting of incomplete writes.
awaitWriteFinish: {
stabilityThreshold: 100, // Waits for the file to remain constant for 100ms
pollInterval: 10 // Polls the file every 10ms after a change
}
});
this._watcher.on('add', (docPath: string, fsStats: any) => {
this._sendDocListAction('addDocs', docPath, getDocListFileInfo(docPath, fsStats, ""));
});
this._watcher.on('change', (docPath: string, fsStats: any) => {
this._sendDocListAction('changeDocs', docPath, getDocListFileInfo(docPath, fsStats, ""));
});
this._watcher.on('unlink', (docPath: string) => {
this._sendDocListAction('removeDocs', docPath, getDocName(docPath));
});
}
/**
* Helper to broadcast a docListAction for a single doc to clients. If the action is not on a
* '.grist' file, it is not sent.
* @param {String} actionType - DocListAction type to send, 'addDocs' | 'removeDocs' | 'changeDocs'.
* @param {String} docPath - System path to the doc including the filename.
* @param {Any} data - Data to send as the message.
*/
private _sendDocListAction(actionType: string, docPath: string, data: any): void {
if (this._comm && gutil.endsWith(docPath, '.grist')) {
log.debug(`Sending ${actionType} action for doc ${getDocName(docPath)}`);
this._comm.broadcastMessage('docListAction', { [actionType]: [data] });
}
}
}
/**
* Helper to return the docname (without .grist) given the path to the .grist file.
*/
function getDocName(docPath: string): string {
return path.basename(docPath, '.grist');
}
/**
* Helper to get the stats used by the Grist DocList for a document.
* @param {String} docPath - System path to the doc including the doc filename.
* @param {Object} fsStat - fs.Stats object describing the file metadata.
* @param {String} tag - The tag indicating the type of doc.
* @return {Promise:Object} Promise for an object containing stats for the requested doc.
*/
function getDocListFileInfo(docPath: string, fsStat: any, tag: DocEntryTag): DocEntry {
return {
docId: undefined, // TODO: Should include docId if it exists
name: getDocName(docPath),
mtime: fsStat.mtime,
size: fsStat.size,
tag
};
}

197
app/server/lib/DocWorker.ts Normal file
View File

@@ -0,0 +1,197 @@
/**
* DocWorker collects the methods and endpoints that relate to a single Grist document.
* In hosted environment, this comprises the functionality of the DocWorker instance type.
*/
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
import {ActiveDoc} from 'app/server/lib/ActiveDoc';
import {assertAccess, getOrSetDocAuth, getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
import {Client} from 'app/server/lib/Client';
import * as Comm from 'app/server/lib/Comm';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import * as log from 'app/server/lib/log';
import {integerParam, optStringParam, stringParam} from 'app/server/lib/requestUtils';
import {OpenMode, quoteIdent, SQLiteDB} from 'app/server/lib/SQLiteDB';
import {generateCSV} from 'app/server/serverMethods';
import * as contentDisposition from 'content-disposition';
import * as express from 'express';
import * as fse from 'fs-extra';
import * as mimeTypes from 'mime-types';
import * as path from 'path';
export interface AttachOptions {
comm: Comm; // Comm object for methods called via websocket
}
export class DocWorker {
private _comm: Comm;
constructor(private _dbManager: HomeDBManager, {comm}: AttachOptions) {
this._comm = comm;
}
public getCSV(req: express.Request, res: express.Response): void {
return generateCSV(req, res, this._comm);
}
public async getAttachment(req: express.Request, res: express.Response): Promise<void> {
try {
const client = this._comm.getClient(stringParam(req.query.clientId));
const activeDoc = this._getActiveDoc(stringParam(req.query.clientId),
integerParam(req.query.docFD));
const ext = path.extname(stringParam(req.query.ident));
const type = mimeTypes.lookup(ext);
let inline = Boolean(req.query.inline);
// Serving up user-uploaded HTML files inline is an open door to XSS attacks.
if (type === "text/html") { inline = false; }
// Construct a content-disposition header of the form 'inline|attachment; filename="NAME"'
const contentDispType = inline ? "inline" : "attachment";
const contentDispHeader = contentDisposition(stringParam(req.query.name), {type: contentDispType});
const data = await activeDoc.getAttachmentData(client, stringParam(req.query.ident));
res.status(200)
.type(ext)
.set('Content-Disposition', contentDispHeader)
.set('Cache-Control', 'private, max-age=3600')
.send(data);
} catch (err) {
res.status(404).send({error: err.toString()});
}
}
public async downloadDoc(req: express.Request, res: express.Response,
storageManager: IDocStorageManager): Promise<void> {
const mreq = req as RequestWithLogin;
if (!mreq.docAuth || !mreq.docAuth.docId) { throw new Error('Cannot find document'); }
const docId = mreq.docAuth.docId;
// Query DB for doc metadata to get the doc title.
const doc = await this._dbManager.getDoc({userId: getUserId(req), org: mreq.org, urlId: docId});
const docTitle = doc.name;
// Get a copy of document for downloading.
const tmpPath = await storageManager.getCopy(docId);
if (req.query.template === '1') {
// If template flag is on, remove data and history from the download.
await removeData(tmpPath);
}
// NOTE: We may want to reconsider the mimeType used for Grist files.
return res.type('application/x-sqlite3')
.download(tmpPath, (optStringParam(req.query.title) || docTitle || 'document') + ".grist", async (err: any) => {
if (err) {
log.error(`Download failure for doc ${docId}`, err);
}
await fse.unlink(tmpPath);
});
}
// Register main methods related to documents.
public registerCommCore(): void {
const comm = this._comm;
comm.registerMethods({
closeDoc: activeDocMethod.bind(null, null, 'closeDoc'),
fetchTable: activeDocMethod.bind(null, 'viewers', 'fetchTable'),
fetchTableSchema: activeDocMethod.bind(null, 'viewers', 'fetchTableSchema'),
useQuerySet: activeDocMethod.bind(null, 'viewers', 'useQuerySet'),
disposeQuerySet: activeDocMethod.bind(null, 'viewers', 'disposeQuerySet'),
applyUserActions: activeDocMethod.bind(null, 'editors', 'applyUserActions'),
applyUserActionsById: activeDocMethod.bind(null, 'editors', 'applyUserActionsById'),
findColFromValues: activeDocMethod.bind(null, 'viewers', 'findColFromValues'),
getFormulaError: activeDocMethod.bind(null, 'viewers', 'getFormulaError'),
importFiles: activeDocMethod.bind(null, 'editors', 'importFiles'),
finishImportFiles: activeDocMethod.bind(null, 'editors', 'finishImportFiles'),
cancelImportFiles: activeDocMethod.bind(null, 'editors', 'cancelImportFiles'),
addAttachments: activeDocMethod.bind(null, 'editors', 'addAttachments'),
removeInstanceFromDoc: activeDocMethod.bind(null, 'editors', 'removeInstanceFromDoc'),
startBundleUserActions: activeDocMethod.bind(null, 'editors', 'startBundleUserActions'),
stopBundleUserActions: activeDocMethod.bind(null, 'editors', 'stopBundleUserActions'),
autocomplete: activeDocMethod.bind(null, 'viewers', 'autocomplete'),
fetchURL: activeDocMethod.bind(null, 'viewers', 'fetchURL'),
getActionSummaries: activeDocMethod.bind(null, 'viewers', 'getActionSummaries'),
reloadDoc: activeDocMethod.bind(null, 'editors', 'reloadDoc'),
fork: activeDocMethod.bind(null, 'viewers', 'fork'),
});
}
// Register methods related to plugins.
public registerCommPlugin(): void {
this._comm.registerMethods({
forwardPluginRpc: activeDocMethod.bind(null, 'editors', 'forwardPluginRpc'),
// TODO: consider not providing reloadPlugins on hosted grist, since it affects the
// plugin manager shared across docs on a given doc worker, and seems useful only in
// standalone case.
reloadPlugins: activeDocMethod.bind(null, 'editors', 'reloadPlugins'),
});
}
// Checks that document is accessible, and adds docAuth information to request.
// Otherwise issues a 403 access denied.
// (This is used for endpoints like /download, /gen-csv, /attachment.)
public async assertDocAccess(
req: express.Request,
res: express.Response,
next: express.NextFunction
) {
const mreq = req as RequestWithLogin;
let urlId: string|undefined;
try {
if (optStringParam(req.query.clientId)) {
const activeDoc = this._getActiveDoc(stringParam(req.query.clientId),
integerParam(req.query.docFD));
// TODO: The docId should be stored in the ActiveDoc class. Currently docName is
// used instead, which will coincide with the docId for hosted grist but not for
// standalone grist.
urlId = activeDoc.docName;
} else {
// Otherwise, if being used without a client, expect the doc query parameter to
// be the docId.
urlId = stringParam(req.query.doc);
}
if (!urlId) { return res.status(403).send({error: 'missing document id'}); }
const docAuth = await getOrSetDocAuth(mreq, this._dbManager, urlId);
assertAccess('viewers', docAuth);
next();
} catch (err) {
log.info(`DocWorker can't access document ${urlId} with userId ${mreq.userId}: ${err}`);
res.status(err.status || 404).send({error: err.toString()});
}
}
private _getActiveDoc(clientId: string, docFD: number): ActiveDoc {
const client = this._comm.getClient(clientId);
const docSession = client.getDocSession(docFD);
return docSession.activeDoc;
}
}
/**
* Translates calls from the browser client into calls of the form
* `activeDoc.method(docSession, ...args)`.
*/
async function activeDocMethod(role: 'viewers'|'editors'|null, methodName: string, client: Client,
docFD: number, ...args: any[]): Promise<any> {
const docSession = client.getDocSession(docFD);
const activeDoc = docSession.activeDoc;
if (role) { await docSession.authorizer.assertAccess(role); }
// Include a basic log record for each ActiveDoc method call.
log.rawDebug('activeDocMethod', activeDoc.getLogMeta(client, methodName));
return (activeDoc as any)[methodName](docSession, ...args);
}
/**
* Remove rows from all user tables, and wipe as much history as we can.
*/
async function removeData(filename: string) {
const db = await SQLiteDB.openDBRaw(filename, OpenMode.OPEN_EXISTING);
const tableIds = (await db.all("SELECT name FROM sqlite_master WHERE type='table'"))
.map(row => row.name as string)
.filter(name => !name.startsWith('_grist'));
for (const tableId of tableIds) {
await db.run(`DELETE FROM ${quoteIdent(tableId)}`);
}
const history = new ActionHistoryImpl(db);
await history.deleteActions(1);
await db.run('VACUUM');
await db.close();
}

62
app/server/lib/DocWorkerMap.ts Normal file
View File

@@ -0,0 +1,62 @@
/**
* Defines the IDocWorkerMap interface we need to assign a DocWorker to a doc, and to look it up.
* TODO This is not yet implemented, there is only a hard-coded stub.
*/
import { IElectionStore } from 'app/server/lib/IElectionStore';
import { IPermitStore } from 'app/server/lib/Permit';
export interface DocWorkerInfo {
id: string;
// The public base URL for the docWorker, which tells the browser how to connect to it. E.g.
// https://docworker-17.getgrist.com/ or http://localhost:8080/v/gtag/
publicUrl: string;
// The internal base URL for the docWorker.
internalUrl: string;
}
export interface DocStatus {
// MD5 hash of the SQLite file for this document as stored on S3. We use MD5 because it is
// automatically computed by S3 (except for multipart uploads). Null indicates a new file.
docMD5: string|null;
// DocWorker most recently, or currently, responsible for the file.
docWorker: DocWorkerInfo;
// Whether the file is currently open on this DocWorker.
isActive: boolean;
}
/**
* Assignment of documents to workers, and other storage related to distributed work.
*/
export interface IDocWorkerMap extends IPermitStore, IElectionStore {
// Looks up which DocWorker is responsible for this docId.
getDocWorker(docId: string): Promise<DocStatus|null>;
// Assigns a DocWorker to this docId if one is not yet assigned.
assignDocWorker(docId: string): Promise<DocStatus>;
// Assigns a particular DocWorker to this docId if one is not yet assigned.
getDocWorkerOrAssign(docId: string, workerId: string): Promise<DocStatus>;
updateDocStatus(docId: string, checksum: string): Promise<void>;
addWorker(info: DocWorkerInfo): Promise<void>;
removeWorker(workerId: string): Promise<void>;
// Set whether worker is accepting new assignments. This does not automatically
// release existing assignments.
setWorkerAvailability(workerId: string, available: boolean): Promise<void>;
// Releases doc from worker, freeing it to be assigned elsewhere.
// Assigments should only be released for workers that are now unavailable.
releaseAssignment(workerId: string, docId: string): Promise<void>;
// Get all assignments for a worker. Should only be queried for a worker that
// is currently unavailable.
getAssignments(workerId: string): Promise<string[]>;
}

135
app/server/lib/ExpandedQuery.ts Normal file
View File

@@ -0,0 +1,135 @@
import {Query} from 'app/common/ActiveDocAPI';
import {ApiError} from 'app/common/ApiError';
import {DocData} from 'app/common/DocData';
import {parseFormula} from 'app/common/Formula';
import {removePrefix} from 'app/common/gutil';
import {quoteIdent} from 'app/server/lib/SQLiteDB';
/**
* Represents a query for Grist data with support for SQL-based
* formulas. Use of this representation should be limited to within a
* trusted part of Grist since it assembles SQL strings.
*/
export interface ExpandedQuery extends Query {
// Errors detected for given columns because of formula issues. We
// need to make sure the result of the query contains these error
// objects. It is awkward to write a sql selection that constructs
// an error object, so instead we select 0 in the case of an error,
// and substitute in the error object in javascript after the SQL
// step. That means we need to pass the error message along
// explicitly.
constants?: {
[colId: string]: ['E', string] | ['P'];
};
// A list of join clauses to bring in data from other tables.
joins?: string[];
// A list of selections for regular data and data computed via formulas.
selects?: string[];
}
/**
* Add JOINs and SELECTs to a query in order to implement formulas via SQL.
*
* Supports simple formulas that load a column via a reference.
* The referenced column itself cannot (yet) be a formula.
* Filtered columns cannot (yet) be a formula.
*
* If formulas is not set, we simply mark formula columns as pending.
*/
export function expandQuery(iquery: Query, docData: DocData, formulas: boolean = true): ExpandedQuery {
const query: ExpandedQuery = {
tableId: iquery.tableId,
filters: iquery.filters,
limit: iquery.limit
};
// Look up the main table for the query.
const tables = docData.getTable('_grist_Tables')!;
const columns = docData.getTable('_grist_Tables_column')!;
const tableRef = tables.findRow('tableId', query.tableId);
if (!tableRef) { throw new ApiError('table not found', 404); }
// Find any references to other tables.
const dataColumns = columns.filterRecords({parentId: tableRef, isFormula: false});
const references = new Map<string, string>();
for (const column of dataColumns) {
const refTableId = removePrefix(column.type as string, 'Ref:');
if (refTableId) { references.set(column.colId as string, refTableId); }
}
// Start accumulating a set of joins and selects needed for the query.
const joins = new Set<string>();
const selects = new Set<string>();
// Select all data columns
selects.add(`${quoteIdent(query.tableId)}.*`);
// Iterate through all formulas, adding joins and selects as we go.
if (formulas) {
const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true});
for (const column of formulaColumns) {
const formula = parseFormula(column.formula as string);
const colId = column.colId as string;
let sqlFormula = "";
let error = "";
if (formula.kind === 'foreignColumn') {
const altTableId = references.get(formula.refColId);
const altTableRef = tables.findRow('tableId', altTableId);
if (altTableId && altTableRef) {
const altColumn = columns.filterRecords({parentId: altTableRef, isFormula: false, colId: formula.colId});
// TODO: deal with a formula column in the other table.
if (altColumn.length > 0) {
const alias = `${query.tableId}_${formula.refColId}`;
joins.add(`LEFT JOIN ${quoteIdent(altTableId)} AS ${quoteIdent(alias)} ` +
`ON ${quoteIdent(alias)}.id = ` +
`${quoteIdent(query.tableId)}.${quoteIdent(formula.refColId)}`);
sqlFormula = `${quoteIdent(alias)}.${quoteIdent(formula.colId)}`;
} else {
error = "Cannot find column";
}
} else {
error = "Cannot find table";
}
} else if (formula.kind === 'column') {
const altColumn = columns.filterRecords({parentId: tableRef, isFormula: false, colId: formula.colId});
// TODO: deal with a formula column.
if (altColumn.length > 0) {
sqlFormula = `${quoteIdent(query.tableId)}.${quoteIdent(formula.colId)}`;
} else {
error = "Cannot find column";
}
} else if (formula.kind === 'literalNumber') {
sqlFormula = `${formula.value}`;
} else if (formula.kind === 'error') {
error = formula.msg;
} else {
throw new Error('Unrecognized type of formula');
}
if (error) {
// We add a trivial selection, and store errors in the query for substitution later.
sqlFormula = '0';
if (!query.constants) { query.constants = {}; }
query.constants[colId] = ['E', error];
}
if (sqlFormula) {
selects.add(`${sqlFormula} as ${quoteIdent(colId)}`);
}
}
} else {
const formulaColumns = columns.filterRecords({parentId: tableRef, isFormula: true});
for (const column of formulaColumns) {
if (!column.formula) { continue; } // Columns like this won't get calculated, so skip.
const colId = column.colId as string;
if (!query.constants) { query.constants = {}; }
query.constants[colId] = ['P'];
selects.add(`0 as ${quoteIdent(colId)}`);
}
}
// Copy decisions to the query object, and return.
query.joins = [...joins];
query.selects = [...selects];
return query;
}

320
app/server/lib/ExternalStorage.ts Normal file
View File

@@ -0,0 +1,320 @@
import {ObjSnapshot} from 'app/server/lib/DocSnapshots';
import * as log from 'app/server/lib/log';
import {createTmpDir} from 'app/server/lib/uploads';
import {delay} from 'bluebird';
import * as fse from 'fs-extra';
import * as path from 'path';
// A special token representing a deleted document, used in places where a
// checksum is expected otherwise.
export const DELETED_TOKEN = '*DELETED*';
/**
* An external store for the content of files. The store may be either consistent
* or eventually consistent. Specifically, the `exists`, `download`, and `versions`
* methods may return somewhat stale data from time to time.
*
* The store should be versioned; that is, uploads to a `key` should be assigned
* a `snapshotId`, and be accessible later with that `key`/`snapshotId` pair.
* When data is accessed by `snapshotId`, results should be immediately consistent.
*/
export interface ExternalStorage {
// Check if content exists in the store for a given key.
exists(key: string): Promise<boolean>;
// Upload content from file to the given key. Returns a snapshotId if store supports that.
upload(key: string, fname: string): Promise<string|null>;
// Download content from key to given file. Can download a specific version of the key
// if store supports that (should throw a fatal exception if not).
download(key: string, fname: string, snapshotId?: string): Promise<void>;
// Remove content for this key from the store, if it exists. Can delete specific versions
// if specified. If no version specified, all versions are removed. If versions specified,
// newest should be given first.
remove(key: string, snapshotIds?: string[]): Promise<void>;
// List content versions that exist for the given key. More recent versions should
// come earlier in the result list.
versions(key: string): Promise<ObjSnapshot[]>;
// Render the given key as something url-like, for log messages (e.g. "s3://bucket/path")
url(key: string): string;
// Check if an exception thrown by a store method should be treated as fatal.
// Non-fatal exceptions are those that may result from eventual consistency, and
// where a retry could help -- specifically "not found" exceptions.
isFatalError(err: any): boolean;
// Close the storage object.
close(): Promise<void>;
}
/**
* Convenience wrapper to transform keys for an external store.
* E.g. this could convert "<docId>" to "v1/<docId>.grist"
*/
export class KeyMappedExternalStorage implements ExternalStorage {
constructor(private _ext: ExternalStorage,
private _map: (key: string) => string) {
}
public exists(key: string): Promise<boolean> {
return this._ext.exists(this._map(key));
}
public upload(key: string, fname: string) {
return this._ext.upload(this._map(key), fname);
}
public download(key: string, fname: string, snapshotId?: string) {
return this._ext.download(this._map(key), fname, snapshotId);
}
public remove(key: string, snapshotIds?: string[]): Promise<void> {
return this._ext.remove(this._map(key), snapshotIds);
}
public versions(key: string) {
return this._ext.versions(this._map(key));
}
public url(key: string) {
return this._ext.url(this._map(key));
}
public isFatalError(err: any) {
return this._ext.isFatalError(err);
}
public async close() {
// nothing to do
}
}
/**
* A wrapper for an external store that uses checksums and retries
* to compensate for eventual consistency. With this wrapper, the
* store either returns consistent results or fails with an error.
*
* This wrapper works by tracking what is in the external store,
* using content hashes and ids placed in consistent stores. These
* consistent stores are:
*
* - sharedHash: a key/value store containing expected checksums
* of content in the external store. In our setup, this is
* implemented using Redis. Populated on upload and checked on
* download.
* - localHash: a key/value store containing checksums of uploaded
* content. In our setup, this is implemented on the worker's
* disk. This is used to skip unnecessary uploads. Populated
* on download and checked on upload.
* - latestVersion: a key/value store containing snapshotIds of
* uploads. In our setup, this is implemented in the worker's
* memory. Only affects the consistency of the `versions` method.
* Populated on upload and checked on `versions` calls.
* TODO: move to Redis if consistency of `versions` during worker
* transitions becomes important.
*
* It is not important for all this side information to persist very
* long, just long enough to give the store time to become
* consistent.
*
* Keys presented to this class should be file-system safe.
*/
export class ChecksummedExternalStorage implements ExternalStorage {
private _closed: boolean = false;
constructor(private _ext: ExternalStorage, private _options: {
maxRetries: number, // how many time to retry inconsistent downloads
initialDelayMs: number, // how long to wait before retrying
localHash: PropStorage, // key/value store for hashes of downloaded content
sharedHash: PropStorage, // key/value store for hashes of external content
latestVersion: PropStorage, // key/value store for snapshotIds of uploads
computeFileHash: (fname: string) => Promise<string>, // compute hash for file
}) {
}
public async exists(key: string): Promise<boolean> {
return this._retry('exists', async () => {
const hash = await this._options.sharedHash.load(key);
const expected = hash !== null && hash !== DELETED_TOKEN;
const reported = await this._ext.exists(key);
// If we expect an object but store doesn't seem to have it, retry.
if (expected && !reported) { return undefined; }
// If store says there is an object but that is not what we expected (if we
// expected anything), retry.
if (hash && !expected && reported) { return undefined; }
// If expectations are matched, or we don't have expectations, return.
return reported;
});
}
public async upload(key: string, fname: string) {
try {
const checksum = await this._options.computeFileHash(fname);
const prevChecksum = await this._options.localHash.load(key);
if (prevChecksum && prevChecksum === checksum) {
// nothing to do, checksums match
log.info("ext upload: %s unchanged, not sending", key);
return this._options.latestVersion.load(key);
}
const snapshotId = await this._ext.upload(key, fname);
log.info("ext upload: %s checksum %s", this._ext.url(key), checksum);
if (snapshotId) { await this._options.latestVersion.save(key, snapshotId); }
await this._options.localHash.save(key, checksum);
await this._options.sharedHash.save(key, checksum);
return snapshotId;
} catch (err) {
log.error("ext upload: %s failure to send, error %s", key, err.message);
throw err;
}
}
public async remove(key: string, snapshotIds?: string[]) {
try {
// Removing most recent version by id is not something we should be doing, and
// if we want to do it it would need to be done carefully - so just forbid it.
if (snapshotIds && snapshotIds.includes(await this._options.latestVersion.load(key) || '')) {
throw new Error('cannot remove most recent version of a document by id');
}
await this._ext.remove(key, snapshotIds);
log.info("ext remove: %s version %s", this._ext.url(key), snapshotIds || 'ALL');
if (!snapshotIds) {
await this._options.latestVersion.save(key, DELETED_TOKEN);
await this._options.sharedHash.save(key, DELETED_TOKEN);
}
} catch (err) {
log.error("ext delete: %s failure to remove, error %s", key, err.message);
throw err;
}
}
public download(key: string, fname: string, snapshotId?: string) {
return this.downloadTo(key, key, fname, snapshotId);
}
/**
* We may want to download material from one key and henceforth treat it as another
* key (specifically for forking a document). Since this class crossreferences the
* key in the external store with other consistent stores, it needs to know we are
* doing that. So we add a downloadTo variant that takes before and after keys.
*/
public async downloadTo(fromKey: string, toKey: string, fname: string, snapshotId?: string) {
await this._retry('download', async () => {
const {tmpDir, cleanupCallback} = await createTmpDir({});
const tmpPath = path.join(tmpDir, `${toKey}.grist-tmp`); // NOTE: assumes key is file-system safe.
try {
await this._ext.download(fromKey, tmpPath, snapshotId);
const checksum = await this._options.computeFileHash(tmpPath);
// Check for consistency if mutable data fetched.
if (!snapshotId) {
const expectedChecksum = await this._options.sharedHash.load(fromKey);
// Let null docMD5s pass. Otherwise we get stuck if redis is cleared.
// Otherwise, make sure what we've got matches what we expect to get.
// S3 is eventually consistent - if you overwrite an object in it, and then read from it,
// you may get an old version for some time.
// If a snapshotId was specified, we can skip this check.
if (expectedChecksum && expectedChecksum !== checksum) {
log.error("ext download: data for %s has wrong checksum: %s (expected %s)", fromKey,
checksum,
expectedChecksum);
return undefined;
}
}
// If successful, rename the temporary file to its proper name. The destination should NOT
// exist in this case, and this should fail if it does.
await fse.move(tmpPath, fname, {overwrite: false});
await this._options.localHash.save(toKey, checksum);
log.info("ext download: %s%s%s with checksum %s", fromKey,
snapshotId ? ` [VersionId ${snapshotId}]` : '',
fromKey !== toKey ? ` as ${toKey}` : '',
checksum);
return true;
} catch (err) {
log.error("ext download: failed to fetch data (%s): %s", fromKey, err.message);
throw err;
} finally {
await cleanupCallback();
}
});
}
public async versions(key: string) {
return this._retry('versions', async () => {
const snapshotId = await this._options.latestVersion.load(key);
if (snapshotId === DELETED_TOKEN) { return []; }
const result = await this._ext.versions(key);
if (snapshotId && (result.length === 0 || result[0].snapshotId !== snapshotId)) {
// Result is not consistent yet.
return undefined;
}
return result;
});
}
public url(key: string): string {
return this._ext.url(key);
}
public isFatalError(err: any): boolean {
return this._ext.isFatalError(err);
}
public async close() {
this._closed = true;
}
/**
* Call an operation until it returns a value other than undefined.
*
* While the operation returns undefined, it will be retried for some period.
* This period is chosen to be long enough for S3 to become consistent.
*
* If the operation throws an error, and that error is not fatal (as determined
* by `isFatalError`, then it will also be retried. Fatal errors are thrown
* immediately.
*
* Once the operation returns a result, we pass that along. If it fails to
* return a result after all the allowed retries, a special exception is thrown.
*/
private async _retry<T>(name: string, operation: () => Promise<T|undefined>): Promise<T> {
let backoffCount = 1;
let backoffFactor = this._options.initialDelayMs;
const problems = new Array<[number, string|Error]>();
const start = Date.now();
while (backoffCount <= this._options.maxRetries) {
try {
const result = await operation();
if (result !== undefined) { return result; }
problems.push([Date.now() - start, 'not ready']);
} catch (err) {
if (this._ext.isFatalError(err)) {
throw err;
}
problems.push([Date.now() - start, err]);
}
// Wait some time before attempting to reload from s3. The longer we wait, the greater
// the odds of success. In practice, a second should be more than enough almost always.
await delay(Math.round(backoffFactor));
if (this._closed) { throw new Error('storage closed'); }
backoffCount++;
backoffFactor *= 1.7;
}
log.error(`operation failed to become consistent: ${name} - ${problems}`);
throw new Error(`operation failed to become consistent: ${name} - ${problems}`);
}
}
/**
* Small interface for storing hashes and ids.
*/
export interface PropStorage {
save(key: string, val: string): Promise<void>;
load(key: string): Promise<string|null>;
}

49
app/server/lib/FileParserElement.ts Normal file
View File

@@ -0,0 +1,49 @@
import {PluginInstance} from 'app/common/PluginInstance';
import {ParseFileAPI} from 'app/plugin/FileParserAPI';
import {checkers} from 'app/plugin/TypeCheckers';
import {FileParser} from 'app/plugin/PluginManifest';
import * as path from 'path';
/**
* Encapsulates together a file parse contribution with its plugin instance and callable stubs for
* `parseFile` implementation provided by the plugin.
*
* Implements as well a `getMatching` static method to get all file parsers matching a filename from
* the list of plugin instances.
*
*/
export class FileParserElement {
/**
* Get all file parser that matches fileName from the list of plugins instances.
*/
public static getMatching(pluginInstances: PluginInstance[], fileName: string): FileParserElement[] {
const fileParserElements: FileParserElement[] = [];
for (const plugin of pluginInstances) {
const fileParsers = plugin.definition.manifest.contributions.fileParsers;
if (fileParsers) {
for (const fileParser of fileParsers) {
if (matchFileParser(fileParser, fileName)) {
fileParserElements.push(new FileParserElement(plugin, fileParser));
}
}
}
}
return fileParserElements;
}
public parseFileStub: ParseFileAPI;
private constructor(public plugin: PluginInstance, public fileParser: FileParser) {
this.parseFileStub = plugin.getStub<ParseFileAPI>(fileParser.parseFile, checkers.ParseFileAPI);
}
}
function matchFileParser(fileParser: FileParser, fileName: string): boolean {
const ext = path.extname(fileName).slice(1),
fileExtensions = fileParser.fileExtensions;
return fileExtensions && fileExtensions.includes(ext);
}

1423
app/server/lib/FlexServer.ts Normal file
View File

File diff suppressed because it is too large Load Diff

26
app/server/lib/GristServer.ts Normal file
View File

@@ -0,0 +1,26 @@
import {SessionUserObj} from 'app/server/lib/BrowserSession';
import * as Comm from 'app/server/lib/Comm';
import {Hosts} from 'app/server/lib/extractOrg';
import {ICreate} from 'app/server/lib/ICreate';
import {Sessions} from 'app/server/lib/Sessions';
import * as express from 'express';
/**
* Basic information about a Grist server. Accessible in many
* contexts, including request handlers and ActiveDoc methods.
*/
export interface GristServer {
readonly create: ICreate;
getHost(): string;
getHomeUrl(req: express.Request, relPath?: string): string;
getHomeUrlByDocId(docId: string, relPath?: string): Promise<string>;
}
export interface GristLoginMiddleware {
getLoginRedirectUrl(target: URL): Promise<string>;
getSignUpRedirectUrl(target: URL): Promise<string>;
getLogoutRedirectUrl(nextUrl: URL, userSession: SessionUserObj): Promise<string>;
// Returns arbitrary string for log.
addEndpoints(app: express.Express, comm: Comm, sessions: Sessions, hosts: Hosts): string;
}

89
app/server/lib/HostedMetadataManager.ts Normal file
View File

@@ -0,0 +1,89 @@
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import * as log from 'app/server/lib/log';
/**
* HostedMetadataManager handles pushing document metadata changes to the Home database when
* a doc is updated. Currently only updates doc updatedAt time.
*/
export class HostedMetadataManager {
// updatedAt times as UTC ISO strings mapped by docId.
private _updatedAt: {[docId: string]: string} = {};
// Set if the class holder is closing and no further pushes should be scheduled.
private _closing: boolean = false;
// Last push time in ms since epoch.
private _lastPushTime: number = 0.0;
// Callback for next opportunity to push changes.
private _timeout: any = null;
// Mantains the update Promise to wait on it if the class is closing.
private _push: Promise<any>|null;
/**
* Create an instance of HostedMetadataManager.
* The minPushDelay is the delay in seconds between metadata pushes to the database.
*/
constructor(private _dbManager: HomeDBManager, private _minPushDelay: number = 60) {}
/**
* Close the manager. Send out any pending updates and prevent more from being scheduled.
*/
public async close(): Promise<void> {
// Finish up everything outgoing
this._closing = true; // Pushes will no longer be scheduled.
if (this._timeout) {
clearTimeout(this._timeout);
this._timeout = null;
// Since an update was scheduled, perform one final update now.
this._update();
}
if (this._push) { await this._push; }
}
/**
* Schedule a call to _update some time from now.
*/
public scheduleUpdate(docId: string): void {
// Update updatedAt even if an update is already scheduled - if the update has not yet occurred,
// the more recent updatedAt time will be used.
this._updatedAt[docId] = new Date().toISOString();
if (this._timeout || this._closing) { return; }
const minDelay = this._minPushDelay * 1000;
// Set the push to occur at least the minDelay after the last push time.
const delay = Math.round(minDelay - (Date.now() - this._lastPushTime));
this._timeout = setTimeout(() => this._update(), delay < 0 ? 0 : delay);
}
public setDocsUpdatedAt(docUpdateMap: {[docId: string]: string}): Promise<any> {
return this._dbManager.setDocsUpdatedAt(docUpdateMap);
}
/**
* Push all metadata updates to the databse.
*/
private _update(): void {
if (this._push) { return; }
if (this._timeout) {
clearTimeout(this._timeout);
this._timeout = null;
}
this._push = this._performUpdate()
.catch(err => { log.error("HostedMetadataManager error performing update: ", err); })
.then(() => { this._push = null; });
}
/**
* This is called by the update function to actually perform the update. This should not
* be called unless to force an immediate update.
*/
private async _performUpdate(): Promise<void> {
// Await the database if it is not yet connected.
const docUpdates = this._updatedAt;
this._updatedAt = {};
this._lastPushTime = Date.now();
await this.setDocsUpdatedAt(docUpdates);
}
}

713
app/server/lib/HostedStorageManager.ts Normal file
View File

@@ -0,0 +1,713 @@
import * as sqlite3 from '@gristlabs/sqlite3';
import {mapGetOrSet} from 'app/common/AsyncCreate';
import {delay} from 'app/common/delay';
import {DocEntry} from 'app/common/DocListAPI';
import {buildUrlId, parseUrlId} from 'app/common/gristUrls';
import {KeyedOps} from 'app/common/KeyedOps';
import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
import {getUserId} from 'app/server/lib/Authorizer';
import {checksumFile} from 'app/server/lib/checksumFile';
import {OptDocSession} from 'app/server/lib/DocSession';
import {DocSnapshotPruner, DocSnapshots} from 'app/server/lib/DocSnapshots';
import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage} from 'app/server/lib/ExternalStorage';
import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
import {ICreate} from 'app/server/lib/ICreate';
import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
import * as log from 'app/server/lib/log';
import {fromCallback} from 'app/server/lib/serverUtils';
import * as fse from 'fs-extra';
import * as path from 'path';
import * as uuidv4 from "uuid/v4";
// Check for a valid document id.
const docIdRegex = /^[-=_\w~%]+$/;
// Wait this long after a change to the document before trying to make a backup of it.
const GRIST_BACKUP_DELAY_SECS = parseInt(process.env.GRIST_BACKUP_DELAY_SECS || '15', 10);
// This constant controls how many pages of the database we back up in a single step.
// The larger it is, the faster the backup overall, but the slower each step is.
// Slower steps result in longer periods when the database is locked, without any
// opportunity for a waiting client to get in and make a write.
// The size of a page, as far as sqlite is concerned, is 4096 bytes.
const PAGES_TO_BACKUP_PER_STEP = 1024; // Backup is made in 4MB chunks.
// Between steps of the backup, we pause in case a client is waiting to make a write.
// The shorter the pause, the greater the odds that the client won't be able to make
// its write, but the faster the backup will complete.
const PAUSE_BETWEEN_BACKUP_STEPS_IN_MS = 10;
function checkValidDocId(docId: string): void {
if (!docIdRegex.test(docId)) {
throw new Error(`Invalid docId ${docId}`);
}
}
interface HostedStorageOptions {
secondsBeforePush: number;
secondsBeforeFirstRetry: number;
pushDocUpdateTimes: boolean;
testExternalStorage?: ExternalStorage;
}
const defaultOptions: HostedStorageOptions = {
secondsBeforePush: GRIST_BACKUP_DELAY_SECS,
secondsBeforeFirstRetry: 3.0,
pushDocUpdateTimes: true
};
/**
* HostedStorageManager manages Grist files in the hosted environment for a particular DocWorker.
* These files are stored on S3 and synced to the local file system. It matches the interface of
* DocStorageManager (used for standalone Grist), but is more limited, e.g. does not expose the
* list of local files.
*
* In hosted environment, documents are uniquely identified by docId, which serves as the
* canonical docName. This ID does not change on renaming. HostedStorageManager knows nothing of
* friendlier doc names.
*
* TODO: Listen (to Redis?) to find out when a local document has been deleted or renamed.
* (In case of rename, something on the DocWorker needs to inform the client about the rename)
* TODO: Do something about the active flag in redis DocStatus.
* TODO: Add an explicit createFlag in DocStatus for clarity and simplification.
*/
export class HostedStorageManager implements IDocStorageManager {
// Handles pushing doc metadata changes when the doc is updated.
private _metadataManager: HostedMetadataManager|null = null;
// Maps docId to the promise for when the document is present on the local filesystem.
private _localFiles = new Map<string, Promise<boolean>>();
// Access external storage.
private _ext: ChecksummedExternalStorage;
// Prune external storage.
private _pruner: DocSnapshotPruner;
// If _disableS3 is set, don't actually communicate with S3 - keep everything local.
private _disableS3 = (process.env.GRIST_DISABLE_S3 === 'true');
// A set of filenames currently being created or downloaded.
private _prepareFiles = new Set<string>();
// Ongoing and scheduled uploads for documents.
private _uploads: KeyedOps;
// Set once the manager has been closed.
private _closed: boolean = false;
private _baseStore: ExternalStorage; // External store for documents, without checksumming.
/**
* Initialize with the given root directory, which should be a fully-resolved path.
* If s3Bucket is blank, S3 storage will be disabled.
*/
constructor(
private _docsRoot: string,
private _docWorkerId: string,
s3Bucket: string,
s3Prefix: string, // Should end in / if non-empty.
private _docWorkerMap: IDocWorkerMap,
dbManager: HomeDBManager,
create: ICreate,
options: HostedStorageOptions = defaultOptions
) {
if (s3Bucket === '') { this._disableS3 = true; }
// We store documents either in a test store, or in an s3 store
// at s3://<s3Bucket>/<s3Prefix><docId>.grist
const externalStore = options.testExternalStorage ||
(this._disableS3 ? undefined : create.ExternalStorage(s3Bucket, s3Prefix));
if (!externalStore) { this._disableS3 = true; }
const secondsBeforePush = options.secondsBeforePush;
const secondsBeforeFirstRetry = options.secondsBeforeFirstRetry;
if (options.pushDocUpdateTimes) {
this._metadataManager = new HostedMetadataManager(dbManager);
}
this._uploads = new KeyedOps(key => this._pushToS3(key), {
delayBeforeOperationMs: secondsBeforePush * 1000,
retry: true,
logError: (key, failureCount, err) => {
log.error("HostedStorageManager: error pushing %s (%d): %s", key, failureCount, err);
}
});
if (!this._disableS3) {
this._baseStore = externalStore!;
// Whichever store we have, we use checksums to deal with
// eventual consistency.
const versions = new Map<string, string>();
this._ext = new ChecksummedExternalStorage(this._baseStore, {
maxRetries: 4,
initialDelayMs: secondsBeforeFirstRetry * 1000,
computeFileHash: this._getHash.bind(this),
sharedHash: {
save: async (key, checksum) => {
await this._docWorkerMap.updateDocStatus(key, checksum);
},
load: async (key) => {
const docStatus = await this._docWorkerMap.getDocWorker(key);
return docStatus && docStatus.docMD5 || null;
}
},
localHash: {
save: async (key, checksum) => {
const fname = this._getHashFile(this.getPath(key));
await fse.writeFile(fname, checksum);
},
load: async (key) => {
const fname = this._getHashFile(this.getPath(key));
if (!await fse.pathExists(fname)) { return null; }
return await fse.readFile(fname, 'utf8');
}
},
latestVersion: {
save: async (key, ver) => {
versions.set(key, ver);
},
load: async (key) => versions.get(key) || null
}
});
// The pruner could use an inconsistent store without any real loss overall,
// but tests are easier if it is consistent.
this._pruner = new DocSnapshotPruner(this._ext, {
delayBeforeOperationMs: 0, // prune as soon as we've made a first upload.
minDelayBetweenOperationsMs: secondsBeforePush * 4000, // ... but wait awhile before
// pruning again.
});
}
}
/**
* Send a document to S3, without doing anything fancy. Assumes this is the first time
* the object is written in S3 - so no need to worry about consistency.
*/
public async addToStorage(docId: string) {
if (this._disableS3) { return; }
await this._ext.upload(docId, this.getPath(docId));
}
public getPath(docName: string): string {
// docName should just be a docId; we use basename to protect against some possible hack attempts.
checkValidDocId(docName);
return path.join(this._docsRoot, `${path.basename(docName, '.grist')}.grist`);
}
// We don't deal with sample docs
public getSampleDocPath(sampleDocName: string): string|null { return null; }
/**
* Translates a possibly non-canonical docName to a canonical one. Returns a bare docId,
* stripping out any possible path components or .grist extension. (We don't expect these to
* ever be used, but stripping seems better than asserting.)
*/
public async getCanonicalDocName(altDocName: string): Promise<string> {
return path.basename(altDocName, '.grist');
}
/**
* Prepares a document for use locally. Here we sync the doc from S3 to the local filesystem.
* Returns whether the document is new (needs to be created).
* Calling this method multiple times in parallel for the same document is treated as a sign
* of a bug.
*/
public async prepareLocalDoc(docName: string, docSession: OptDocSession): Promise<boolean> {
// We could be reopening a document that is still closing down.
// Wait for that to happen. TODO: we could also try to interrupt the closing-down process.
await this.closeDocument(docName);
if (this._prepareFiles.has(docName)) {
throw new Error(`Tried to call prepareLocalDoc('${docName}') twice in parallel`);
}
try {
this._prepareFiles.add(docName);
const isNew = !(await this._ensureDocumentIsPresent(docName, docSession));
return isNew;
} finally {
this._prepareFiles.delete(docName);
}
}
// Gets a copy of the document, eg. for downloading. Returns full file path.
// Copy won't change if edits are made to the document. It is caller's responsibility
// to delete the result.
public async getCopy(docName: string): Promise<string> {
const present = await this._ensureDocumentIsPresent(docName, {client: null});
if (!present) {
throw new Error('cannot copy document that does not exist yet');
}
return await this._prepareBackup(docName, uuidv4());
}
public async replace(docId: string, options: DocReplacementOptions): Promise<void> {
// Make sure the current version of the document is flushed.
await this.flushDoc(docId);
// Figure out the source s3 key to copy from. For this purpose, we need to
// remove any snapshotId embedded in the document id.
const rawSourceDocId = options.sourceDocId || docId;
const parts = parseUrlId(rawSourceDocId);
const sourceDocId = buildUrlId({...parts, snapshotId: undefined});
const snapshotId = options.snapshotId || parts.snapshotId;
if (sourceDocId === docId && !snapshotId) { return; }
// Basic implementation for when S3 is not available.
if (this._disableS3) {
if (snapshotId) {
throw new Error('snapshots not supported without S3');
}
if (await fse.pathExists(this.getPath(sourceDocId))) {
await fse.copy(this.getPath(sourceDocId), this.getPath(docId));
return;
} else {
throw new Error(`cannot find ${docId}`);
}
}
// While replacing, move the current version of the document aside. If a problem
// occurs, move it back.
const docPath = this.getPath(docId);
const tmpPath = `${docPath}-replacing`;
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(tmpPath);
if (await fse.pathExists(docPath)) {
await fse.move(docPath, tmpPath);
}
try {
// Fetch new content from S3.
if (!await this._fetchFromS3(docId, {sourceDocId, snapshotId})) {
throw new Error('Cannot fetch document');
}
// Make sure the new content is considered new.
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(this._getHashFile(this.getPath(docId)));
this.markAsChanged(docId);
this.markAsEdited(docId);
} catch (err) {
log.error("HostedStorageManager: problem replacing %s: %s", docId, err);
await fse.move(tmpPath, docPath, {overwrite: true});
throw err;
} finally {
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(tmpPath);
}
// Flush the document immediately if it has been changed.
await this.flushDoc(docId);
}
// We don't deal with listing documents.
public async listDocs(): Promise<DocEntry[]> { return []; }
public async deleteDoc(docName: string, deletePermanently?: boolean): Promise<void> {
if (!deletePermanently) {
throw new Error("HostedStorageManager only implements permanent deletion in deleteDoc");
}
await this.closeDocument(docName);
if (!this._disableS3) {
await this._ext.remove(docName);
}
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(this.getPath(docName));
await fse.remove(this._getHashFile(this.getPath(docName)));
}
// We don't implement document renames.
public async renameDoc(oldName: string, newName: string): Promise<void> {
throw new Error("HostedStorageManager does not implement renameDoc");
}
/**
* We handle backups by syncing the current version of the file as a new object version in S3,
* with the requested backupTag as an S3 tag.
*/
public async makeBackup(docName: string, backupTag: string): Promise<string> {
// TODO Must implement backups: currently this will prevent open docs that need migration.
// TODO: This method isn't used by SQLiteDB when migrating DB versions, but probably should be.
return "I_totally_did_not_back_up_your_document_sorry_not_sorry";
}
/**
* Electron version only. Shows the given doc in the file explorer.
*/
public async showItemInFolder(docName: string): Promise<void> {
throw new Error("HostedStorageManager does not implement showItemInFolder");
}
/**
* Close the storage manager. Make sure any pending changes reach S3 first.
*/
public async closeStorage(): Promise<void> {
await this._uploads.wait(() => log.info('HostedStorageManager: waiting for closeStorage to finish'));
// Close metadata manager.
if (this._metadataManager) { await this._metadataManager.close(); }
// Finish up everything incoming. This is most relevant for tests.
// Wait for any downloads to wind up, since there's no easy way to cancel them.
while (this._prepareFiles.size > 0) { await delay(100); }
await Promise.all(this._localFiles.values());
this._closed = true;
if (this._ext) { await this._ext.close(); }
if (this._pruner) { await this._pruner.close(); }
}
/**
* Allow storage manager to be used again - used in tests.
*/
public testReopenStorage() {
this._closed = false;
}
public async testWaitForPrunes() {
if (this._pruner) { await this._pruner.wait(); }
}
/**
* Get direct access to the external store - used in tests.
*/
public testGetExternalStorage(): ExternalStorage {
return this._baseStore;
}
// return true if document is backed up to s3.
public isSaved(docName: string): boolean {
return !this._uploads.hasPendingOperation(docName);
}
// pick up the pace of pushing to s3, from leisurely to urgent.
public prepareToCloseStorage() {
if (this._pruner) {
this._pruner.close().catch(e => log.error("HostedStorageManager: pruning error %s", e));
}
this._uploads.expediteOperations();
}
/**
* Finalize any operations involving the named document.
*/
public async closeDocument(docName: string): Promise<void> {
if (this._localFiles.has(docName)) {
await this._localFiles.get(docName);
}
this._localFiles.delete(docName);
return this.flushDoc(docName);
}
/**
* Make sure document is backed up to s3.
*/
public async flushDoc(docName: string): Promise<void> {
while (!this.isSaved(docName)) {
log.info('HostedStorageManager: waiting for document to finish: %s', docName);
await this._uploads.expediteOperationAndWait(docName);
}
}
/**
* This is called when a document may have been changed, via edits or migrations etc.
*/
public markAsChanged(docName: string): void {
if (parseUrlId(docName).snapshotId) { return; }
if (this._localFiles.has(docName)) {
// Make sure the file is marked as locally present (it may be newly created).
this._localFiles.set(docName, Promise.resolve(true));
}
if (this._disableS3) { return; }
if (this._closed) { throw new Error("HostedStorageManager.markAsChanged called after closing"); }
this._uploads.addOperation(docName);
}
/**
* This is called when a document was edited by the user.
*/
public markAsEdited(docName: string): void {
if (parseUrlId(docName).snapshotId) { return; }
// Schedule a metadata update for the modified doc.
if (this._metadataManager) { this._metadataManager.scheduleUpdate(docName); }
}
/**
* Check if there is a pending change to be pushed to S3.
*/
public needsUpdate(): boolean {
return this._uploads.hasPendingOperations();
}
public async getSnapshots(docName: string): Promise<DocSnapshots> {
if (this._disableS3) {
return {
snapshots: [{
snapshotId: 'current',
lastModified: new Date(),
docId: docName,
}]
};
}
const versions = await this._ext.versions(docName);
const parts = parseUrlId(docName);
return {
snapshots: versions
.map(v => ({
lastModified: v.lastModified,
snapshotId: v.snapshotId,
docId: buildUrlId({...parts, snapshotId: v.snapshotId}),
}))
};
}
/**
* Makes sure a document is present locally, fetching it from S3 if necessary.
* Returns true on success, false if document not found. It is safe to call
* this method multiple times in parallel.
*/
private async _ensureDocumentIsPresent(docName: string,
docSession: OptDocSession): Promise<boolean> {
// AsyncCreate.mapGetOrSet ensures we don't start multiple promises to talk to S3/Redis
// and that we clean up the failed key in case of failure.
return mapGetOrSet(this._localFiles, docName, async () => {
if (this._closed) { throw new Error("HostedStorageManager._ensureDocumentIsPresent called after closing"); }
checkValidDocId(docName);
const {trunkId, forkId, forkUserId, snapshotId} = parseUrlId(docName);
// If forkUserId is set to a valid user id, we can only create a fork if we know the
// requesting user and their id matches the forkUserId.
const userId = (docSession.client && docSession.client.getCachedUserId()) ||
(docSession.req && getUserId(docSession.req));
const canCreateFork = forkUserId ? (forkUserId === userId) : true;
const docStatus = await this._docWorkerMap.getDocWorkerOrAssign(docName, this._docWorkerId);
if (!docStatus.isActive) { throw new Error(`Doc is not active on a DocWorker: ${docName}`); }
if (docStatus.docWorker.id !== this._docWorkerId) {
throw new Error(`Doc belongs to a different DocWorker (${docStatus.docWorker.id}): ${docName}`);
}
if (this._disableS3) {
// skip S3, just use file system
let present: boolean = await fse.pathExists(this.getPath(docName));
if (forkId && !present) {
if (!canCreateFork) { throw new Error(`Cannot create fork`); }
if (snapshotId && snapshotId !== 'current') {
throw new Error(`cannot find snapshot ${snapshotId} of ${docName}`);
}
if (await fse.pathExists(this.getPath(trunkId))) {
await fse.copy(this.getPath(trunkId), this.getPath(docName));
present = true;
}
}
return present;
}
const existsLocally = await fse.pathExists(this.getPath(docName));
if (existsLocally) {
if (!docStatus.docMD5 || docStatus.docMD5 === DELETED_TOKEN) {
// New doc appears to already exist, but not in S3 (according to redis).
// Go ahead and use local version.
return true;
} else {
// Doc exists locally and in S3 (according to redis).
// Make sure the checksum matches.
const checksum = await this._getHash(await this._prepareBackup(docName));
if (checksum === docStatus.docMD5) {
// Fine, accept the doc as existing on our file system.
return true;
} else {
log.info("Local hash does not match redis: %s vs %s", checksum, docStatus.docMD5);
// The file that exists locally does not match S3. But S3 is the canonical version.
// On the assumption that the local file is outdated, delete it.
// TODO: may want to be more careful in case the local file has modifications that
// simply never made it to S3 due to some kind of breakage.
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(this.getPath(docName));
}
}
}
return this._fetchFromS3(docName, {
trunkId: forkId ? trunkId : undefined, snapshotId, canCreateFork
});
});
}
/**
* Fetch a document from s3 and save it locally as destId.grist
*
* If the document is not present in s3:
* + If it has a trunk:
* - If we do not not have permission to create a fork, we throw an error
* - Else we fetch the document from the trunk instead
* + Otherwise return false
*
* Forks of fork will not spark joy at this time. An attempt to
* fork a fork will result in a new fork of the original trunk.
*/
private async _fetchFromS3(destId: string, options: {sourceDocId?: string,
trunkId?: string,
snapshotId?: string,
canCreateFork?: boolean}): Promise<boolean> {
const destIdWithoutSnapshot = buildUrlId({...parseUrlId(destId), snapshotId: undefined});
let sourceDocId = options.sourceDocId || destIdWithoutSnapshot;
if (!await this._ext.exists(destIdWithoutSnapshot)) {
if (!options.trunkId) { return false; } // Document not found in S3
// No such fork in s3 yet, try from trunk (if we are allowed to create the fork).
if (!options.canCreateFork) { throw new Error('Cannot create fork'); }
// The special NEW_DOCUMENT_CODE trunk means we should create an empty document.
if (options.trunkId === NEW_DOCUMENT_CODE) { return false; }
if (!await this._ext.exists(options.trunkId)) { throw new Error('Cannot find original'); }
sourceDocId = options.trunkId;
}
await this._ext.downloadTo(sourceDocId, destId, this.getPath(destId), options.snapshotId);
return true;
}
/**
* Get a checksum for the given file (absolute path).
*/
private _getHash(srcPath: string): Promise<string> {
return checksumFile(srcPath, 'md5');
}
/**
* We'll save hashes in a file with the suffix -hash.
*/
private _getHashFile(docPath: string): string {
return docPath + "-hash";
}
/**
* Makes a copy of a document to a file with the suffix -backup. The copy is
* made using Sqlite's backup API. The backup is made incrementally so the db
* is never locked for long by the backup. The backup process will survive
* transient locks on the db.
*/
private async _prepareBackup(docId: string, postfix: string = 'backup'): Promise<string> {
const docPath = this.getPath(docId);
const tmpPath = `${docPath}-${postfix}`;
return backupSqliteDatabase(docPath, tmpPath, undefined, postfix);
}
/**
* Send a document to S3.
*/
private async _pushToS3(docId: string): Promise<void> {
let tmpPath: string|null = null;
try {
if (this._prepareFiles.has(docId)) {
throw new Error('too soon to consider pushing');
}
tmpPath = await this._prepareBackup(docId);
await this._ext.upload(docId, tmpPath);
this._pruner.requestPrune(docId);
} finally {
// Clean up backup.
// NOTE: fse.remove succeeds also when the file does not exist.
if (tmpPath) { await fse.remove(tmpPath); }
}
}
}
/**
* Make a copy of a sqlite database safely and without locking it for long periods, using the
* sqlite backup api.
* @param src: database to copy
* @param dest: file to which we copy the database
* @param testProgress: a callback used for test purposes to monitor detailed timing of backup.
* @param label: a tag to add to log messages
* @return dest
*/
export async function backupSqliteDatabase(src: string, dest: string,
testProgress?: (e: BackupEvent) => void,
label?: string): Promise<string> {
log.debug(`backupSqliteDatabase: starting copy of ${src} (${label})`);
let db: sqlite3.DatabaseWithBackup|null = null;
let success: boolean = false;
try {
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(dest); // Just in case some previous process terminated very badly.
// Sqlite will try to open any existing material at this
// path prior to overwriting it.
await fromCallback(cb => { db = new sqlite3.Database(dest, cb) as sqlite3.DatabaseWithBackup; });
// Turn off protections that can slow backup steps. If the app or OS
// crashes, the backup may be corrupt. In Grist use case, if app or OS
// crashes, no use will be made of backup, so we're OK.
// This sets flags matching the --async option to .backup in the sqlite3
// shell program: https://www.sqlite.org/src/info/7b6a605b1883dfcb
await fromCallback(cb => db!.exec("PRAGMA synchronous=OFF; PRAGMA journal_mode=OFF;", cb));
if (testProgress) { testProgress({action: 'open', phase: 'before'}); }
const backup: sqlite3.Backup = db!.backup(src, 'main', 'main', false);
if (testProgress) { testProgress({action: 'open', phase: 'after'}); }
let remaining: number = -1;
let prevError: Error|null = null;
let errorMsgTime: number = 0;
let restartMsgTime: number = 0;
for (;;) {
// For diagnostic purposes, issue a message if the backup appears to have been
// restarted by sqlite. The symptom of a restart we use is that the number of
// pages remaining in the backup increases rather than decreases. That number
// is reported by backup.remaining (after an initial period of where sqlite
// doesn't yet know how many pages there are and reports -1).
// So as not to spam the log if the user is making a burst of changes, we report
// this message at most once a second.
// See https://www.sqlite.org/c3ref/backup_finish.html and
// https://github.com/mapbox/node-sqlite3/pull/1116 for api details.
if (remaining >= 0 && backup.remaining > remaining && Date.now() - restartMsgTime > 1000) {
log.info(`backupSqliteDatabase: copy of ${src} (${label}) restarted`);
restartMsgTime = Date.now();
}
remaining = backup.remaining;
if (testProgress) { testProgress({action: 'step', phase: 'before'}); }
let isCompleted: boolean = false;
try {
isCompleted = Boolean(await fromCallback(cb => backup.step(PAGES_TO_BACKUP_PER_STEP, cb)));
} catch (err) {
if (String(err) !== String(prevError) || Date.now() - errorMsgTime > 1000) {
log.info(`backupSqliteDatabase (${src} ${label}): ${err}`);
errorMsgTime = Date.now();
}
prevError = err;
if (backup.failed) { throw new Error(`backupSqliteDatabase (${src} ${label}): internal copy failed`); }
}
if (testProgress) { testProgress({action: 'step', phase: 'after'}); }
if (isCompleted) {
log.info(`backupSqliteDatabase: copy of ${src} (${label}) completed successfully`);
success = true;
break;
}
await delay(PAUSE_BETWEEN_BACKUP_STEPS_IN_MS);
}
} finally {
if (testProgress) { testProgress({action: 'close', phase: 'before'}); }
try {
if (db) { await fromCallback(cb => db!.close(cb)); }
} catch (err) {
log.debug(`backupSqliteDatabase: problem stopping copy of ${src} (${label}): ${err}`);
}
if (!success) {
// Something went wrong, remove backup if it was started.
try {
// NOTE: fse.remove succeeds also when the file does not exist.
await fse.remove(dest);
} catch (err) {
log.debug(`backupSqliteDatabase: problem removing copy of ${src} (${label}): ${err}`);
}
}
if (testProgress) { testProgress({action: 'close', phase: 'after'}); }
log.debug(`backupSqliteDatabase: stopped copy of ${src} (${label})`);
}
return dest;
}
/**
* A summary of an event during a backup. Emitted for test purposes, to check timing.
*/
export interface BackupEvent {
action: 'step' | 'close' | 'open';
phase: 'before' | 'after';
}

7
app/server/lib/IBilling.ts Normal file
View File

@@ -0,0 +1,7 @@
import * as express from 'express';
export interface IBilling {
addEndpoints(app: express.Express): void;
addEventHandlers(): void;
addWebhooks(app: express.Express): void;
}

30
app/server/lib/ICreate.ts Normal file
View File

@@ -0,0 +1,30 @@
import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
import { ActiveDoc } from 'app/server/lib/ActiveDoc';
import { ScopedSession } from 'app/server/lib/BrowserSession';
import * as Comm from 'app/server/lib/Comm';
import { DocManager } from 'app/server/lib/DocManager';
import { ExternalStorage } from 'app/server/lib/ExternalStorage';
import { GristServer } from 'app/server/lib/GristServer';
import { IBilling } from 'app/server/lib/IBilling';
import { IDocStorageManager } from 'app/server/lib/IDocStorageManager';
import { IInstanceManager } from 'app/server/lib/IInstanceManager';
import { ILoginSession } from 'app/server/lib/ILoginSession';
import { INotifier } from 'app/server/lib/INotifier';
import { ISandbox, ISandboxCreationOptions } from 'app/server/lib/ISandbox';
import { IShell } from 'app/server/lib/IShell';
import { PluginManager } from 'app/server/lib/PluginManager';
export interface ICreate {
LoginSession(comm: Comm, sid: string, domain: string, scopeSession: ScopedSession,
instanceManager: IInstanceManager|null): ILoginSession;
Billing(dbManager: HomeDBManager): IBilling;
Notifier(dbManager: HomeDBManager, homeUrl: string): INotifier;
Shell(): IShell|undefined;
ExternalStorage(bucket: string, prefix: string): ExternalStorage|undefined;
ActiveDoc(docManager: DocManager, docName: string): ActiveDoc;
DocManager(storageManager: IDocStorageManager, pluginManager: PluginManager,
homeDbManager: HomeDBManager|null, gristServer: GristServer): DocManager;
NSandbox(options: ISandboxCreationOptions): ISandbox;
sessionSecret(): string;
}

35
app/server/lib/IDocStorageManager.ts Normal file
View File

@@ -0,0 +1,35 @@
import {DocEntry} from 'app/common/DocListAPI';
import {DocReplacementOptions} from 'app/common/UserAPI';
import {OptDocSession} from 'app/server/lib/DocSession';
import {DocSnapshots} from 'app/server/lib/DocSnapshots';
export interface IDocStorageManager {
getPath(docName: string): string;
getSampleDocPath(sampleDocName: string): string|null;
getCanonicalDocName(altDocName: string): Promise<string>;
// This method must not be called for the same docName twice in parallel.
// In the current implementation, it is called in the context of an
// AsyncCreate[docName].
prepareLocalDoc(docName: string, docSession: OptDocSession): Promise<boolean>;
listDocs(): Promise<DocEntry[]>;
deleteDoc(docName: string, deletePermanently?: boolean): Promise<void>;
renameDoc(oldName: string, newName: string): Promise<void>;
makeBackup(docName: string, backupTag: string): Promise<string>;
showItemInFolder(docName: string): Promise<void>;
closeStorage(): Promise<void>;
closeDocument(docName: string): Promise<void>;
markAsChanged(docName: string): void; // document needs a backup (edits, migrations, etc)
markAsEdited(docName: string): void; // document was edited by a user
testReopenStorage(): void; // restart storage during tests
addToStorage(docName: string): void; // add a new local document to storage
prepareToCloseStorage(): void; // speed up sync with remote store
getCopy(docName: string): Promise<string>; // get an immutable copy of a document
flushDoc(docName: string): Promise<void>; // flush a document to persistent storage
getSnapshots(docName: string): Promise<DocSnapshots>;
replace(docName: string, options: DocReplacementOptions): Promise<void>;
}

5
app/server/lib/IInstanceManager.ts Normal file
View File

@@ -0,0 +1,5 @@
import { ILoginSession } from 'app/server/lib/ILoginSession';
export interface IInstanceManager {
getLoginSession(instanceId: string): ILoginSession;
}

15
app/server/lib/ILoginSession.ts Normal file
View File

@@ -0,0 +1,15 @@
import {UserProfile} from 'app/common/LoginSessionAPI';
import {Client} from 'app/server/lib/Client';
export interface ILoginSession {
clients: Set<Client>;
getEmail(): Promise<string>;
// Log out
clearSession(): Promise<void>;
// For testing only. If no email address, profile is wiped, otherwise it is set.
testSetProfile(profile: UserProfile|null): Promise<void>;
updateTokenForTesting(idToken: string): Promise<void>;
getCurrentTokenForTesting(): Promise<string|null>;
useTestToken(idToken: string): Promise<void>;
}

4
app/server/lib/INotifier.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface INotifier {
// for test purposes, check if any notifications are in progress
readonly testPending: boolean;
}

28
app/server/lib/ISandbox.ts Normal file
View File

@@ -0,0 +1,28 @@
import * as log from 'app/server/lib/log';
/**
* Starting to whittle down the options used when creating a sandbox, to leave more
* freedom in how the sandbox works.
*/
export interface ISandboxCreationOptions {
comment?: string; // an argument to add in command line when possible, so it shows in `ps`
logCalls?: boolean;
logMeta?: log.ILogMeta;
logTimes?: boolean;
// This batch of options is used by SafePythonComponent, so are important for importers.
entryPoint?: string; // main script to call - leave undefined for default
sandboxMount?: string; // if defined, make this path available read-only as "/sandbox"
importMount?: string; // if defined, make this path available read-only as "/importdir"
}
export interface ISandbox {
shutdown(): Promise<unknown>; // TODO: tighten up this type.
pyCall(funcName: string, ...varArgs: unknown[]): Promise<any>;
reportMemoryUsage(): Promise<void>;
}
export interface ISandboxCreator {
create(options: ISandboxCreationOptions): ISandbox;
}

4
app/server/lib/IShell.ts Normal file
View File

@@ -0,0 +1,4 @@
export interface IShell {
moveItemToTrash(docPath: string): void;
showItemInFolder(docPath: string): void;
}

34
app/server/lib/ITestingHooks-ti.ts Normal file
View File

@@ -0,0 +1,34 @@
/**
* This module was automatically generated by `ts-interface-builder`
*/
import * as t from "ts-interface-checker";
// tslint:disable:object-literal-key-quotes
export const ITestingHooks = t.iface([], {
"getOwnPort": t.func("number"),
"getPort": t.func("number"),
"updateAuthToken": t.func("void", t.param("instId", "string"), t.param("authToken", "string")),
"getAuthToken": t.func(t.union("string", "null"), t.param("instId", "string")),
"useTestToken": t.func("void", t.param("instId", "string"), t.param("token", "string")),
"setLoginSessionProfile": t.func("void", t.param("gristSidCookie", "string"),
t.param("profile", t.union("UserProfile", "null")), t.param("org", "string", true)),
"setServerVersion": t.func("void", t.param("version", t.union("string", "null"))),
"disconnectClients": t.func("void"),
"commShutdown": t.func("void"),
"commRestart": t.func("void"),
"commSetClientPersistence": t.func("void", t.param("ttlMs", "number")),
"closeDocs": t.func("void"),
"setDocWorkerActivation": t.func("void", t.param("workerId", "string"),
t.param("active", t.union(t.lit('active'),
t.lit('inactive'),
t.lit('crash')))),
"flushAuthorizerCache": t.func("void"),
"getDocClientCounts": t.func(t.array(t.tuple("string", "number"))),
"setActiveDocTimeout": t.func("number", t.param("seconds", "number")),
});
const exportedTypeSuite: t.ITypeSuite = {
ITestingHooks,
UserProfile: t.name("object"),
};
export default exportedTypeSuite;

20
app/server/lib/ITestingHooks.ts Normal file
View File

@@ -0,0 +1,20 @@
import {UserProfile} from 'app/common/LoginSessionAPI';
export interface ITestingHooks {
getOwnPort(): number;
getPort(): number;
updateAuthToken(instId: string, authToken: string): Promise<void>;
getAuthToken(instId: string): Promise<string|null>;
useTestToken(instId: string, token: string): Promise<void>;
setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise<void>;
setServerVersion(version: string|null): Promise<void>;
disconnectClients(): Promise<void>;
commShutdown(): Promise<void>;
commRestart(): Promise<void>;
commSetClientPersistence(ttlMs: number): Promise<void>;
closeDocs(): Promise<void>;
setDocWorkerActivation(workerId: string, active: 'active'|'inactive'|'crash'): Promise<void>;
flushAuthorizerCache(): Promise<void>;
getDocClientCounts(): Promise<Array<[string, number]>>;
setActiveDocTimeout(seconds: number): Promise<number>;
}

354
app/server/lib/NSandbox.ts Normal file
View File

@@ -0,0 +1,354 @@
/**
* JS controller for the pypy sandbox.
*/
import * as pidusage from '@gristlabs/pidusage';
import * as marshal from 'app/common/marshal';
import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox';
import * as log from 'app/server/lib/log';
import * as sandboxUtil from 'app/server/lib/sandboxUtil';
import * as shutdown from 'app/server/lib/shutdown';
import {Throttle} from 'app/server/lib/Throttle';
import {ChildProcess, spawn, SpawnOptions} from 'child_process';
import * as path from 'path';
import {Stream, Writable} from 'stream';
type SandboxMethod = (...args: any[]) => any;
export interface ISandboxCommand {
process: string;
}
export interface ISandboxOptions {
args: string[]; // The arguments to pass to the python process.
exports?: {[name: string]: SandboxMethod}; // Functions made available to the sandboxed process.
logCalls?: boolean; // (Not implemented) Whether to log all system calls from the python sandbox.
logTimes?: boolean; // Whether to log time taken by calls to python sandbox.
unsilenceLog?: boolean; // Don't silence the sel_ldr logging.
selLdrArgs?: string[]; // Arguments passed to selLdr, for instance the following sets an
// environment variable `{ ... selLdrArgs: ['-E', 'PYTHONPATH=grist'] ... }`.
logMeta?: log.ILogMeta; // Log metadata (e.g. including docId) to report in all log messages.
command?: ISandboxCommand;
}
// Options for low-level spawning of selLdr sandbox process.
export interface ISpawnOptions extends SpawnOptions {
unsilenceLog?: boolean; // Don't silence the sel_ldr logging.
command?: ISandboxCommand;
}
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
// Type for basic message identifiers, available as constants in sandboxUtil.
type MsgCode = null | true | false;
export class NSandbox implements ISandbox {
/**
* Helper function to run the nacl sandbox. It takes care of most arguments, similarly to
* nacl/bin/run script, but without the reliance on bash. We can't use bash when -r/-w options
* because on Windows it doesn't pass along the open file descriptors. Bash is also unavailable
* when installing a standalone version on Windows.
* @param selLdrArgs: Arguments to pass to sel_ldr;
* @param pythonArgs: Arguments to pass to python within the sandbox.
* @param spawnOptions: extra options for child_process.spawn(), such as 'stdio'.
*/
public static spawn(selLdrArgs: string[], pythonArgs: string[], spawnOptions: ISpawnOptions = {}): ChildProcess {
const unsilenceLog = spawnOptions.unsilenceLog;
delete spawnOptions.unsilenceLog;
const command = spawnOptions.command;
delete spawnOptions.command;
if (command) {
return spawn(command.process, pythonArgs,
{env: {PYTHONPATH: 'grist:thirdparty'},
cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
}
const noLog = unsilenceLog ? [] :
(process.env.OS === 'Windows_NT' ? ['-l', 'NUL'] : ['-l', '/dev/null']);
return spawn('sandbox/nacl/bin/sel_ldr', [
'-B', './sandbox/nacl/lib/irt_core.nexe', '-m', './sandbox/nacl/root:/:ro',
...noLog,
...selLdrArgs,
'./sandbox/nacl/lib/runnable-ld.so',
'--library-path', '/slib', '/python/bin/python2.7.nexe',
...pythonArgs
],
{env: {}, ...spawnOptions},
);
}
public readonly childProc: ChildProcess;
private _logTimes: boolean;
private _exportedFunctions: {[name: string]: SandboxMethod};
private _marshaller = new marshal.Marshaller({stringToBuffer: true, version: 2});
private _unmarshaller = new marshal.Unmarshaller({ bufferToString: false });
// Members used for reading from the sandbox process.
private _pendingReads: ResolveRejectPair[] = [];
private _isReadClosed = false;
private _isWriteClosed = false;
private _logMeta: log.ILogMeta;
private _streamToSandbox: Writable;
private _streamFromSandbox: Stream;
private _throttle: Throttle | undefined;
/*
* Callers may listen to events from sandbox.childProc (a ChildProcess), e.g. 'close' and 'error'.
* The sandbox listens for 'aboutToExit' event on the process, to properly shut down.
*/
constructor(options: ISandboxOptions) {
this._logTimes = Boolean(options.logTimes || options.logCalls);
this._exportedFunctions = options.exports || {};
const selLdrArgs = options.selLdrArgs || [];
// We use these options to set up communication with the sandbox:
// -r 3:3 to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the
// inside, for reading from the inside. This becomes `this._streamToSandbox`.
// -w 4:4 to associate FD 4 on the outside with FD 4 on the inside for writing from the inside.
// This becomes `this._streamFromSandbox`
this.childProc = NSandbox.spawn(['-r', '3:3', '-w', '4:4', ...selLdrArgs], options.args, {
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
unsilenceLog: options.unsilenceLog,
command: options.command
});
this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
log.rawDebug("Sandbox started", this._logMeta);
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
this.childProc.on('close', this._onExit.bind(this));
this.childProc.on('error', this._onError.bind(this));
this.childProc.stdout.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
this.childProc.stderr.on('data', sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta));
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
this._streamFromSandbox.on('end', () => this._onSandboxClose());
this._streamFromSandbox.on('error', (err) => {
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
this._onSandboxClose();
});
this._streamToSandbox.on('error', (err) => {
if (!this._isWriteClosed) {
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
}
});
// On shutdown, shutdown the child process cleanly, and wait for it to exit.
shutdown.addCleanupHandler(this, this.shutdown);
if (process.env.GRIST_THROTTLE_CPU) {
this._throttle = new Throttle({
pid: this.childProc.pid,
logMeta: this._logMeta,
});
}
}
/**
* Shuts down the sandbox process cleanly, and wait for it to exit.
* @return {Promise} Promise that's resolved with [code, signal] when the sandbox exits.
*/
public async shutdown() {
log.rawDebug("Sandbox shutdown starting", this._logMeta);
shutdown.removeCleanupHandlers(this);
// The signal ensures the sandbox process exits even if it's hanging in an infinite loop or
// long computation. It doesn't get a chance to clean up, but since it is sandboxed, there is
// nothing it needs to clean up anyway.
const timeoutID = setTimeout(() => {
log.rawWarn("Sandbox sending SIGKILL", this._logMeta);
this.childProc.kill('SIGKILL');
}, 1000);
const result = await new Promise((resolve, reject) => {
if (this._isWriteClosed) { resolve(); }
this.childProc.on('error', reject);
this.childProc.on('close', resolve);
this._close();
});
// In the normal case, the kill timer is pending when the process exits, and we can clear it. If
// the process got killed, the timer is invalid, and clearTimeout() does nothing.
clearTimeout(timeoutID);
return result;
}
/**
* Makes a call to the python process implementing our calling convention on stdin/stdout.
* @param funcName The name of the python RPC function to call.
* @param args Arguments to pass to the given function.
* @returns A promise for the return value from the Python function.
*/
public pyCall(funcName: string, ...varArgs: unknown[]): Promise<any> {
const startTime = Date.now();
this._sendData(sandboxUtil.CALL, Array.from(arguments));
return this._pyCallWait(funcName, startTime);
}
/**
* Returns the RSS (resident set size) of the sandbox process, in bytes.
*/
public async reportMemoryUsage() {
const memory = (await pidusage(this.childProc.pid)).memory;
log.rawDebug('Sandbox memory', {memory, ...this._logMeta});
}
private async _pyCallWait(funcName: string, startTime: number): Promise<any> {
try {
return await new Promise((resolve, reject) => {
this._pendingReads.push([resolve, reject]);
});
} finally {
if (this._logTimes) {
log.rawDebug(`Sandbox pyCall[${funcName}] took ${Date.now() - startTime} ms`, this._logMeta);
}
}
}
private _close() {
if (this._throttle) { this._throttle.stop(); }
if (!this._isWriteClosed) {
// Close the pipe to the sandbox, which should cause the sandbox to exit cleanly.
this._streamToSandbox.end();
this._isWriteClosed = true;
}
}
private _onExit(code: number, signal: string) {
this._close();
log.rawDebug(`Sandbox exited with code ${code} signal ${signal}`, this._logMeta);
}
private _onError(err: Error) {
this._close();
log.rawWarn(`Sandbox could not be spawned: ${err}`, this._logMeta);
}
/**
* Send a message to the sandbox process with the given message code and data.
*/
private _sendData(msgCode: MsgCode, data: any) {
if (this._isReadClosed) {
throw new sandboxUtil.SandboxError("PipeToSandbox is closed");
}
this._marshaller.marshal(msgCode);
this._marshaller.marshal(data);
return this._streamToSandbox.write(this._marshaller.dumpAsBuffer());
}
/**
* Process a buffer of data received from the sandbox process.
*/
private _onSandboxData(data: any) {
this._unmarshaller.parse(data, buf => {
const value = marshal.loads(buf, { bufferToString: true });
this._onSandboxMsg(value[0], value[1]);
});
}
/**
* Process the closing of the pipe by the sandboxed process.
*/
private _onSandboxClose() {
if (this._throttle) { this._throttle.stop(); }
this._isReadClosed = true;
// Clear out all reads pending on PipeFromSandbox, rejecting them with the given error.
const err = new sandboxUtil.SandboxError("PipeFromSandbox is closed");
this._pendingReads.forEach(resolvePair => resolvePair[1](err));
this._pendingReads = [];
}
/**
* Process a parsed message from the sandboxed process.
*/
private _onSandboxMsg(msgCode: MsgCode, data: any) {
if (msgCode === sandboxUtil.CALL) {
// Handle calls FROM the sandbox.
if (!Array.isArray(data) || data.length === 0) {
log.rawWarn("Sandbox invalid call from the sandbox", this._logMeta);
} else {
const fname = data[0];
const args = data.slice(1);
log.rawDebug(`Sandbox got call to ${fname} (${args.length} args)`, this._logMeta);
Promise.resolve()
.then(() => {
const func = this._exportedFunctions[fname];
if (!func) { throw new Error("No such exported function: " + fname); }
return func(...args);
})
.then((ret) => {
this._sendData(sandboxUtil.DATA, ret);
}, (err) => {
this._sendData(sandboxUtil.EXC, err.toString());
})
.catch((err) => {
log.rawDebug(`Sandbox sending response failed: ${err}`, this._logMeta);
});
}
} else {
// Handle return values for calls made to the sandbox.
const resolvePair = this._pendingReads.shift();
if (resolvePair) {
if (msgCode === sandboxUtil.EXC) {
resolvePair[1](new sandboxUtil.SandboxError(data));
} else if (msgCode === sandboxUtil.DATA) {
resolvePair[0](data);
} else {
log.rawWarn("Sandbox invalid message from sandbox", this._logMeta);
}
}
}
}
}
export class NSandboxCreator implements ISandboxCreator {
public constructor(private _flavor: 'pynbox' | 'unsandboxed') {
}
public create(options: ISandboxCreationOptions): ISandbox {
const defaultEntryPoint = this._flavor === 'pynbox' ? 'grist/main.pyc' : 'grist/main.py';
const args = [options.entryPoint || defaultEntryPoint];
if (!options.entryPoint && options.comment) {
// When using default entry point, we can add on a comment as an argument - it isn't
// used, but will show up in `ps` output for the sandbox process. Comment is intended
// to be a document name/id.
args.push(options.comment);
}
const selLdrArgs: string[] = [];
if (options.sandboxMount) {
selLdrArgs.push(
// TODO: Only modules that we share with plugins should be mounted. They could be gathered in
// a "$APPROOT/sandbox/plugin" folder, only which get mounted.
'-E', 'PYTHONPATH=grist:thirdparty',
'-m', `${options.sandboxMount}:/sandbox:ro`);
}
if (options.importMount) {
selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`);
}
return new NSandbox({
args,
logCalls: options.logCalls,
logMeta: options.logMeta,
logTimes: options.logTimes,
selLdrArgs,
...(this._flavor === 'pynbox' ? {} : {
command: {
process: "python2.7"
}
})
});
}
}

206
app/server/lib/OnDemandActions.ts Normal file
View File

@@ -0,0 +1,206 @@
import {BulkColValues, ColValues, DocAction, isSchemaAction, TableDataAction, UserAction} from 'app/common/DocActions';
import {DocData} from 'app/common/DocData';
import {TableData} from 'app/common/TableData';
import {IndexColumns} from 'app/server/lib/DocStorage';
const ACTION_TYPES = new Set(['AddRecord', 'BulkAddRecord', 'UpdateRecord', 'BulkUpdateRecord',
'RemoveRecord', 'BulkRemoveRecord']);
export interface ProcessedAction {
stored: DocAction[];
undo: DocAction[];
retValues: any;
}
export interface OnDemandStorage {
getNextRowId(tableId: string): Promise<number>;
fetchActionData(tableId: string, rowIds: number[], colIds?: string[]): Promise<TableDataAction>;
}
/**
* Handle converting UserActions to DocActions for onDemand tables.
*/
export class OnDemandActions {
private _tablesMeta: TableData = this._docData.getTable('_grist_Tables')!;
private _columnsMeta: TableData = this._docData.getTable('_grist_Tables_column')!;
constructor(private _storage: OnDemandStorage, private _docData: DocData) {}
// TODO: Ideally a faster data structure like an index by tableId would be used to decide whether
// the table is onDemand.
public isOnDemand(tableId: string): boolean {
const tableRef = this._tablesMeta.findRow('tableId', tableId);
// OnDemand tables must have a record in the _grist_Tables metadata table.
return tableRef ? Boolean(this._tablesMeta.getValue(tableRef, 'onDemand')) : false;
}
/**
* Convert a UserAction into stored and undo DocActions as well as return values.
*/
public processUserAction(action: UserAction): Promise<ProcessedAction> {
const a = action.map(item => item as any);
switch (a[0]) {
case "ApplyUndoActions": return this._doApplyUndoActions(a[1]);
case "AddRecord": return this._doAddRecord (a[1], a[2], a[3]);
case "BulkAddRecord": return this._doBulkAddRecord (a[1], a[2], a[3]);
case "UpdateRecord": return this._doUpdateRecord (a[1], a[2], a[3]);
case "BulkUpdateRecord": return this._doBulkUpdateRecord(a[1], a[2], a[3]);
case "RemoveRecord": return this._doRemoveRecord (a[1], a[2]);
case "BulkRemoveRecord": return this._doBulkRemoveRecord(a[1], a[2]);
default: throw new Error(`Received unknown action ${action[0]}`);
}
}
/**
* Splits an array of UserActions into two separate arrays of normal and onDemand actions.
*/
public splitByOnDemand(actions: UserAction[]): [UserAction[], UserAction[]] {
const normal: UserAction[] = [];
const onDemand: UserAction[] = [];
actions.forEach(a => {
// Check that the actionType can be applied without the sandbox and also that the action
// is on a data table.
const isOnDemandAction = ACTION_TYPES.has(a[0] as string);
const isDataTableAction = typeof a[1] === 'string' && !(a[1] as string).startsWith('_grist_');
if (a[0] === 'ApplyUndoActions') {
// Split actions inside the undo action array.
const [undoNormal, undoOnDemand] = this.splitByOnDemand(a[1] as UserAction[]);
if (undoNormal.length > 0) {
normal.push(['ApplyUndoActions', undoNormal]);
}
if (undoOnDemand.length > 0) {
onDemand.push(['ApplyUndoActions', undoOnDemand]);
}
} else if (isDataTableAction && isOnDemandAction && this.isOnDemand(a[1] as string)) {
// Check whether the tableId belongs to an onDemand table.
onDemand.push(a);
} else {
normal.push(a);
}
});
return [normal, onDemand];
}
/**
* Compute the indexes we would like to have, given the current schema.
*/
public getDesiredIndexes(): IndexColumns[] {
const desiredIndexes: IndexColumns[] = [];
for (const c of this._columnsMeta.getRecords()) {
const t = this._tablesMeta.getRecord(c.parentId as number);
if (t && t.onDemand && c.type && (c.type as string).startsWith('Ref:')) {
desiredIndexes.push({tableId: t.tableId as string, colId: c.colId as string});
}
}
return desiredIndexes;
}
/**
* Check if an action represents a schema change on an onDemand table.
*/
public isSchemaAction(docAction: DocAction): boolean {
return isSchemaAction(docAction) && this.isOnDemand(docAction[1]);
}
private async _doApplyUndoActions(actions: DocAction[]) {
const undo: DocAction[] = [];
for (const a of actions) {
const converted = await this.processUserAction(a);
undo.concat(converted.undo);
}
return {
stored: actions,
undo,
retValues: null
};
}
private async _doAddRecord(
tableId: string,
rowId: number|null,
colValues: ColValues
): Promise<ProcessedAction> {
if (rowId === null) {
rowId = await this._storage.getNextRowId(tableId);
}
// Set the manualSort to be the same as the rowId. This forces new rows to always be added
// at the end of the table.
colValues.manualSort = rowId;
return {
stored: [['AddRecord', tableId, rowId, colValues]],
undo: [['RemoveRecord', tableId, rowId]],
retValues: rowId
};
}
private async _doBulkAddRecord(
tableId: string,
rowIds: Array<number|null>,
colValues: BulkColValues
): Promise<ProcessedAction> {
// When unset, we will set the rowId values to count up from the greatest
// values already in the table.
if (rowIds[0] === null) {
const nextRowId = await this._storage.getNextRowId(tableId);
for (let i = 0; i < rowIds.length; i++) {
rowIds[i] = nextRowId + i;
}
}
// Set the manualSort values to be the same as the rowIds. This forces new rows to always be
// added at the end of the table.
colValues.manualSort = rowIds;
return {
stored: [['BulkAddRecord', tableId, rowIds as number[], colValues]],
undo: [['BulkRemoveRecord', tableId, rowIds as number[]]],
retValues: rowIds
};
}
private async _doUpdateRecord(
tableId: string,
rowId: number,
colValues: ColValues
): Promise<ProcessedAction> {
const [, , oldRowIds, oldColValues] =
await this._storage.fetchActionData(tableId, [rowId], Object.keys(colValues));
return {
stored: [['UpdateRecord', tableId, rowId, colValues]],
undo: [['BulkUpdateRecord', tableId, oldRowIds, oldColValues]],
retValues: null
};
}
private async _doBulkUpdateRecord(
tableId: string,
rowIds: number[],
colValues: BulkColValues
): Promise<ProcessedAction> {
const [, , oldRowIds, oldColValues] =
await this._storage.fetchActionData(tableId, rowIds, Object.keys(colValues));
return {
stored: [['BulkUpdateRecord', tableId, rowIds, colValues]],
undo: [['BulkUpdateRecord', tableId, oldRowIds, oldColValues]],
retValues: null
};
}
private async _doRemoveRecord(tableId: string, rowId: number): Promise<ProcessedAction> {
const [, , oldRowIds, oldColValues] = await this._storage.fetchActionData(tableId, [rowId]);
return {
stored: [['RemoveRecord', tableId, rowId]],
undo: [['BulkAddRecord', tableId, oldRowIds, oldColValues]],
retValues: null
};
}
private async _doBulkRemoveRecord(tableId: string, rowIds: number[]): Promise<ProcessedAction> {
const [, , oldRowIds, oldColValues] = await this._storage.fetchActionData(tableId, rowIds);
return {
stored: [['BulkRemoveRecord', tableId, rowIds]],
undo: [['BulkAddRecord', tableId, oldRowIds, oldColValues]],
retValues: null
};
}
}

78
app/server/lib/PluginEndpoint.ts Normal file
View File

@@ -0,0 +1,78 @@
import {FlexServer} from 'app/server/lib/FlexServer';
import * as log from 'app/server/lib/log';
import {PluginManager} from 'app/server/lib/PluginManager';
import * as express from 'express';
import * as mimeTypes from 'mime-types';
import * as path from 'path';
// Get the url where plugin material should be served from.
export function getUntrustedContentOrigin(): string|undefined {
return process.env.APP_UNTRUSTED_URL;
}
// Get the host serving plugin material
export function getUntrustedContentHost(): string|undefined {
const origin = getUntrustedContentOrigin();
if (!origin) { return; }
return new URL(origin).host;
}
// Add plugin endpoints to be served on untrusted host
export function addPluginEndpoints(server: FlexServer, pluginManager: PluginManager) {
const host = getUntrustedContentHost();
if (host) {
server.app.get(/^\/plugins\/(installed|builtIn)\/([^/]+)\/(.+)/, (req, res) =>
servePluginContent(req, res, pluginManager, host));
}
}
// Serve content for plugins with various checks that it is being accessed as we expect.
function servePluginContent(req: express.Request, res: express.Response,
pluginManager: PluginManager, untrustedContentHost: string) {
const pluginKind = req.params[0];
const pluginId = req.params[1];
const pluginPath = req.params[2];
// We should not serve untrusted content (as from plugins) from the same domain as the main app
// (at least not html pages), as it's an open door to XSS attacks.
// - For hosted version, we serve it from a separate domain name.
// - For electron version, we give access to protected <webview> content based on a special header.
// - We also allow "application/javascript" content from the main domain for serving the
// WebWorker main script, since that's hard to distinguish in electron case, and should not
// enable XSS.
if (matchHost(req.get('host'), untrustedContentHost) ||
req.get('X-From-Plugin-WebView') === "true" ||
mimeTypes.lookup(path.extname(pluginPath)) === "application/javascript") {
const dirs = pluginManager.dirs();
const contentRoot = pluginKind === "installed" ? dirs.installed : dirs.builtIn;
// Note that pluginPath may not be safe, but `sendFile` with the "root" option restricts
// relative paths to be within the root folder (see the 3rd party library unit-test:
// https://github.com/pillarjs/send/blob/3daa901cf731b86187e4449fa2c52f971e0b3dbc/test/send.js#L1363)
return res.sendFile(`${pluginId}/${pluginPath}`, {root: contentRoot});
}
log.warn(`Refusing to serve untrusted plugin content on ${req.get('host')}`);
res.status(403).end('Plugin content is not accessible to this request');
}
// Middleware to restrict some assets to untrusted host.
export function limitToPlugins(handler: express.RequestHandler) {
const host = getUntrustedContentHost();
return function(req: express.Request, resp: express.Response, next: express.NextFunction) {
if (!host) { return next(); }
if (matchHost(req.get('host'), host) || req.get('X-From-Plugin-WebView') === "true") {
return handler(req, resp, next);
}
return next();
};
}
// Compare hosts, bearing in mind that if they happen to be on port 443 the
// port number may or may not be included. This assumes we are serving over https.
function matchHost(host1: string|undefined, host2: string) {
if (!host1) { return false; }
if (host1 === host2) { return true; }
if (host1.indexOf(':') === -1) { host1 += ":443"; }
if (host2.indexOf(':') === -1) { host2 += ":443"; }
return host1 === host2;
}

165
app/server/lib/PluginManager.ts Normal file
View File

@@ -0,0 +1,165 @@
import {DirectoryScanEntry, LocalPlugin} from 'app/common/plugin';
import * as log from 'app/server/lib/log';
import {readManifest} from 'app/server/lib/manifest';
import {getAppPathTo} from 'app/server/lib/places';
import * as fse from 'fs-extra';
import * as path from 'path';
/**
* Various plugins' related directories.
*/
export interface PluginDirectories {
/**
* Directory where built in plugins are located.
*/
readonly builtIn?: string;
/**
* Directory where user installed plugins are localted.
*/
readonly installed?: string;
}
/**
*
* The plugin manager class is responsible for providing both built in and installed plugins and
* spawning server side plugins's.
*
* Usage:
*
* const pluginManager = new PluginManager(appRoot, userRoot);
* await pluginManager.initialize();
*
*/
export class PluginManager {
public pluginsLoaded: Promise<void>;
// ========== Instance members and methods ==========
private _dirs: PluginDirectories;
private _validPlugins: LocalPlugin[] = [];
private _entries: DirectoryScanEntry[] = [];
/**
* @param {string} userRoot: path to user's grist directory; `null` is allowed, to only uses built in plugins.
*
*/
public constructor(public appRoot?: string, userRoot?: string) {
this._dirs = {
installed: userRoot ? path.join(userRoot, 'plugins') : undefined,
builtIn: appRoot ? getAppPathTo(appRoot, 'plugins') : undefined
};
}
public dirs(): PluginDirectories {return this._dirs; }
/**
* Create tmp dir and load plugins.
*/
public async initialize(): Promise<void> {
try {
await (this.pluginsLoaded = this.loadPlugins());
} catch (err) {
log.error("PluginManager's initialization failed: ", err);
throw err;
}
}
/**
* Re-load plugins (litterally re-run `loadPlugins`).
*/
// TODO: it's not clear right now what we do on reload. Do we deactivate plugins that were removed
// from the fs? Do we update plugins that have changed on the fs ?
public async reloadPlugins(): Promise<void> {
return await this.loadPlugins();
}
/**
* Discover both builtIn and user installed plugins. Logs any failures that happens when scanning
* a directory (ie: manifest missing or manifest validation errors etc...)
*/
public async loadPlugins(): Promise<void> {
this._entries = [];
// Load user installed plugins
if (this._dirs.installed) {
this._entries.push(...await scanDirectory(this._dirs.installed, "installed"));
}
// Load builtIn plugins
if (this._dirs.builtIn) {
this._entries.push(...await scanDirectory(this._dirs.builtIn, "builtIn"));
}
if (!process.env.GRIST_EXPERIMENTAL_PLUGINS ||
process.env.GRIST_EXPERIMENTAL_PLUGINS === '0') {
// Remove experimental plugins
this._entries = this._entries.filter(entry => {
if (entry.manifest && entry.manifest.experimental) {
log.warn("Ignoring experimental plugin %s", entry.id);
return false;
}
return true;
});
}
this._validPlugins = this._entries.filter(entry => !entry.errors).map(entry => entry as LocalPlugin);
this._logScanningReport();
}
public getPlugins(): LocalPlugin[] {
return this._validPlugins;
}
private _logScanningReport() {
const invalidPlugins = this._entries.filter( entry => entry.errors);
if (invalidPlugins.length) {
for (const plugin of invalidPlugins) {
log.warn(`Error loading plugins: Failed to load extension from ${plugin.path}\n` +
(plugin.errors!).map(m => " - " + m).join("\n ")
);
}
}
log.info(`Found ${this._validPlugins.length} valid plugins on the system`);
for (const p of this._validPlugins) {
log.debug("PLUGIN %s -- %s", p.id, p.path);
}
}
}
async function scanDirectory(dir: string, kind: "installed"|"builtIn"): Promise<DirectoryScanEntry[]> {
const plugins: DirectoryScanEntry[] = [];
let listDir;
try {
listDir = await fse.readdir(dir);
} catch (e) {
// non existing dir is treated as an empty dir
log.info(`No plugins directory: ${e.message}`);
return [];
}
for (const id of listDir) {
const folderPath = path.join(dir, id),
plugin: DirectoryScanEntry = {
path: folderPath,
id: `${kind}/${id}`
};
try {
plugin.manifest = await readManifest(folderPath);
} catch (e) {
plugin.errors = [];
if (e.message) {
plugin.errors.push(e.message);
}
if (e.notices) {
plugin.errors.push(...e.notices);
}
}
plugins.push(plugin);
}
return plugins;
}

534
app/server/lib/SQLiteDB.ts Normal file
View File

@@ -0,0 +1,534 @@
/**
* SQLiteDB provides a clean Promise-based interface to SQLite along with an organized way to
* specify the initial structure of the database and migrations when this structure changes.
*
* Here's a simple example,
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Note how the create() function and the first migration are identical here. But they'll diverge
* once we make a change to the schema. E.g. the next change could look like this:
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT, B NUMERIC)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* },
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("ALTER TABLE Foo ADD COLUMN B NUMERIC");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Now a new document will have two columns. A document created with the first version of the code
* will gain a second column when opened with the new code. If a migration happened during open,
* you may examine two properties of the returned db object:
*
* db.migrationBackupPath -- set to the path of the pre-migration backup file.
* db.migrationError -- set to the Error object if the migration failed.
*
* This module uses SQLite's "user_version" pragma to keep track of the version number of a
* migration. It does not require, support, or record backwards migrations, but it will warn of
* inconsistencies that may arise during development. In that case, remember you have a backup
* from each migration.
*
* If you are starting with an existing unversioned DB, the first migration should have code to
* bring such DBs to a common state.
*
* const schemaInfo: SQLiteDB.SchemaInfo = {
* async create(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE Foo (A TEXT)");
* await db.exec("CREATE TABLE Bar (B TEXT)");
* },
* migrations: [
* async function(db: SQLiteDB.SQLiteDB) {
* await db.exec("CREATE TABLE IF NOT EXISTS Foo (A TEXT)");
* await db.exec("CREATE TABLE IF NOT EXISTS Bar (B TEXT)");
* }
* ],
* }
* const db = await SQLiteDB.openDB("pathToDB", schemaInfo, SQLiteDB.OpenMode.OPEN_CREATE);
*
* Once using this module with versioning, future changes would be made by adding one item to the
* "migrations" array, and modifying create() to create correct new documents.
*/
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {timeFormat} from 'app/common/timeFormat';
import * as docUtils from 'app/server/lib/docUtils';
import * as log from 'app/server/lib/log';
import {fromCallback} from 'app/server/lib/serverUtils';
import * as sqlite3 from '@gristlabs/sqlite3';
import * as assert from 'assert';
import {each} from 'bluebird';
import * as fse from 'fs-extra';
import fromPairs = require('lodash/fromPairs');
import isEqual = require('lodash/isEqual');
import noop = require('lodash/noop');
import range = require('lodash/range');
// Describes the result of get() and all() database methods.
export interface ResultRow {
[column: string]: any;
}
// Describes how to create a new DB or migrate an old one. Any changes to the DB must be reflected
// in the 'create' function, and added as new entries in the 'migrations' array. Existing
// 'migration' entries may not be modified; they are used to migrate older DBs.
export interface SchemaInfo {
// Creates a structure for a new DB (i.e. execs CREATE TABLE statements).
readonly create: DBFunc;
// List of functions that perform DB migrations from one version to the next. This array's
// length determines the schema version, which is stored in user_version SQLite property.
//
// The very first migration should normally be identical to the original version of create().
// I.e. initially SchemaInfo should be { create: X, migrations: [X] }, where the two X's
// represent two copies of the same code. Don't go for code reuse here. When the schema is
// modified, you will change it to { create: X2, migrations: [X, Y] }. Keeping the unchanged
// copy of X is important as a reference to see that X + Y produces the same DB as X2.
//
// If you may open DBs created without versioning (e.g. predate use of this module), such DBs
// will go through all migrations including the very first one. In this case, the first
// migration's job is to bring any older DB to the same consistent state.
readonly migrations: ReadonlyArray<DBFunc>;
}
export type DBFunc = (db: SQLiteDB) => Promise<void>;
export enum OpenMode {
OPEN_CREATE, // Open DB or create if doesn't exist (the default mode for sqlite3 module)
OPEN_EXISTING, // Open DB or fail if doesn't exist
OPEN_READONLY, // Open DB in read-only mode or fail if doens't exist.
CREATE_EXCL, // Create new DB or fail if it already exists.
}
/**
* An interface implemented both by SQLiteDB and DocStorage (by forwarding). Methods
* documented in SQLiteDB.
*/
export interface ISQLiteDB {
exec(sql: string): Promise<void>;
run(sql: string, ...params: any[]): Promise<void>;
get(sql: string, ...params: any[]): Promise<ResultRow|undefined>;
all(sql: string, ...params: any[]): Promise<ResultRow[]>;
prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement>;
execTransaction<T>(callback: () => Promise<T>): Promise<T>;
runAndGetId(sql: string, ...params: any[]): Promise<number>;
requestVacuum(): Promise<boolean>;
}
/**
* Wrapper around sqlite3.Database. This class provides many of the same methods, but promisified.
* In addition, it offers:
*
* SQLiteDB.openDB(): Opens a DB, and initialize or migrate it to correct schema.
* db.execTransaction(cb): Runs a callback in the context of a new DB transaction.
*/
export class SQLiteDB {
/**
* Opens a database or creates a new one, according to OpenMode enum. The schemaInfo specifies
* how to initialize a new database, and how to migrate an existing one from an older version.
* If the database was migrated, its "migrationBackupPath" property will be set.
*
* If a migration was needed but failed, the DB remains unchanged, and gets opened anyway.
* We report the migration error, and expose it via .migrationError property.
*/
public static async openDB(dbPath: string, schemaInfo: SchemaInfo,
mode: OpenMode = OpenMode.OPEN_CREATE): Promise<SQLiteDB> {
const db = await SQLiteDB.openDBRaw(dbPath, mode);
const userVersion: number = await db.getMigrationVersion();
// It's possible that userVersion is 0 for a non-empty DB if it was created without this
// module. In that case, we apply migrations starting with the first one.
if (userVersion === 0 && (await isEmpty(db))) {
await db._initNewDB(schemaInfo);
} else if (mode === OpenMode.CREATE_EXCL) {
await db.close();
throw new ErrorWithCode('EEXISTS', `EEXISTS: Database already exists: ${dbPath}`);
} else {
// Don't attempt migrations in OPEN_READONLY mode.
if (mode === OpenMode.OPEN_READONLY) {
const targetVer: number = schemaInfo.migrations.length;
if (userVersion < targetVer) {
db._migrationError = new Error(`SQLiteDB[${dbPath}] needs migration but is readonly`);
}
} else {
try {
db._migrationBackupPath = await db._migrate(userVersion, schemaInfo);
} catch (err) {
db._migrationError = err;
}
}
await db._reportSchemaDiscrepancies(schemaInfo);
}
return db;
}
/**
* Opens a database or creates a new one according to OpenMode value. Does not check for or do
* any migrations.
*/
public static async openDBRaw(dbPath: string,
mode: OpenMode = OpenMode.OPEN_CREATE): Promise<SQLiteDB> {
const sqliteMode: number =
// tslint:disable-next-line:no-bitwise
(mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) |
(mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0);
let _db: sqlite3.Database;
await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); });
if (SQLiteDB._addOpens(dbPath, 1) > 1) {
log.warn("SQLiteDB[%s] avoid opening same DB more than once", dbPath);
}
return new SQLiteDB(_db!, dbPath);
}
/**
* Reads the migration version from the database without any attempts to migrate it.
*/
public static async getMigrationVersion(dbPath: string): Promise<number> {
const db = await SQLiteDB.openDBRaw(dbPath, OpenMode.OPEN_READONLY);
try {
return await db.getMigrationVersion();
} finally {
await db.close();
}
}
// It is a bad idea to open the same database file multiple times, because simultaneous use can
// cause SQLITE_BUSY errors, and artificial delays (default of 1 sec) when there is contention.
// We keep track of open DB paths, and warn if one is opened multiple times.
private static _openPaths: Map<string, number> = new Map();
// Convert the "create" function from schemaInfo into a DBMetadata object that describes the
// tables, columns, and types. This is used for checking if an open database matches the
// schema we expect, including after a migration, and reporting discrepancies.
private static async _getExpectedMetadata(schemaInfo: SchemaInfo): Promise<DBMetadata> {
// We cache the result and associate it with the create function, since it's not that cheap to
// build. To build the metadata, we open an in-memory DB and apply "create" function to it.
// Note that for tiny DBs it takes <10ms.
if (!dbMetadataCache.has(schemaInfo.create)) {
const db = await SQLiteDB.openDB(':memory:', schemaInfo, OpenMode.CREATE_EXCL);
dbMetadataCache.set(schemaInfo.create, await db.collectMetadata());
await db.close();
}
return dbMetadataCache.get(schemaInfo.create)!;
}
// Private helper to keep track of opens for the same path. Returns the number of times this
// path is open, after adding the delta. Use delta of +1 for open, -1 for close.
private static _addOpens(dbPath: string, delta: number): number {
const newCount = (SQLiteDB._openPaths.get(dbPath) || 0) + delta;
if (newCount > 0) {
SQLiteDB._openPaths.set(dbPath, newCount);
} else {
SQLiteDB._openPaths.delete(dbPath);
}
return newCount;
}
private _prevTransaction: Promise<any> = Promise.resolve();
private _inTransaction: boolean = false;
private _migrationBackupPath: string|null = null;
private _migrationError: Error|null = null;
private _needVacuum: boolean = false;
private constructor(private _db: sqlite3.Database, private _dbPath: string) {
// Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow
// This isn't enough for transactions, which we serialize explicitly.
this._db.serialize();
}
/**
* If a DB was migrated on open, this will be set to the path of the pre-migration backup copy.
* If migration failed, open throws with unchanged DB and no backup file.
*/
public get migrationBackupPath(): string|null { return this._migrationBackupPath; }
/**
* If a needed migration failed, the DB will be opened anyway, with this property set to the
* error. E.g. you may use it like so:
* sdb = await SQLiteDB.openDB(...)
* if (sdb.migrationError) { throw sdb.migrationError; }
*/
public get migrationError(): Error|null { return this._migrationError; }
// The following methods mirror https://github.com/mapbox/node-sqlite3/wiki/API, but return
// Promises. We use fromCallback() rather than use promisify, to get better type-checking.
public exec(sql: string): Promise<void> {
return fromCallback(cb => this._db.exec(sql, cb));
}
public run(sql: string, ...params: any[]): Promise<void> {
return fromCallback(cb => this._db.run(sql, ...params, cb));
}
public get(sql: string, ...params: any[]): Promise<ResultRow|undefined> {
return fromCallback(cb => this._db.get(sql, ...params, cb));
}
public all(sql: string, ...params: any[]): Promise<ResultRow[]> {
return fromCallback(cb => this._db.all(sql, ...params, cb));
}
public allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
// allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib.
return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb));
}
public prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement> {
let stmt: sqlite3.Statement;
// The original interface is a little strange; we resolve to Statement if prepare() succeeded.
return fromCallback(cb => { stmt = this._db.prepare(sql, ...params, cb); }).then(() => stmt);
}
/**
* VACUUM the DB either immediately or, if in a transaction, after that transaction.
*/
public async requestVacuum(): Promise<boolean> {
if (this._inTransaction) {
this._needVacuum = true;
return false;
}
await this.exec("VACUUM");
log.info("SQLiteDB[%s]: DB VACUUMed", this._dbPath);
this._needVacuum = false;
return true;
}
/**
* Run each of the statements in turn. Each statement is either a string, or an array of arguments
* to db.run, e.g. [sqlString, [params...]].
*/
public runEach(...statements: Array<string | [string, any[]]>): Promise<void> {
return each(statements, (stmt: any) => {
return (Array.isArray(stmt) ? this.run(stmt[0], ...stmt[1]) :
this.exec(stmt))
.catch(err => { log.warn(`SQLiteDB: Failed to run ${stmt}`); throw err; });
});
}
public close(): Promise<void> {
return fromCallback(cb => this._db.close(cb))
.then(() => { SQLiteDB._addOpens(this._dbPath, -1); });
}
/**
* As for run(), but captures the last_insert_rowid after the statement executes. This
* is sqlite's rowid for the last insert made on this database connection. This method
* is only useful if the sql is actually an INSERT operation, but we don't check this.
*/
public runAndGetId(sql: string, ...params: any[]): Promise<number> {
return new Promise<number>((resolve, reject) => {
this._db.run(sql, ...params, function(this: any, err: any) {
if (err) {
reject(err);
} else {
resolve(this.lastID);
}
});
});
}
/**
* Runs callback() in the context of a new DB transaction, committing on success and rolling
* back on error in the callback. The callback may return a promise, which will be waited for.
* The callback is called with no arguments.
*
* This method can be nested. The result is one big merged transaction that will succeed or
* roll back as a single unit.
*/
public async execTransaction<T>(callback: () => Promise<T>): Promise<T> {
if (this._inTransaction) {
return callback();
}
let outerResult;
try {
outerResult = await (this._prevTransaction = this._execTransactionImpl(async () => {
this._inTransaction = true;
let innerResult;
try {
innerResult = await callback();
} finally {
this._inTransaction = false;
}
return innerResult;
}));
} finally {
if (this._needVacuum) {
await this.requestVacuum();
}
}
return outerResult;
}
/**
* Returns the 'user_version' saved in the database that reflects the current DB schema. It is 0
* initially, and we update it to 1 or higher when initializing or migrating the database.
*/
public async getMigrationVersion(): Promise<number> {
const row = await this.get("PRAGMA user_version");
return (row && row.user_version) || 0;
}
/**
* Creates a DBMetadata object mapping DB's table names to column names to column types. Used
* for reporting discrepancies in DB schema, and exposed for tests.
*
* Optionally, a list of table names can be supplied, and metadata will be omitted for any
* tables not named in that list.
*/
public async collectMetadata(names?: string[]): Promise<DBMetadata> {
const tables = await this.all("SELECT name FROM sqlite_master WHERE type='table'");
const metadata: DBMetadata = {};
for (const t of tables) {
if (names && !names.includes(t.name)) { continue; }
const infoRows = await this.all(`PRAGMA table_info(${quoteIdent(t.name)})`);
const columns = fromPairs(infoRows.map(r => [r.name, r.type]));
metadata[t.name] = columns;
}
return metadata;
}
// Implementation of execTransction.
private async _execTransactionImpl<T>(callback: () => Promise<T>): Promise<T> {
// We need to swallow errors, so that one failed transaction doesn't cause the next one to fail.
await this._prevTransaction.catch(noop);
await this.exec("BEGIN");
try {
const value = await callback();
await this.exec("COMMIT");
return value;
} catch (err) {
try {
await this.exec("ROLLBACK");
} catch (rollbackErr) {
log.error("SQLiteDB[%s]: Rollback failed: %s", this._dbPath, rollbackErr);
}
throw err; // Throw the original error from the transaction.
}
}
/**
* Applies schemaInfo.create function to initialize a new DB.
*/
private async _initNewDB(schemaInfo: SchemaInfo): Promise<void> {
await this.execTransaction(async () => {
const targetVer: number = schemaInfo.migrations.length;
await schemaInfo.create(this);
await this.exec(`PRAGMA user_version = ${targetVer}`);
});
}
/**
* Applies migrations to this database according to MigrationInfo. In all cases, checks the
* database schema against MigrationInfo.currentSchema, and warns of discrepancies.
*
* If migration succeeded, it leaves a backup file and returns its path. If no migration was
* needed, returns null. If migration failed, leaves DB unchanged and throws Error.
*/
private async _migrate(actualVer: number, schemaInfo: SchemaInfo): Promise<string|null> {
const targetVer: number = schemaInfo.migrations.length;
let backupPath: string|null = null;
if (actualVer > targetVer) {
log.warn("SQLiteDB[%s]: DB is at version %s ahead of target version %s",
this._dbPath, actualVer, targetVer);
} else if (actualVer < targetVer) {
log.info("SQLiteDB[%s]: DB needs migration from version %s to %s",
this._dbPath, actualVer, targetVer);
const versions = range(actualVer, targetVer);
backupPath = await createBackupFile(this._dbPath, actualVer);
try {
await this.execTransaction(async () => {
for (const versionNum of versions) {
await schemaInfo.migrations[versionNum](this);
}
await this.exec(`PRAGMA user_version = ${targetVer}`);
});
// After a migration, reduce the sqlite file size. This must be run outside a transaction.
await this.run("VACUUM");
log.info("SQLiteDB[%s]: DB backed up to %s, migrated to %s",
this._dbPath, backupPath, targetVer);
} catch (err) {
// If the transaction failed, we trust SQLite to have left the DB in unmodified state, so
// we remove the pointless backup.
await fse.remove(backupPath);
backupPath = null;
log.warn("SQLiteDB[%s]: DB migration from %s to %s failed: %s",
this._dbPath, actualVer, targetVer, err);
err.message = `SQLiteDB[${this._dbPath}] migration to ${targetVer} failed: ${err.message}`;
throw err;
}
}
return backupPath;
}
private async _reportSchemaDiscrepancies(schemaInfo: SchemaInfo): Promise<void> {
// Regardless of where we started, warn if DB doesn't match expected schema.
const expected = await SQLiteDB._getExpectedMetadata(schemaInfo);
const metadata = await this.collectMetadata(Object.keys(expected));
for (const tname in expected) {
if (expected.hasOwnProperty(tname) && !isEqual(metadata[tname], expected[tname])) {
log.warn("SQLiteDB[%s]: table %s does not match schema: %s != %s",
this._dbPath, tname, JSON.stringify(metadata[tname]), JSON.stringify(expected[tname]));
}
}
}
}
// Every SchemaInfo.create function determines a DB structure. We can get it by initializing a
// dummy DB, and we use it to do sanity checking, in particular after migrations. To avoid
// creating dummy DBs multiple times, the result is cached, keyed by the "create" function itself.
const dbMetadataCache: Map<DBFunc, DBMetadata> = new Map();
interface DBMetadata {
[tableName: string]: {
[colName: string]: string; // Maps column name to SQLite type, e.g. "TEXT".
};
}
// Helper to see if a database is empty.
async function isEmpty(db: SQLiteDB): Promise<boolean> {
return (await db.get("SELECT count(*) as count FROM sqlite_master"))!.count === 0;
}
/**
* Copies filePath to "filePath.YYYY-MM-DD.V0[-N].bak", adding "-N" suffix (starting at "-2") if
* needed to ensure the path is new. Returns the backup path.
*/
async function createBackupFile(filePath: string, versionNum: number): Promise<string> {
const backupPath = await docUtils.createNumberedTemplate(
`${filePath}.${timeFormat('D', new Date())}.V${versionNum}{NUM}.bak`,
docUtils.createExclusive);
await docUtils.copyFile(filePath, backupPath);
return backupPath;
}
/**
* Validate and quote SQL identifiers such as table and column names.
*/
export function quoteIdent(ident: string): string {
assert(/^\w+$/.test(ident), `SQL identifier is not valid: ${ident}`);
return `"${ident}"`;
}

73
app/server/lib/SafePythonComponent.ts Normal file
View File

@@ -0,0 +1,73 @@
import {LocalPlugin} from 'app/common/plugin';
import {BaseComponent, createRpcLogger} from 'app/common/PluginInstance';
import {GristServer} from 'app/server/lib/GristServer';
import {ISandbox} from 'app/server/lib/ISandbox';
import * as log from 'app/server/lib/log';
import {IMsgCustom, IMsgRpcCall} from 'grain-rpc';
import * as path from 'path';
// TODO safePython component should be able to call other components function
// TODO calling a function on safePython component with a name that was not register chould fail
// gracefully.
/**
* The safePython component used by a PluginInstance.
*
* It uses `NSandbox` implementation of rpc for calling methods within the sandbox.
*/
export class SafePythonComponent extends BaseComponent {
private _sandbox: ISandbox;
private _logMeta: log.ILogMeta;
// safe python component does not need pluginInstance.rpc because it is not possible to forward
// calls to other component from within python
constructor(private _localPlugin: LocalPlugin,
private _mainPath: string, private _tmpDir: string,
docName: string, private _server: GristServer,
rpcLogger = createRpcLogger(log, `PLUGIN ${_localPlugin.id}/${_mainPath} SafePython:`)) {
super(_localPlugin.manifest, rpcLogger);
this._logMeta = {plugin: _localPlugin.id, docId: docName};
}
/**
* `SafePythonComponent` activation creates the Sandbox. Throws if the plugin has no `safePyton`
* components.
*/
protected async activateImplementation(): Promise<void> {
if (!this._tmpDir) {
throw new Error("Sanbox should have a tmpDir");
}
this._sandbox = this._server.create.NSandbox({
entryPoint: this._mainPath,
sandboxMount: path.join(this._localPlugin.path, 'sandbox'),
importMount: this._tmpDir,
logTimes: true,
logMeta: this._logMeta,
});
}
protected async deactivateImplementation(): Promise<void> {
log.info('SafePython deactivating ...');
if (!this._sandbox) {
log.info(' sandbox is undefined');
}
if (this._sandbox) {
await this._sandbox.shutdown();
log.info('SafePython done deactivating the sandbox');
delete this._sandbox;
}
}
protected doForwardCall(c: IMsgRpcCall): Promise<any> {
if (!this._sandbox) { throw new Error("Component should have be activated"); }
const {meth, iface, args} = c;
const funcName = meth === "invoke" ? iface : iface + "." + meth;
return this._sandbox.pyCall(funcName, ...args);
}
protected doForwardMessage(c: IMsgCustom): Promise<any> {
throw new Error("Forwarding messages to python sandbox is not supported");
}
}

41
app/server/lib/ServerColumnGetters.ts Normal file
View File

@@ -0,0 +1,41 @@
import {ColumnGetters} from 'app/common/ColumnGetters';
import * as gristTypes from 'app/common/gristTypes';
/**
*
* An implementation of ColumnGetters for the server, currently
* drawing on the data and metadata prepared for CSV export.
*
*/
export class ServerColumnGetters implements ColumnGetters {
private _rowIndices: Map<number, number>;
private _colIndices: Map<number, string>;
constructor(rowIds: number[], private _dataByColId: {[colId: string]: any}, private _columns: any[]) {
this._rowIndices = new Map<number, number>(rowIds.map((rowId, r) => [rowId, r] as [number, number]));
this._colIndices = new Map<number, string>(_columns.map(col => [col.id, col.colId] as [number, string]));
}
public getColGetter(colRef: number): ((rowId: number) => any) | null {
const colId = this._colIndices.get(colRef);
if (colId === undefined) {
return null;
}
const col = this._dataByColId[colId];
return rowId => {
const idx = this._rowIndices.get(rowId);
if (idx === undefined) {
return null;
}
return col[idx];
};
}
public getManualSortGetter(): ((rowId: number) => any) | null {
const manualSortCol = this._columns.find(c => c.colId === gristTypes.MANUALSORT);
if (!manualSortCol) {
return null;
}
return this.getColGetter(manualSortCol.id);
}
}

205
app/server/lib/ServerMetrics.js Normal file
View File

@@ -0,0 +1,205 @@
const _ = require('underscore');
const net = require('net');
const Promise = require('bluebird');
const log = require('./log');
const MetricCollector = require('app/common/MetricCollector');
const metricConfig = require('app/common/metricConfig');
const shutdown = require('./shutdown');
const version = require('app/common/version');
const crypto = require('crypto');
// Grist Metrics EC2 instance host and port
const host = 'metrics.getgrist.com';
const port = '2023'; // Plain-text port of carbon-aggregator
// Global reference to an instance of this class established in the constuctor.
var globalServerMetrics = null;
/**
* Server-facing class for initializing server metrics collection.
* Establishes interval attempts to push measured server metrics to the prometheus PushGateway
* on creation.
* @param {Object} user - Instance of User.js server class, which contains config settings.
*/
function ServerMetrics() {
MetricCollector.call(this);
this.socket = null;
// Randomly generated id to differentiate between metrics from this server and others.
this.serverId = crypto.randomBytes(8).toString('hex');
this.serverMetrics = this.initMetricTools(metricConfig.serverMetrics);
this.clientNames = null;
this.enabled = false;
// Produce the prefix string for all metrics.
// NOTE: If grist-rt is used instead of grist-raw for some metrics, this must be changed.
let versionStr = version.version.replace(/\W/g, '-');
let channelStr = version.channel.replace(/\W/g, '-');
this._prefix = `grist-raw.instance.${channelStr}.${versionStr}`;
globalServerMetrics = this;
// This will not send metrics when they are disabled since there is a check in pushMetrics.
shutdown.addCleanupHandler(null, () => this.attemptPush());
}
_.extend(ServerMetrics.prototype, MetricCollector.prototype);
/**
* Checks the given preferences object from the user configuration and starts pushing metrics
* to carbon if metrics are enabled. Otherwise, ends the socket connection if there is one.
*/
ServerMetrics.prototype.handlePreferences = function(config) {
config = config || {};
this.enabled = config.enableMetrics;
Promise.resolve(this.enabled && this._connectSocket())
.then(() => {
if (this.enabled) {
this._push = setTimeout(() => this.attemptPush(), metricConfig.SERVER_PUSH_INTERVAL);
} else if (this.socket) {
this.socket.end();
}
});
};
ServerMetrics.prototype.disable = function() {
this.enabled = false;
if (this._push) {
clearTimeout(this._push);
this._push = null;
}
if (this._collect) {
clearTimeout(this._collect);
this._collect = null;
}
}
/**
* Returns a promise for a socket connection to the Carbon metrics collection server.
* The promise will not fail because of connection errors, rather it will be continuously
* re-evaluated until it connects. The retry rate is specified in metricConfig.js
*/
ServerMetrics.prototype._connectSocket = function() {
if (!this.enabled) { return Promise.resolve(); }
var socket = null;
log.info('Attempting connection to Carbon metrics server');
return new Promise((resolve, reject) => {
socket = net.connect({host: host, port: port}, () => {
log.info('Connected to Carbon metrics server');
this.socket = socket;
resolve();
});
socket.setEncoding('utf8');
socket.on('error', err => {
log.warn('Carbon metrics connection error: %s', err);
if (this.socket) {
this.socket.end();
this.socket = null;
}
reject(err);
});
})
.catch(() => {
return Promise.delay(metricConfig.CONN_RETRY)
.then(() => this._connectSocket());
});
};
// Returns a map from metric names (as entered in metricConfig.js) to their metricTools.
ServerMetrics.prototype.getMetrics = function() {
return this.serverMetrics;
};
// Pushes ready server and client metrics to the aggregator
ServerMetrics.prototype.pushMetrics = function(metrics) {
if (this.enabled) {
return this._request(metrics.join(""))
.finally(() => {
this._push = setTimeout(() => this.attemptPush(), metricConfig.SERVER_PUSH_INTERVAL);
});
}
};
ServerMetrics.prototype._request = function(text) {
return new Promise(resolve => {
if (!this.enabled) {
resolve();
return;
}
this.socket.write(text, 'utf8', () => {
log.info('Pushed metrics to Carbon');
resolve();
});
})
.catch(() => {
return this._connectSocket()
.then(() => this._request(text));
});
};
/**
* Function exposed to comm interface to provide server with client list of metrics.
* Used so that ServerMetrics can associate indices to client metric names.
* @param {Array} metricNames - A list of client metric names in the order in which values will be sent.
*/
ServerMetrics.prototype.registerClientMetrics = function(client, metricNames) {
this.clientNames = metricNames;
};
/**
* Function exposed to comm interface to allow client metrics to be pushed to this file,
* so that they may in turn be pushed to Carbon with the server metrics.
* @param {Array} data - A list of client buckets as defined in ClientMetrics.js's createBucket
*/
ServerMetrics.prototype.pushClientMetrics = function(client, data) {
// Merge ready client bucket metrics into ready server buckets.
if (!this.clientNames) {
throw new Error("Client metrics must be registered");
}
data.forEach(clientBucket => {
// Label the bucket with the client id so that clients' metrics do not replace one another
let clientData = clientBucket.values.map((val, i) => {
return this._stringifyMetric(this.clientNames[i], client.clientId, val, clientBucket.startTime);
}).join("");
this.queueBucket(clientData);
});
};
ServerMetrics.prototype.get = function(name) {
this.prepareCompletedBuckets(Date.now());
return this.serverMetrics[name];
};
/**
* Creates string bucket with metrics in carbon's text format.
* For details, see phriction documentation: https://phab.getgrist.com/w/metrics/
*/
ServerMetrics.prototype.createBucket = function(bucketStart) {
var data = [];
var bucketEnd = bucketStart + metricConfig.BUCKET_SIZE;
this.forEachBucketMetric(bucketEnd, tool => {
if (tool.getValue(bucketEnd) !== null) {
data.push(this._stringifyMetric(tool.getName(), this.serverId, tool.getValue(bucketEnd), bucketStart));
}
});
return data.join("");
};
// Helper to stringify individual metrics for carbon's text format.
ServerMetrics.prototype._stringifyMetric = function(name, id, val, startTime) {
// Server/client id is added to name for differentiating inputs to aggregator
return `${this._prefix}.${name}.${id} ${val} ${startTime/1000}\n`;
};
/**
* Static get method to retreive server metric recording tools.
* IMPORTANT: Usage involves the side effect of updating completed buckets and
* adding them to a ready object. get() results should not be assigned to variables and
* reused, rather get() should be called each time a metric is needed.
*/
ServerMetrics.get = function(name) {
if (!globalServerMetrics) {
throw new Error('Must create ServerMetrics instance to access server metrics.');
}
return globalServerMetrics.get(name);
};
module.exports = ServerMetrics;

107
app/server/lib/Sessions.ts Normal file
View File

@@ -0,0 +1,107 @@
import {ScopedSession} from 'app/server/lib/BrowserSession';
import * as Comm from 'app/server/lib/Comm';
import {GristServer} from 'app/server/lib/GristServer';
import {cookieName, SessionStore} from 'app/server/lib/gristSessions';
import {IInstanceManager} from 'app/server/lib/IInstanceManager';
import {ILoginSession} from 'app/server/lib/ILoginSession';
import * as cookie from 'cookie';
import * as cookieParser from 'cookie-parser';
import {Request} from 'express';
interface Session {
scopedSession: ScopedSession;
loginSession?: ILoginSession;
}
/**
*
* A collection of all the sessions relevant to this instance of Grist.
*
* This collection was previously maintained by the Comm object. This
* class is added as a stepping stone to disentangling session management
* from code related to websockets.
*
* The collection caches all existing interfaces to sessions.
* LoginSessions play an important role in standalone Grist and address
* end-to-end sharing concerns. ScopedSessions play an important role in
* hosted Grist and address per-organization scoping of identity.
*
* TODO: now this is separated out, we could refactor to share sessions
* across organizations. Currently, when a user moves between organizations,
* the session interfaces are not shared. This was for simplicity in working
* with existing code.
*
*/
export class Sessions {
private _sessions = new Map<string, Session>();
constructor(private _sessionSecret: string, private _sessionStore: SessionStore, private _server: GristServer) {
}
/**
* Get the session id and organization from the request, and return the
* identified session.
*/
public getOrCreateSessionFromRequest(req: Request): Session {
const sid = this.getSessionIdFromRequest(req);
const org = (req as any).org;
if (!sid) { throw new Error("session not found"); }
return this.getOrCreateSession(sid, org);
}
/**
* Get or create a session given the session id and organization name.
*/
public getOrCreateSession(sid: string, domain: string): Session {
const key = this._getSessionOrgKey(sid, domain);
if (!this._sessions.has(key)) {
const scopedSession = new ScopedSession(sid, this._sessionStore, domain);
this._sessions.set(key, {scopedSession});
}
return this._sessions.get(key)!;
}
/**
* Access a LoginSession interface, creating it if necessary. For creation,
* purposes, Comm, and optionally InstanceManager objects are needed.
*
*/
public getOrCreateLoginSession(sid: string, domain: string, comm: Comm,
instanceManager: IInstanceManager|null): ILoginSession {
const sess = this.getOrCreateSession(sid, domain);
if (!sess.loginSession) {
sess.loginSession = this._server.create.LoginSession(comm, sid, domain, sess.scopedSession,
instanceManager);
}
return sess.loginSession;
}
/**
* Returns the sessionId from the signed grist cookie.
*/
public getSessionIdFromCookie(gristCookie: string) {
return cookieParser.signedCookie(gristCookie, this._sessionSecret);
}
/**
* Get the session id from the grist cookie. Returns null if no cookie found.
*/
public getSessionIdFromRequest(req: Request): string|null {
if (req.headers.cookie) {
const cookies = cookie.parse(req.headers.cookie);
const sessionId = this.getSessionIdFromCookie(cookies[cookieName]);
return sessionId;
}
return null;
}
/**
* Get a per-organization, per-session key.
* Grist has historically cached sessions in memory by their session id.
* With the introduction of per-organization identity, that cache is now
* needs to be keyed by the session id and organization name.
*/
private _getSessionOrgKey(sid: string, domain: string): string {
return `${sid}__${domain}`;
}
}

387
app/server/lib/Sharing.ts Normal file
View File

@@ -0,0 +1,387 @@
import {ActionBundle, LocalActionBundle, UserActionBundle} from 'app/common/ActionBundle';
import {ActionInfo, Envelope, getEnvContent} from 'app/common/ActionBundle';
import {DocAction, UserAction} from 'app/common/DocActions';
import {allToken, Peer} from 'app/common/sharing';
import {timeFormat} from 'app/common/timeFormat';
import * as log from 'app/server/lib/log';
import {shortDesc} from 'app/server/lib/shortDesc';
import * as assert from 'assert';
import * as Deque from 'double-ended-queue';
import {ActionHistory, asActionGroup} from './ActionHistory';
import {ActiveDoc} from './ActiveDoc';
import {Client} from './Client';
import {WorkCoordinator} from './WorkCoordinator';
// Describes the request to apply a UserActionBundle. It includes a Client (so that broadcast
// message can set `.fromSelf` property), and methods to resolve or reject the promise for when
// the action is applied. Note that it may not be immediate in case we are in the middle of
// processing hub actions or rebasing.
interface UserRequest {
action: UserActionBundle;
client: Client|null;
resolve(result: UserResult): void;
reject(err: Error): void;
}
// The result of applying a UserRequest, used to resolve the promise. It includes the retValues
// (one for each UserAction in the bundle) and the actionNum of the applied LocalActionBundle.
interface UserResult {
actionNum: number;
retValues: any[];
isModification: boolean;
}
// Internally-used enum to distinguish if applied actions should be logged as local or shared.
enum Branch { Local, Shared }
export class Sharing {
protected _activeDoc: ActiveDoc;
protected _actionHistory: ActionHistory;
protected _hubQueue: Deque<ActionBundle> = new Deque();
protected _pendingQueue: Deque<UserRequest> = new Deque();
protected _workCoordinator: WorkCoordinator;
constructor(activeDoc: ActiveDoc, actionHistory: ActionHistory) {
// TODO actionHistory is currently unused (we use activeDoc.actionLog).
assert(actionHistory.isInitialized());
this._activeDoc = activeDoc;
this._actionHistory = actionHistory;
this._workCoordinator = new WorkCoordinator(() => this._doNextStep());
}
/** Initialize the sharing for a previously-shared doc. */
public async openSharedDoc(hub: any, docId: string): Promise<void> {
throw new Error('openSharedDoc not implemented');
}
/** Initialize the sharing for a newly-shared doc. */
public async createSharedDoc(hub: any, docId: string, docName: string, peers: Peer[]): Promise<void> {
throw new Error('openSharedDoc not implemented');
}
/**
* Returns whether this doc is shared. It's shared if and only if HubDocClient is set (though it
* may be disconnected).
*/
public isShared(): boolean { return false; }
public isSharingActivated(): boolean { return false; }
/** Returns the instanceId if the doc is shared or null otherwise. */
public get instanceId(): string|null { return null; }
public isOwnEnvelope(recipients: string[]): boolean { return true; }
public async sendLocalAction(): Promise<void> {
throw new Error('sendLocalAction not implemented');
}
public async shareDoc(docName: string, peers: Peer[]): Promise<void> {
throw new Error('shareDoc not implemented');
}
public async removeInstanceFromDoc(): Promise<string> {
throw new Error('removeInstanceFromDoc not implemented');
}
/**
* The only public interface. This may be called at any time, including while rebasing.
* WorkCoordinator ensures that actual work will only happen once other work finishes.
*/
public addUserAction(userRequest: UserRequest) {
this._pendingQueue.push(userRequest);
this._workCoordinator.ping();
}
// Returns a promise if there is some work happening, or null if there isn't.
private _doNextStep(): Promise<void>|null {
if (this._hubQueue.isEmpty()) {
if (!this._pendingQueue.isEmpty()) {
return this._applyLocalAction();
} else if (this.isSharingActivated() && this._actionHistory.haveLocalUnsent()) {
return this.sendLocalAction();
} else {
return null;
}
} else {
if (!this._actionHistory.haveLocalActions()) {
return this._applyHubAction();
} else {
return this._mergeInHubAction();
}
}
}
private async _applyLocalAction(): Promise<void> {
assert(this._hubQueue.isEmpty() && !this._pendingQueue.isEmpty());
const userRequest: UserRequest = this._pendingQueue.shift()!;
try {
const ret = await this.doApplyUserActionBundle(userRequest.action, userRequest.client);
userRequest.resolve(ret);
} catch (e) {
log.warn("Unable to apply action...", e);
userRequest.reject(e);
}
}
private async _applyHubAction(): Promise<void> {
assert(!this._hubQueue.isEmpty() && !this._actionHistory.haveLocalActions());
const action: ActionBundle = this._hubQueue.shift()!;
try {
await this.doApplySharedActionBundle(action);
} catch (e) {
log.error("Unable to apply hub action... skipping");
}
}
private async _mergeInHubAction(): Promise<void> {
assert(!this._hubQueue.isEmpty() && this._actionHistory.haveLocalActions());
const action: ActionBundle = this._hubQueue.peekFront()!;
try {
const accepted = await this._actionHistory.acceptNextSharedAction(action.actionHash);
if (accepted) {
this._hubQueue.shift();
} else {
await this._rebaseLocalActions();
}
} catch (e) {
log.error("Unable to apply hub action... skipping");
}
}
private async _rebaseLocalActions(): Promise<void> {
const rebaseQueue: Deque<UserActionBundle> = new Deque<UserActionBundle>();
try {
await this.createCheckpoint();
const actions: LocalActionBundle[] = await this._actionHistory.fetchAllLocal();
assert(actions.length > 0);
await this.doApplyUserActionBundle(this._createUndo(actions), null);
rebaseQueue.push(...actions.map((a) => getUserActionBundle(a)));
await this._actionHistory.clearLocalActions();
} catch (e) {
log.error("Can't undo local actions; sharing is off");
await this.rollbackToCheckpoint();
// TODO this.disconnect();
// TODO errorState = true;
return;
}
assert(!this._actionHistory.haveLocalActions());
while (!this._hubQueue.isEmpty()) {
await this._applyHubAction();
}
const rebaseFailures: Array<[UserActionBundle, UserActionBundle]> = [];
while (!rebaseQueue.isEmpty()) {
const action: UserActionBundle = rebaseQueue.shift()!;
const adjusted: UserActionBundle = this._mergeAdjust(action);
try {
await this.doApplyUserActionBundle(adjusted, null);
} catch (e) {
log.warn("Unable to apply rebased action...");
rebaseFailures.push([action, adjusted]);
}
}
if (rebaseFailures.length > 0) {
await this.createBackupAtCheckpoint();
// TODO we should notify the user too.
log.error('Rebase failed to reapply some of your actions, backup of local at...');
}
await this.releaseCheckpoint();
}
// ======================================================================
private doApplySharedActionBundle(action: ActionBundle): Promise<UserResult> {
const userActions: UserAction[] = [
['ApplyDocActions', action.stored.map(envContent => envContent[1])]
];
return this.doApplyUserActions(action.info[1], userActions, Branch.Shared, null);
}
private doApplyUserActionBundle(action: UserActionBundle, client: Client|null): Promise<UserResult> {
return this.doApplyUserActions(action.info, action.userActions, Branch.Local, client);
}
private async doApplyUserActions(info: ActionInfo, userActions: UserAction[],
branch: Branch, client: Client|null): Promise<UserResult> {
const sandboxActionBundle = await this._activeDoc.applyActionsToDataEngine(userActions);
// A trivial action does not merit allocating an actionNum,
// logging, and sharing. Since we currently don't store
// calculated values in the database, it is best not to log the
// action that initializes them when the document is opened cold
// (without cached ActiveDoc) - otherwise we'll end up with spam
// log entries for each time the document is opened cold.
const trivial = (userActions.length === 1 &&
userActions[0][0] === 'Calculate' &&
sandboxActionBundle.stored.length === 0);
const actionNum = trivial ? 0 :
(branch === Branch.Shared ? this._actionHistory.getNextHubActionNum() :
this._actionHistory.getNextLocalActionNum());
const localActionBundle: LocalActionBundle = {
actionNum,
// The ActionInfo should go into the envelope that includes all recipients.
info: [findOrAddAllEnvelope(sandboxActionBundle.envelopes), info],
envelopes: sandboxActionBundle.envelopes,
stored: sandboxActionBundle.stored,
calc: sandboxActionBundle.calc,
undo: getEnvContent(sandboxActionBundle.undo),
userActions,
actionHash: null, // Gets set below by _actionHistory.recordNext...
parentActionHash: null, // Gets set below by _actionHistory.recordNext...
};
this._logActionBundle(`doApplyUserActions (${Branch[branch]})`, localActionBundle);
// TODO Note that the sandbox may produce actions which are not addressed to us (e.g. when we
// have EDIT permission without VIEW). These are not sent to the browser or the database. But
// today they are reflected in the sandbox. Should we (or the sandbox) immediately undo the
// full change, and then redo only the actions addressed to ourselves? Let's cross that bridge
// when we come to it. For now we only log skipped envelopes as "alien" in _logActionBundle().
const ownActionBundle: LocalActionBundle = this._filterOwnActions(localActionBundle);
// Apply the action to the database, and record in the action log.
if (!trivial) {
await this._activeDoc.docStorage.execTransaction(async () => {
await this._activeDoc.docStorage.applyStoredActions(getEnvContent(ownActionBundle.stored));
if (this.isShared() && branch === Branch.Local) {
// this call will compute an actionHash for localActionBundle
await this._actionHistory.recordNextLocalUnsent(localActionBundle);
} else {
// Before sharing is enabled, actions are immediately marked as "shared" (as if accepted
// by the hub). The alternative of keeping actions on the "local" branch until sharing is
// enabled is less suitable, because such actions could have empty envelopes, and cannot
// be shared. Once sharing is enabled, we would share a snapshot at that time.
await this._actionHistory.recordNextShared(localActionBundle);
}
if (client && client.clientId) {
this._actionHistory.setActionClientId(localActionBundle.actionHash!, client.clientId);
}
});
}
await this._activeDoc.processActionBundle(ownActionBundle);
// In the future, we'll save (and share) the result of applying one bundle of UserActions
// as a single ActionBundle with one actionNum. But the old ActionLog saves on UserAction
// per actionNum, using linkId to "bundle" them for the purpose of undo-redo. We simulate
// it here by breaking up ActionBundle into as many old-style ActionGroups as there are
// UserActions, and associating all DocActions with the first of these ActionGroups.
// Broadcast the action to connected browsers.
const actionGroup = asActionGroup(this._actionHistory, localActionBundle, {
client,
retValues: sandboxActionBundle.retValues,
summarize: true,
});
await this._activeDoc.docClients.broadcastDocMessage(client || null, 'docUserAction', {
actionGroup,
docActions: getEnvContent(localActionBundle.stored).concat(
getEnvContent(localActionBundle.calc))
});
return {
actionNum: localActionBundle.actionNum,
retValues: sandboxActionBundle.retValues,
isModification: sandboxActionBundle.stored.length > 0
};
}
private _mergeAdjust(action: UserActionBundle): UserActionBundle {
// TODO: This is where we adjust actions after rebase, e.g. add delta to rowIds and such.
return action;
}
/**
* Creates a UserActionBundle with a single 'ApplyUndoActions' action, which combines the undo
* actions addressed to ourselves from all of the passed-in LocalActionBundles.
*/
private _createUndo(localActions: LocalActionBundle[]): UserActionBundle {
assert(localActions.length > 0);
const undo: DocAction[] = [];
for (const local of localActions) {
undo.push(...local.undo);
}
const first = localActions[0];
return {
info: {
time: Date.now(),
user: first.info[1].user,
inst: first.info[1].inst,
desc: "UNDO BEFORE REBASE",
otherId: 0,
linkId: 0,
},
userActions: [['ApplyUndoActions', undo]]
};
}
// Our beautiful little checkpointing interface, used to handle errors during rebase.
private createCheckpoint() { /* TODO */ }
private releaseCheckpoint() { /* TODO */ }
private rollbackToCheckpoint() { /* TODO */ }
private createBackupAtCheckpoint() { /* TODO */ }
/**
* Reduces a LocalActionBundle down to only those actions addressed to ourselves.
*/
private _filterOwnActions(localActionBundle: LocalActionBundle): LocalActionBundle {
const includeEnv: boolean[] = localActionBundle.envelopes.map(
(e) => this.isOwnEnvelope(e.recipients));
return Object.assign({}, localActionBundle, {
stored: localActionBundle.stored.filter((ea) => includeEnv[ea[0]]),
calc: localActionBundle.calc.filter((ea) => includeEnv[ea[0]]),
});
}
/** Log an action bundle to the debug log. */
private _logActionBundle(prefix: string, actionBundle: ActionBundle) {
const includeEnv = actionBundle.envelopes.map((e) => this.isOwnEnvelope(e.recipients));
log.debug("%s: ActionBundle #%s with #%s envelopes: %s",
prefix, actionBundle.actionNum, actionBundle.envelopes.length,
infoDesc(actionBundle.info[1]));
actionBundle.envelopes.forEach((env, i) =>
log.debug("%s: env #%s: %s", prefix, i, env.recipients.join(' ')));
actionBundle.stored.forEach((envAction, i) =>
log.debug("%s: stored #%s [%s%s]: %s", prefix, i, envAction[0],
(includeEnv[envAction[0]] ? "" : " alien"),
shortDesc(envAction[1])));
actionBundle.calc.forEach((envAction, i) =>
log.debug("%s: calc #%s [%s%s]: %s", prefix, i, envAction[0],
(includeEnv[envAction[0]] ? "" : " alien"),
shortDesc(envAction[1])));
}
}
/**
* Returns the index of the envelope containing the '#ALL' recipient, adding such an envelope to
* the provided array if it wasn't already there.
*/
export function findOrAddAllEnvelope(envelopes: Envelope[]): number {
const i = envelopes.findIndex(e => e.recipients.includes(allToken));
if (i >= 0) { return i; }
envelopes.push({recipients: [allToken]});
return envelopes.length - 1;
}
/**
* Convert actionInfo to a concise human-readable description, for debugging.
*/
function infoDesc(info: ActionInfo): string {
const timestamp = timeFormat('A', new Date(info.time));
const desc = info.desc ? ` desc=[${info.desc}]` : '';
const otherId = info.otherId ? ` [otherId=${info.otherId}]` : '';
const linkId = info.linkId ? ` [linkId=${info.linkId}]` : '';
return `${timestamp} on ${info.inst} by ${info.user}${desc}${otherId}${linkId}`;
}
/**
* Extract a UserActionBundle from a LocalActionBundle, which contains a superset of data.
*/
function getUserActionBundle(localAction: LocalActionBundle): UserActionBundle {
return {
info: localAction.info[1],
userActions: localAction.userActions
};
}

74
app/server/lib/TagChecker.ts Normal file
View File

@@ -0,0 +1,74 @@
import {tbind} from 'app/common/tbind';
import {NextFunction, Request, RequestHandler, Response} from 'express';
export type RequestWithTag = Request & {tag: string|null};
/**
*
* Middleware to handle a /v/TAG/ prefix on urls.
*
*/
export class TagChecker {
// Use app.use(tagChecker.inspectTag) to strip /v/TAG/ from urls (if it is present).
// If the tag is present and matches what is expected, then `tag` is set on the request.
// If the tag is present but does not match what is expected, a 400 response is returned.
// If the tag is absent, `tag` is not set on the request.
public readonly inspectTag: RequestHandler = tbind(this._inspectTag, this);
// Use app.get('/path', tagChecker.requireTag, ...) to serve something only if the tag was
// present in the url. If the tag was not present, the route will not match and express will
// look further.
public readonly requireTag: RequestHandler = tbind(this._requireTag, this);
// pass in the tag to expect.
public constructor(public tag: string) {
}
// Like requireTag but for use wrapping other handlers in app.use().
// Whatever it wraps will be skipped if that tag was not set.
// See https://github.com/expressjs/express/issues/2591
public withTag(handler: RequestHandler) {
return function fn(req: Request, resp: Response, next: NextFunction) {
if (!(req as RequestWithTag).tag) { return next(); }
return handler(req, resp, next);
};
}
// Removes tag from url if present.
// Returns [remainder, tagInUrl, isMatch]
private _removeTag(url: string): [string, string|null, boolean] {
if (url.startsWith('/v/')) {
const taggedUrl = url.match(/^\/v\/([a-zA-Z0-9.\-_]+)(\/.*)/);
if (taggedUrl) {
const tag = taggedUrl[1];
// Turn off tag matching as we transition to serving
// static resources from CDN. We don't have version-sensitive
// routing, so under ordinary operation landing page html served
// by one home server could have its assets served by another home server.
// Once the CDN is active, those asset requests won't reach the home
// servers. TODO: turn tag matching back on when tag mismatches
// imply a bug.
return [taggedUrl[2], tag, true /* tag === this.tag */];
}
}
return [url, null, true];
}
private async _inspectTag(req: Request, resp: Response, next: NextFunction) {
const [newUrl, urlTag, isOk] = this._removeTag(req.url);
if (!isOk) {
return resp.status(400).send({error: "Tag mismatch",
expected: this.tag,
received: urlTag});
}
req.url = newUrl;
(req as RequestWithTag).tag = urlTag;
return next();
}
private async _requireTag(req: Request, resp: Response, next: NextFunction) {
if ((req as RequestWithTag).tag) { return next(); }
return next('route');
}
}

205
app/server/lib/TestingHooks.ts Normal file
View File

@@ -0,0 +1,205 @@
import * as net from 'net';
import {UserProfile} from 'app/common/LoginSessionAPI';
import {Deps as ActiveDocDeps} from 'app/server/lib/ActiveDoc';
import * as Comm from 'app/server/lib/Comm';
import {ILoginSession} from 'app/server/lib/ILoginSession';
import * as log from 'app/server/lib/log';
import {IMessage, Rpc} from 'grain-rpc';
import {createCheckers} from 'ts-interface-checker';
import {FlexServer} from './FlexServer';
import {IInstanceManager} from './IInstanceManager';
import {ITestingHooks} from './ITestingHooks';
import ITestingHooksTI from './ITestingHooks-ti';
import {connect, fromCallback} from './serverUtils';
export function startTestingHooks(socketPath: string, port: number, instanceManager: IInstanceManager,
comm: Comm, flexServer: FlexServer,
workerServers: FlexServer[]): Promise<net.Server> {
// Create socket server listening on the given path for testing connections.
return new Promise((resolve, reject) => {
const server = net.createServer();
server.on('error', reject);
server.on('listening', () => resolve(server));
server.on('connection', socket => {
// On connection, create an Rpc object communicating over that socket.
const rpc = connectToSocket(new Rpc({logger: {}}), socket);
// Register the testing implementation.
rpc.registerImpl('testing',
new TestingHooks(port, instanceManager, comm, flexServer, workerServers),
createCheckers(ITestingHooksTI).ITestingHooks);
});
server.listen(socketPath);
});
}
function connectToSocket(rpc: Rpc, socket: net.Socket): Rpc {
socket.setEncoding('utf8');
socket.on('data', (buf: string) => rpc.receiveMessage(JSON.parse(buf)));
rpc.setSendMessage((m: IMessage) => fromCallback(cb => socket.write(JSON.stringify(m), 'utf8', cb)));
return rpc;
}
export interface TestingHooksClient extends ITestingHooks {
close(): void;
}
export async function connectTestingHooks(socketPath: string): Promise<TestingHooksClient> {
const socket = await connect(socketPath);
const rpc = connectToSocket(new Rpc({logger: {}}), socket);
return Object.assign(rpc.getStub<TestingHooks>('testing', createCheckers(ITestingHooksTI).ITestingHooks), {
close: () => socket.end(),
});
}
export class TestingHooks implements ITestingHooks {
constructor(
private _port: number,
private _instanceManager: IInstanceManager,
private _comm: Comm,
private _server: FlexServer,
private _workerServers: FlexServer[]
) {}
public getOwnPort(): number {
log.info("TestingHooks.getOwnPort called");
return this._server.getOwnPort();
}
public getPort(): number {
log.info("TestingHooks.getPort called");
return this._port;
}
public async updateAuthToken(instId: string, authToken: string): Promise<void> {
log.info("TestingHooks.updateAuthToken called with", instId, authToken);
const loginSession = this._getLoginSession(instId);
await loginSession.updateTokenForTesting(authToken);
}
public async getAuthToken(instId: string): Promise<string|null> {
log.info("TestingHooks.getAuthToken called with", instId);
const loginSession = this._getLoginSession(instId);
return await loginSession.getCurrentTokenForTesting();
}
public async useTestToken(instId: string, token: string): Promise<void> {
log.info("TestingHooks.useTestToken called with", token);
const loginSession = this._getLoginSession(instId);
return await loginSession.useTestToken(token);
}
public async setLoginSessionProfile(gristSidCookie: string, profile: UserProfile|null, org?: string): Promise<void> {
log.info("TestingHooks.setLoginSessionProfile called with", gristSidCookie, profile, org);
const sessionId = this._comm.getSessionIdFromCookie(gristSidCookie);
const loginSession = this._comm.getOrCreateSession(sessionId, {org});
return await loginSession.testSetProfile(profile);
}
public async setServerVersion(version: string|null): Promise<void> {
log.info("TestingHooks.setServerVersion called with", version);
this._comm.setServerVersion(version);
for (const server of this._workerServers) {
await server.comm.setServerVersion(version);
}
}
public async disconnectClients(): Promise<void> {
log.info("TestingHooks.disconnectClients called");
this._comm.destroyAllClients();
for (const server of this._workerServers) {
await server.comm.destroyAllClients();
}
}
public async commShutdown(): Promise<void> {
log.info("TestingHooks.commShutdown called");
await this._comm.testServerShutdown();
for (const server of this._workerServers) {
await server.comm.testServerShutdown();
}
}
public async commRestart(): Promise<void> {
log.info("TestingHooks.commRestart called");
await this._comm.testServerRestart();
for (const server of this._workerServers) {
await server.comm.testServerRestart();
}
}
// Set how long new clients will persist after disconnection.
// Call with 0 to return to default duration.
public async commSetClientPersistence(ttlMs: number) {
log.info("TestingHooks.setClientPersistence called with", ttlMs);
this._comm.testSetClientPersistence(ttlMs);
for (const server of this._workerServers) {
await server.comm.testSetClientPersistence(ttlMs);
}
}
public async closeDocs(): Promise<void> {
log.info("TestingHooks.closeDocs called");
if (this._server) {
await this._server.closeDocs();
}
for (const server of this._workerServers) {
await server.closeDocs();
}
}
public async setDocWorkerActivation(workerId: string, active: 'active'|'inactive'|'crash'):
Promise<void> {
log.info("TestingHooks.setDocWorkerActivation called with", workerId, active);
for (const server of this._workerServers) {
if (server.worker.id === workerId || server.worker.publicUrl === workerId) {
switch (active) {
case 'active':
await server.restartListening();
break;
case 'inactive':
await server.stopListening();
break;
case 'crash':
await server.stopListening('crash');
break;
}
return;
}
}
throw new Error(`could not find worker: ${workerId}`);
}
public async flushAuthorizerCache(): Promise<void> {
log.info("TestingHooks.flushAuthorizerCache called");
this._server.dbManager.flushDocAuthCache();
for (const server of this._workerServers) {
await server.dbManager.flushDocAuthCache();
}
}
// Returns a Map from docId to number of connected clients for all open docs across servers,
// but represented as an array of pairs, to be serializable.
public async getDocClientCounts(): Promise<Array<[string, number]>> {
log.info("TestingHooks.getDocClientCounts called");
const counts = new Map<string, number>();
for (const server of [this._server, ...this._workerServers]) {
const c = await server.getDocClientCounts();
for (const [key, val] of c) {
counts.set(key, (counts.get(key) || 0) + val);
}
}
return Array.from(counts);
}
// Sets the seconds for ActiveDoc timeout, and returns the previous value.
public async setActiveDocTimeout(seconds: number): Promise<number> {
const prev = ActiveDocDeps.ACTIVEDOC_TIMEOUT;
ActiveDocDeps.ACTIVEDOC_TIMEOUT = seconds;
return prev;
}
private _getLoginSession(instId: string): ILoginSession {
return this._instanceManager.getLoginSession(instId);
}
}

252
app/server/lib/Throttle.ts Normal file
View File

@@ -0,0 +1,252 @@
/**
*
* Simple CPU throttling implementation.
*
* For this setup, a sandbox attempting to use 100% of cpu over an
* extended period will end up throttled, in the steady-state, to
* 10% of cpu.
*
* Very simple mechanism to begin with. "ctime" is measured for the
* sandbox, being the cumulative time charged to the user (directly or
* indirectly) by the OS for that process. If the average increase in
* ctime over a time period is over 10% (targetRate) of that time period,
* throttling kicks in, and the process will be paused/unpaused via
* signals on a duty cycle.
*
* Left for future work: more careful shaping of CPU throttling, and
* factoring in a team-site level credit system or similar.
*
*/
import * as pidusage from '@gristlabs/pidusage';
import * as log from 'app/server/lib/log';
/**
* Parameters related to throttling.
*/
export interface ThrottleTiming {
dutyCyclePositiveMs: number; // when throttling, how much uninterrupted time to give
// the process before pausing it. The length of the
// non-positive cycle is chosen to achieve the desired
// cpu usage.
samplePeriodMs: number; // how often to sample cpu usage and update throttling
targetAveragingPeriodMs: number; // (rough) time span to average cpu usage over.
minimumAveragingPeriodMs: number; // minimum time span before throttling is considered.
// No throttling will occur before a process has run
// for at least this length of time.
minimumLogPeriodMs: number; // minimum time between log messages about throttling.
targetRate: number; // when throttling, aim for this fraction of cpu usage
// per unit time.
maxThrottle: number; // maximum ratio of negative duty cycle phases to
// positive.
}
/**
* Some parameters that seem reasonable defaults.
*/
const defaultThrottleTiming: ThrottleTiming = {
dutyCyclePositiveMs: 50,
samplePeriodMs: 1000,
targetAveragingPeriodMs: 20000,
minimumAveragingPeriodMs: 6000,
minimumLogPeriodMs: 10000,
targetRate: 0.25,
maxThrottle: 10,
};
/**
* A sample of cpu usage.
*/
interface MeterSample {
time: number; // time at which sample was made (as reported by Date.now())
cpuDuration: number; // accumulated "ctime" measured by pidusage
offDuration: number; // accumulated clock time for which process was paused (approximately)
}
/**
* A throttling implementation for a process. Supply a pid, and it will try to keep that
* process from consuming too much cpu until stop() is called.
*/
export class Throttle {
private _timing: ThrottleTiming; // overall timing parameters
private _meteringInterval: NodeJS.Timeout | undefined; // timer for cpu measurements
private _dutyCycleTimeout: NodeJS.Timeout | undefined; // driver for throttle duty cycle
private _throttleFactor: number = 0; // relative length of paused phase
private _sample: MeterSample | undefined; // latest measurement.
private _anchor: MeterSample | undefined; // sample from past for averaging
private _nextAnchor: MeterSample | undefined; // upcoming replacement for _anchor
private _lastLogTime: number | undefined; // time of last throttle log message
private _offDuration: number = 0; // cumulative time spent paused
private _stopped: boolean = false; // set when stop has been called
/**
* Start monitoring the given process and throttle as needed.
*/
constructor(private readonly _options: {
pid: number,
logMeta: log.ILogMeta,
timing?: ThrottleTiming
}) {
this._timing = this._options.timing || defaultThrottleTiming;
this._meteringInterval = setInterval(() => this._update(), this._timing.samplePeriodMs);
}
/**
* Stop all activity.
*/
public stop() {
this._stopped = true;
this._stopMetering();
this._stopThrottling();
}
/**
* Read the last cpu usage sample made, for test purposes.
*/
public get testStats(): MeterSample|undefined {
return this._sample;
}
/**
* Measure cpu usage and update whether and how much we are throttling the process.
*/
private async _update() {
// Measure cpu usage to date.
let cpuDuration: number;
try {
cpuDuration = (await pidusage(this._options.pid)).ctime;
} catch (e) {
// process may have disappeared.
log.rawDebug(`Throttle measurement error: ${e}`, this._options.logMeta);
return;
}
const now = Date.now();
const current: MeterSample = { time: now, cpuDuration, offDuration: this._offDuration };
this._sample = current;
// Measuring cpu usage was an async operation, so check that we haven't been stopped
// in the meantime. Otherwise we could sneak in and restart a throttle duty cycle.
if (this._stopped) { return; }
// We keep a reference point in the past called the "anchor". Whenever the anchor
// becomes sufficiently old, we replace it with something newer.
if (!this._anchor) { this._anchor = current; }
if (this._nextAnchor && now - this._anchor.time > this._timing.targetAveragingPeriodMs * 2) {
this._anchor = this._nextAnchor;
this._nextAnchor = undefined;
}
// Keep a replacement for the current anchor in mind.
if (!this._nextAnchor && now - this._anchor.time > this._timing.targetAveragingPeriodMs) {
this._nextAnchor = current;
}
// Check if the anchor is sufficiently old for averages to be meaningful enough
// to support throttling.
const dt = current.time - this._anchor.time;
if (dt < this._timing.minimumAveragingPeriodMs) { return; }
// Calculate the average cpu use per second since the anchor.
const rate = (current.cpuDuration - this._anchor.cpuDuration) / dt;
// If that rate is less than our target rate, don't bother throttling.
const targetRate = this._timing.targetRate;
if (rate <= targetRate) {
this._updateThrottle(0);
return;
}
// Calculate how much time the sandbox was paused since the anchor. This is
// approximate, since we don't line up duty cycles with this update function,
// but it should be good enough for throttling purposes.
const off = current.offDuration - this._anchor.offDuration;
// If the sandbox was never allowed to run, wait a bit longer for a duty cycle to complete.
// This should never happen unless time constants are set too tight relative to the
// maximum length of duty cycle.
const on = dt - off;
if (on <= 0) { return; }
// Calculate the average cpu use per second while the sandbox is unpaused.
const rateWithoutThrottling = (current.cpuDuration - this._anchor.cpuDuration) / on;
// Now pick a throttle level such that, if the sandbox continues using cpu
// at rateWithoutThrottling when it is unpaused, the overall rate matches
// the targetRate.
// one duty cycle lasts: quantum * (1 + throttleFactor)
// (positive cycle lasts 1 quantum; non-positive cycle duration is that of
// positive cycle scaled by throttleFactor)
// cpu use for this cycle is: quantum * rateWithoutThrottling
// cpu use per second is therefore: rateWithoutThrottling / (1 + throttleFactor)
// so: throttleFactor = (rateWithoutThrottling / targetRate) - 1
const throttleFactor = rateWithoutThrottling / targetRate - 1;
// Apply the throttle. Place a cap on it so the duty cycle does not get too long.
// This cap means that low targetRates could be unobtainable.
this._updateThrottle(Math.min(throttleFactor, this._timing.maxThrottle));
if (!this._lastLogTime || now - this._lastLogTime > this._timing.minimumLogPeriodMs) {
this._lastLogTime = now;
log.rawDebug('throttle', {...this._options.logMeta,
throttle: Math.round(this._throttleFactor),
throttledRate: Math.round(rate * 100),
rate: Math.round(rateWithoutThrottling * 100)});
}
}
/**
* Start/stop the throttling duty cycle as necessary.
*/
private _updateThrottle(factor: number) {
// For small factors, let the process run continuously.
if (factor < 0.001) {
if (this._dutyCycleTimeout) { this._stopThrottling(); }
this._throttleFactor = 0;
return;
}
// Set the throttle factor to apply and make sure the duty cycle is running.
this._throttleFactor = factor;
if (!this._dutyCycleTimeout) { this._throttle(true); }
}
/**
* Send CONTinue or STOP signal to process.
*/
private _letProcessRun(on: boolean) {
try {
process.kill(this._options.pid, on ? 'SIGCONT' : 'SIGSTOP');
} catch (e) {
// process may have disappeared
log.rawDebug(`Throttle error: ${e}`, this._options.logMeta);
}
}
/**
* Send CONTinue or STOP signal to process, and schedule next step
* in duty cycle.
*/
private _throttle(on: boolean) {
this._letProcessRun(on);
const dt = this._timing.dutyCyclePositiveMs * (on ? 1.0 : this._throttleFactor);
if (!on) { this._offDuration += dt; }
this._dutyCycleTimeout = setTimeout(() => this._throttle(!on), dt);
}
/**
* Make sure measurement of cpu is stopped.
*/
private _stopMetering() {
if (this._meteringInterval) {
clearInterval(this._meteringInterval);
this._meteringInterval = undefined;
}
}
/**
* Make sure duty cycle is stopped and process is left in running state.
*/
private _stopThrottling() {
if (this._dutyCycleTimeout) {
clearTimeout(this._dutyCycleTimeout);
this._dutyCycleTimeout = undefined;
this._letProcessRun(true);
}
}
}

164
app/server/lib/TimeQuery.ts Normal file
View File

@@ -0,0 +1,164 @@
import {ActionSummary, ColumnDelta, createEmptyActionSummary, createEmptyTableDelta} from 'app/common/ActionSummary';
import {CellDelta} from 'app/common/TabularDiff';
import {concatenateSummaries} from 'app/server/lib/ActionSummary';
import {ISQLiteDB, quoteIdent, ResultRow} from 'app/server/lib/SQLiteDB';
import keyBy = require('lodash/keyBy');
import matches = require('lodash/matches');
import sortBy = require('lodash/sortBy');
import toPairs = require('lodash/toPairs');
/**
* We can combine an ActionSummary with the current state of the database
* to answer questions about the state of the database in the past. This
* is particularly useful for grist metadata tables, which are needed to
* interpret the content of user tables fully.
* - TimeCursor is a simple container for the db and an ActionSummary
* - TimeQuery offers a db-like interface for a given table and set of columns
* - TimeLayout answers a couple of concrete questions about table meta-data using a
* set of TimeQuery objects hooked up to _grist_* tables. It could be used to
* improve the rendering of the ActionLog, for example, although it is not (yet).
*/
/** Track the state of the database at a particular time. */
export class TimeCursor {
public summary: ActionSummary;
constructor(public db: ISQLiteDB) {
this.summary = createEmptyActionSummary();
}
/** add a summary of an action just before the last action applied to the TimeCursor */
public prepend(prevSummary: ActionSummary) {
this.summary = concatenateSummaries([prevSummary, this.summary]);
}
/** add a summary of an action just after the last action applied to the TimeCursor */
public append(nextSummary: ActionSummary) {
this.summary = concatenateSummaries([this.summary, nextSummary]);
}
}
/** internal class for storing a ResultRow dictionary, keyed by rowId */
class ResultRows {
[rowId: number]: ResultRow;
}
/**
* Query the state of a particular table in the past, given a TimeCursor holding the
* current db and a summary of all changes between that past time and now.
* For the moment, for simplicity, names of tables and columns are assumed not to
* change, and TimeQuery should only be used for _grist_* tables.
*/
export class TimeQuery {
private _currentRows: ResultRow[];
private _pastRows: ResultRow[];
constructor(public tc: TimeCursor, public tableId: string, public colIds: string[]) {
}
/** Get fresh data from DB and overlay with any past data */
public async update(): Promise<ResultRow[]> {
this._currentRows = await this.tc.db.all(
`select ${['id', ...this.colIds].map(quoteIdent).join(',')} from ${quoteIdent(this.tableId)}`);
// Let's see everything the summary has accumulated about the table back then.
const td = this.tc.summary.tableDeltas[this.tableId] || createEmptyTableDelta();
// Now rewrite the summary as a ResultRow dictionary, to make it comparable
// with database.
const summaryRows: ResultRows = {};
for (const [colId, columns] of toPairs(td.columnDeltas)) {
for (const [rowId, cell] of toPairs(columns) as unknown as Array<[keyof ColumnDelta, CellDelta]>) {
if (!summaryRows[rowId]) { summaryRows[rowId] = {}; }
const val = cell[0];
summaryRows[rowId][colId] = (val !== null && typeof val === 'object' ) ? val[0] : null;
}
}
// Prepare to access the current database state by rowId.
const rowsById = keyBy(this._currentRows, r => (r.id as number));
// Prepare a list of rowIds at the time of interest.
// The past rows are whatever the db has now, omitting rows that were added
// since the past time, and adding back any rows that were removed since then.
// Careful about the order of this, since rows could be replaced.
const additions = new Set(td.addRows);
const pastRowIds =
new Set([...this._currentRows.map(r => r.id as number).filter(r => !additions.has(r)),
...td.removeRows]);
// Now prepare a row for every expected rowId, using current db data if available
// and relevant, and overlaying past data when available.
this._pastRows = new Array<ResultRow>();
const colIdsOfInterest = new Set(this.colIds);
for (const id of Array.from(pastRowIds).sort()) {
const row: ResultRow = rowsById[id] || {id};
if (summaryRows[id] && !additions.has(id)) {
for (const [colId, val] of toPairs(summaryRows[id])) {
if (colIdsOfInterest.has(colId)) { row[colId] = val; }
}
}
this._pastRows.push(row);
}
return this._pastRows;
}
/**
* Do a query with a single result, specifying any desired filters. Exception thrown
* if there is no result.
*/
public one(args: {[name: string]: any}): ResultRow {
const result = this._pastRows.find(matches(args));
if (!result) {
throw new Error(`could not find: ${JSON.stringify(args)} for ${this.tableId}`);
}
return result;
}
/** Get all results for a query. */
public all(args?: {[name: string]: any}): ResultRow[] {
if (!args) { return this._pastRows; }
return this._pastRows.filter(matches(args));
}
}
/**
* Put some TimeQuery queries to work answering questions about column order and
* user-facing name of tables.
*/
export class TimeLayout {
public tables: TimeQuery;
public fields: TimeQuery;
public columns: TimeQuery;
public views: TimeQuery;
constructor(public tc: TimeCursor) {
this.tables = new TimeQuery(tc, '_grist_Tables', ['tableId', 'primaryViewId']);
this.fields = new TimeQuery(tc, '_grist_Views_section_field',
['parentId', 'parentPos', 'colRef']);
this.columns = new TimeQuery(tc, '_grist_Tables_column', ['parentId', 'colId']);
this.views = new TimeQuery(tc, '_grist_Views', ['id', 'name']);
}
/** update from TimeCursor */
public async update() {
await this.tables.update();
await this.columns.update();
await this.fields.update();
await this.views.update();
}
public getColumnOrder(tableId: string): string[] {
const primaryViewId = this.tables.one({tableId}).primaryViewId;
const preorder = this.fields.all({parentId: primaryViewId});
const precol = keyBy(this.columns.all(), 'id');
const ordered = sortBy(preorder, 'parentPos');
const names = ordered.map(r => precol[r.colRef].colId);
return names;
}
public getTableName(tableId: string): string {
const primaryViewId = this.tables.one({tableId}).primaryViewId;
return this.views.one({id: primaryViewId}).name;
}
}

161
app/server/lib/UnsafeNodeComponent.ts Normal file
View File

@@ -0,0 +1,161 @@
import { ActionRouter } from 'app/common/ActionRouter';
import { LocalPlugin } from 'app/common/plugin';
import { BaseComponent, createRpcLogger, warnIfNotReady } from 'app/common/PluginInstance';
import { GristAPI, RPC_GRISTAPI_INTERFACE } from 'app/plugin/GristAPI';
import * as log from 'app/server/lib/log';
import { getAppPathTo } from 'app/server/lib/places';
import { makeLinePrefixer } from 'app/server/lib/sandboxUtil';
import { exitPromise, timeoutReached } from 'app/server/lib/serverUtils';
import { ChildProcess, fork, ForkOptions } from 'child_process';
import * as fse from 'fs-extra';
import { IMessage, IMsgCustom, IMsgRpcCall, Rpc } from 'grain-rpc';
import * as path from 'path';
// Error for not yet implemented api.
class NotImplemented extends Error {
constructor(name: string) {
super(`calling ${name} from UnsafeNode is not yet implemented`);
}
}
/**
* The unsafeNode component used by a PluginInstance.
*
*/
export class UnsafeNodeComponent extends BaseComponent {
private _child?: ChildProcess; /* plugin node code will run as separate process */
private _exited: Promise<void>; /* fulfulled when process has completed */
private _rpc: Rpc;
private _pluginPath: string;
private _pluginId: string;
private _actionRouter: ActionRouter;
private _gristAPI: GristAPI = {
render() { throw new NotImplemented('render'); },
dispose() { throw new NotImplemented('dispose'); },
subscribe: (tableId: string) => this._actionRouter.subscribeTable(tableId),
unsubscribe: (tableId: string) => this._actionRouter.unsubscribeTable(tableId),
};
/**
*
* @arg parent: the plugin instance this component is part of
* @arg _mainPath: main script file to run
* @arg appRoot: root path for application (important for setting a good NODE_PATH)
* @arg _gristDocPath: path to the current Grist doc (to which this plugin applies).
*
*/
constructor(plugin: LocalPlugin, pluginRpc: Rpc, private _mainPath: string, public appRoot: string,
private _gristDocPath: string,
rpcLogger = createRpcLogger(log, `PLUGIN ${plugin.id}/${_mainPath} UnsafeNode:`)) {
super(plugin.manifest, rpcLogger);
this._pluginPath = plugin.path;
this._pluginId = plugin.id;
this._rpc = new Rpc({
sendMessage: (msg) => this.sendMessage(msg),
logger: rpcLogger,
});
this._rpc.registerForwarder('*', pluginRpc);
this._rpc.registerImpl<GristAPI>(RPC_GRISTAPI_INTERFACE, this._gristAPI);
this._actionRouter = new ActionRouter(this._rpc);
}
public async sendMessage(data: IMessage): Promise<void> {
if (!this._child) {
await this.activateImplementation();
}
this._child!.send(data);
return Promise.resolve();
}
public receiveAction(action: any[]) {
this._actionRouter.process(action)
.catch((err: any) => log.warn('unsafeNode[%s] receiveAction failed with %s',
this._child ? this._child.pid : "NULL", err));
}
/**
*
* Create the child node process needed for this component.
*
*/
protected async activateImplementation(): Promise<void> {
log.info(`unsafeNode operating in ${this._pluginPath}`);
const base = this._pluginPath;
const script = path.resolve(base, this._mainPath);
await fse.access(script, fse.constants.R_OK);
// Time to set up the node search path the client will see.
// We take our own, via Module.globalPaths, a poorly documented
// method listing the search path for the active node program
// https://github.com/nodejs/node/blob/master/test/parallel/test-module-globalpaths-nodepath.js
const paths = require('module').globalPaths.slice().concat([
// add the path to the plugin itself
path.resolve(base),
// add the path to grist's public api
getAppPathTo(this.appRoot, 'public-api'),
// add the path to the node_modules packaged with grist, in electron form
getAppPathTo(this.appRoot, 'node_modules')
]);
const env = Object.assign({}, process.env, {
NODE_PATH: paths.join(path.delimiter),
GRIST_PLUGIN_PATH: `${this._pluginId}/${this._mainPath}`,
GRIST_DOC_PATH: this._gristDocPath,
});
const electronVersion: string = (process.versions as any).electron;
if (electronVersion) {
// Pass along the fact that we are running under an electron-ified node, for the purposes of
// finding binaries (sqlite3 in particular).
env.ELECTRON_VERSION = electronVersion;
}
const child = this._child = fork(script, [], {
env,
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
} as ForkOptions); // Explicit cast only because node-6 typings mistakenly omit stdio property
log.info("unsafeNode[%s] started %s", child.pid, script);
// Important to use exitPromise() before events from child may be received, so don't call
// yield or await between fork and here.
this._exited = exitPromise(child)
.then(code => log.info("unsafeNode[%s] exited with %s", child.pid, code))
.catch(err => log.warn("unsafeNode[%s] failed with %s", child.pid, err))
.then(() => { this._child = undefined; });
child.stdout.on('data', makeLinePrefixer('PLUGIN stdout: '));
child.stderr.on('data', makeLinePrefixer('PLUGIN stderr: '));
warnIfNotReady(this._rpc, 3000, "Plugin isn't ready; be sure to call grist.ready() from plugin");
child.on('message', this._rpc.receiveMessage.bind(this._rpc));
}
/**
*
* Remove the child node process needed for this component.
*
*/
protected async deactivateImplementation(): Promise<void> {
if (!this._child) {
log.info('unsafeNode deactivating: no child process');
} else {
log.info('unsafeNode[%s] deactivate: disconnecting child', this._child.pid);
this._child.disconnect();
if (await timeoutReached(2000, this._exited)) {
log.info("unsafeNode[%s] deactivate: sending SIGTERM", this._child.pid);
this._child.kill('SIGTERM');
}
if (await timeoutReached(5000, this._exited)) {
log.warn("unsafeNode[%s] deactivate: child still has not exited", this._child.pid);
} else {
log.info("unsafeNode deactivate: child exited");
}
}
}
protected doForwardCall(c: IMsgRpcCall): Promise<any> {
return this._rpc.forwardCall({...c, mdest: ''});
}
protected async doForwardMessage(c: IMsgCustom): Promise<any> {
return this._rpc.forwardMessage({...c, mdest: ''});
}
}

66
app/server/lib/WorkCoordinator.ts Normal file
View File

@@ -0,0 +1,66 @@
import * as log from 'app/server/lib/log';
/**
* WorkCoordinator is a helper to do work serially. It takes a doWork() callback which may either
* do some work and return a Promise, or report no work to be done by returning null. After work
* completes, doWork() will be called again; when idle, ping() should be called to retry doWork().
*/
export class WorkCoordinator {
private _doWorkCB: () => Promise<void>|null;
private _tryNextStepCB: () => void;
private _isStepRunning: boolean = false;
private _isStepScheduled: boolean = false;
/**
* The doWork() callback will be called on ping() and whenever previous doWork() promise
* succeeds. If doWork() had nothing to do, it should return null, and will not be called again
* until the next ping().
*
* Note that doWork() should never fail. If it does, exceptions and rejections will be caught
* and logged, and WorkCoordinator will not be called again until the next ping().
*/
constructor(doWork: () => Promise<void>|null) {
this._doWorkCB = doWork;
this._tryNextStepCB = () => this._tryNextStep(); // bound version of _tryNextStep.
}
/**
* Attempt doWork() again. If doWork() is currently running, it will be attempted again on
* completion even if the current run fails.
*/
public ping(): void {
if (!this._isStepScheduled) {
this._isStepScheduled = true;
this._maybeSchedule();
}
}
private async _tryNextStep(): Promise<void> {
this._isStepScheduled = false;
if (!this._isStepRunning) {
this._isStepRunning = true;
try {
const work = this._doWorkCB();
if (work) {
await work;
// Only schedule the next step if some work was done. If _doWorkCB() did nothing, or
// failed, _doWorkCB() will only be called when an external ping() triggers it.
this._isStepScheduled = true;
}
} catch (err) {
// doWork() should NOT fail. If it does, we log the error here, and stop scheduling work
// as if there is no more work to be done.
log.error("WorkCoordinator: error in doWork()", err);
} finally {
this._isStepRunning = false;
this._maybeSchedule();
}
}
}
private _maybeSchedule() {
if (this._isStepScheduled && !this._isStepRunning) {
setImmediate(this._tryNextStepCB);
}
}
}

21
app/server/lib/checksumFile.ts Normal file
View File

@@ -0,0 +1,21 @@
import {createHash} from 'crypto';
import * as fs from 'fs';
/**
* Computes hash of the file at the given path, using 'sha1' by default, or any algorithm
* supported by crypto.createHash().
*/
export async function checksumFile(filePath: string, algorithm: string = 'sha1'): Promise<string> {
const shaSum = createHash(algorithm);
const stream = fs.createReadStream(filePath);
try {
stream.on('data', (data) => shaSum.update(data));
await new Promise<void>((resolve, reject) => {
stream.on('end', resolve);
stream.on('error', reject);
});
return shaSum.digest('hex');
} finally {
stream.removeAllListeners(); // Isn't strictly necessary.
}
}

67
app/server/lib/dbUtils.ts Normal file
View File

@@ -0,0 +1,67 @@
import {synchronizeProducts} from 'app/gen-server/entity/Product';
import {Connection, createConnection} from 'typeorm';
// Summary of migrations found in database and in code.
interface MigrationSummary {
migrationsInDb: string[];
migrationsInCode: string[];
pendingMigrations: string[];
}
// Find the migrations in the database, the migrations in the codebase, and compare the two.
export async function getMigrations(connection: Connection): Promise<MigrationSummary> {
let migrationsInDb: string[];
try {
migrationsInDb = (await connection.query('select name from migrations')).map((rec: any) => rec.name);
} catch (e) {
// If no migrations have run, there'll be no migrations table - which is fine,
// it just means 0 migrations run yet. Sqlite+Postgres report this differently,
// so any query error that mentions the name of our table is treated as ok.
// Everything else is unexpected.
if (!(e.name === 'QueryFailedError' && e.message.includes('migrations'))) {
throw e;
}
migrationsInDb = [];
}
// get the migration names in codebase.
// They are a bit hidden, see typeorm/src/migration/MigrationExecutor::getMigrations
const migrationsInCode: string[] = connection.migrations.map(m => (m.constructor as any).name);
const pendingMigrations = migrationsInCode.filter(m => !migrationsInDb.includes(m));
return {
migrationsInDb,
migrationsInCode,
pendingMigrations,
};
}
/**
* Run any needed migrations, and make sure products are up to date.
*/
export async function updateDb(connection?: Connection) {
connection = connection || await createConnection();
await runMigrations(connection);
await synchronizeProducts(connection, true);
}
export async function runMigrations(connection: Connection) {
// on SQLite, migrations fail if we don't temporarily disable foreign key
// constraint checking. This is because for sqlite typeorm copies each
// table and rebuilds it from scratch for each schema change.
// Also, we need to disable foreign key constraint checking outside of any
// transaction, or it has no effect.
const sqlite = connection.driver.options.type === 'sqlite';
if (sqlite) { await connection.query("PRAGMA foreign_keys = OFF;"); }
await connection.transaction(async tr => {
await tr.connection.runMigrations();
});
if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); }
}
export async function undoLastMigration(connection: Connection) {
const sqlite = connection.driver.options.type === 'sqlite';
if (sqlite) { await connection.query("PRAGMA foreign_keys = OFF;"); }
await connection.transaction(async tr => {
await tr.connection.undoLastMigration();
});
if (sqlite) { await connection.query("PRAGMA foreign_keys = ON;"); }
}

9
app/server/lib/docUtils.d.ts vendored Normal file
View File

@@ -0,0 +1,9 @@
export function makeIdentifier(name: string): string;
export function copyFile(src: string, dest: string): Promise<void>;
export function createNumbered(name: string, separator: string, creator: (path: string) => Promise<void>,
startNum?: number): Promise<string>;
export function createNumberedTemplate(template: string, creator: (path: string) => Promise<void>): Promise<string>;
export function createExclusive(path: string): Promise<void>;
export function realPath(path: string): Promise<string>;
export function pathExists(path: string): Promise<boolean>;
export function isSameFile(path1: string, path2: string): Promise<boolean>;

145
app/server/lib/docUtils.js Normal file
View File

@@ -0,0 +1,145 @@
/**
* Functions generally useful when dealing with Grist documents.
*/
var fs = require('fs');
var fsPath = require('path');
var Promise = require('bluebird');
Promise.promisifyAll(fs);
var nonIdentRegex = /[^\w_]+/g;
/**
* Given a string, converts it to a Grist identifier. Identifiers consist of lowercase
* alphanumeric characters and the underscore.
* @param {String} name The name to convert.
* @returns {String} Identifier.
*/
function makeIdentifier(name) {
// Lowercase and replace consecutive invalid characters with underscores.
return name.toLowerCase().replace(nonIdentRegex, '_');
}
exports.makeIdentifier = makeIdentifier;
/**
* Copies a file, returning a promise that is resolved (with no value) when the copy is complete.
* TODO This needs a unittest.
*/
function copyFile(sourcePath, destPath) {
var sourceStream, destStream;
return new Promise(function(resolve, reject) {
sourceStream = fs.createReadStream(sourcePath);
destStream = fs.createWriteStream(destPath);
sourceStream.on('error', reject);
destStream.on('error', reject);
destStream.on('finish', resolve);
sourceStream.pipe(destStream);
})
.finally(function() {
if (destStream) { destStream.destroy(); }
if (sourceStream) { sourceStream.destroy(); }
});
}
exports.copyFile = copyFile;
/**
* Helper for creating numbered files. Tries to call creator() with name, then (name + separator +
* "2") and so on with incrementing numbers, as long as the promise returned by creator() is
* rejected with err.code of 'EEXIST'. Creator() must return a promise.
* @param {String} name The first name to try.
* @param {String} separator The separator between name and appended numbers.
* @param {Function} creator The function to call with successive names. Must return a promise.
* @param {Number} startNum Optional number to start with; omit to try an unnumbered name first.
* @returns {Promise} Promise for the first name for which creator() succeeded.
*/
function createNumbered(name, separator, creator, startNum) {
var fullName = name + (startNum === undefined ? '' : separator + startNum);
var nextNum = (startNum === undefined ? 2 : startNum + 1);
return creator(fullName)
.then(() => fullName)
.catch(function(err) {
if (err.cause && err.cause.code !== 'EEXIST')
throw err;
return createNumbered(name, separator, creator, nextNum);
});
}
exports.createNumbered = createNumbered;
/**
* An easier-to-use alternative to createNumbered. Pass in a template string containing the
* special token "{NUM}". It will first call creator() with "{NUM}" removed, then with "{NUM}"
* replcaced by "-2", "-3", etc, until creator() succeeds, and will return the value for which it
* suceeded.
*/
function createNumberedTemplate(template, creator) {
const [prefix, suffix] = template.split("{NUM}");
if (typeof prefix !== "string" || typeof suffix !== "string") {
throw new Error(`createNumberedTemplate: invalid template ${template}`);
}
return createNumbered(prefix, "-", (uniqPrefix) => creator(uniqPrefix + suffix))
.then((uniqPrefix) => uniqPrefix + suffix);
}
exports.createNumberedTemplate = createNumberedTemplate;
/**
* Creates a new file, failing if the path already exists.
* @param {String} path: The path to try creating.
* @returns {Promise} Resolved if the path was created, rejected if it already existed (with
* err.cause.code === EEXIST) or if there was another error creating it.
*/
function createExclusive(path) {
return fs.openAsync(path, 'wx').then(fd => fs.closeAsync(fd));
}
exports.createExclusive = createExclusive;
/**
* Returns the canonicalized absolute path for the given path, using fs.realpath, but allowing
* non-existent paths. In case of non-existent path, the longest existing prefix is resolved and
* the rest kept unchanged.
* @param {String} path: Path to resolve.
* @return {Promise:String} Promise for the resolved path.
*/
function realPath(path) {
return fs.realpathAsync(path)
.catch(() =>
realPath(fsPath.dirname(path))
.then(dir => fsPath.join(dir, fsPath.basename(path)))
);
}
exports.realPath = realPath;
/**
* Returns a promise that resolves to true or false based on whether the path exists. If other
* errors occur, this promise may still be rejected.
*/
function pathExists(path) {
return fs.accessAsync(path)
.then(() => true)
.catch({code: 'ENOENT'}, () => false)
.catch({code: 'ENOTDIR'}, () => false);
}
exports.pathExists = pathExists;
/**
* Returns a promise that resolves to true or false based on whether the two paths point to the
* same file. If errors occur, this promise may be rejected.
*/
function isSameFile(path1, path2) {
return Promise.join(fs.lstatAsync(path1), fs.lstatAsync(path2), (stat1, stat2) => {
if (stat1.dev === stat2.dev && stat1.ino === stat2.ino) {
return true;
}
return false;
})
.catch({code: 'ENOENT'}, () => false)
.catch({code: 'ENOTDIR'}, () => false);
}
exports.isSameFile = isSameFile;

36
app/server/lib/expressWrap.ts Normal file
View File

@@ -0,0 +1,36 @@
import {RequestWithLogin} from 'app/server/lib/Authorizer';
import * as log from 'app/server/lib/log';
import * as express from 'express';
/**
* Wrapper for async express endpoints to catch errors and forward them to the error handler.
*/
export function expressWrap(callback: express.RequestHandler): express.RequestHandler {
return async (req, res, next) => {
try {
await callback(req, res, next);
} catch (err) {
next(err);
}
};
}
/**
* Error-handling middleware that responds to errors in json. The status code is taken from
* error.status property (for which ApiError is convenient), and defaults to 500.
*/
export const jsonErrorHandler: express.ErrorRequestHandler = (err, req, res, next) => {
const mreq = req as RequestWithLogin;
log.warn("Error during api call to %s: (%s) user %d params %s body %s", req.path, err.message,
mreq.userId,
JSON.stringify(req.params), JSON.stringify(req.body));
res.status(err.status || 500).json({error: err.message || 'internal error',
details: err.details});
};
/**
* Middleware that responds with a 404 status and a json error object.
*/
export const jsonNotFoundHandler: express.RequestHandler = (req, res, next) => {
res.status(404).json({error: `not found: ${req.url}`});
};

166
app/server/lib/extractOrg.ts Normal file
View File

@@ -0,0 +1,166 @@
import { ApiError } from 'app/common/ApiError';
import { mapGetOrSet, MapWithTTL } from 'app/common/AsyncCreate';
import { extractOrgParts, getKnownOrg, isCustomHost } from 'app/common/gristUrls';
import { Organization } from 'app/gen-server/entity/Organization';
import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
import { NextFunction, Request, RequestHandler, Response } from 'express';
import { IncomingMessage } from 'http';
// How long we cache information about the relationship between
// orgs and custom hosts. The higher this is, the fewer requests
// to the DB needed, but the longer it will take for changes
// to custom host setting to take effect. Also, since the caching
// is done on individual servers/workers, it could be inconsistent
// between servers/workers for some time. During this period,
// redirect cycles are possible.
// Units are milliseconds.
const ORG_HOST_CACHE_TTL = 60 * 1000;
export interface RequestOrgInfo {
org: string;
isCustomHost: boolean; // when set, the request's domain is a recognized custom host linked
// with the specified org.
// path remainder after stripping /o/{org} if any.
url: string;
}
export type RequestWithOrg = Request & Partial<RequestOrgInfo>;
/**
* Manage the relationship between orgs and custom hosts in the url.
*/
export class Hosts {
// Cache of orgs (e.g. "fancy" of "fancy.getgrist.com") associated with custom hosts
// (e.g. "www.fancypants.com")
private _host2org = new MapWithTTL<string, Promise<string|undefined>>(ORG_HOST_CACHE_TTL);
// Cache of custom hosts associated with orgs.
private _org2host = new MapWithTTL<string, Promise<string|undefined>>(ORG_HOST_CACHE_TTL);
// baseDomain should start with ".". It may be undefined for localhost or single-org mode.
constructor(private _baseDomain: string|undefined, private _dbManager: HomeDBManager) {
}
/**
* Use app.use(hosts.extractOrg) to set req.org, req.isCustomHost, and to strip
* /o/ORG/ from urls (when present).
*
* If Host header has a getgrist.com subdomain, then it must match the value in /o/ORG (when
* present), and req.org will be set to the subdomain. On mismatch, a 400 response is returned.
*
* If Host header is a localhost domain, then req.org is set to the value in /o/ORG when
* present, and to "" otherwise.
*
* If Host header is something else, we query the db for an org whose host value matches.
* If found, req.org is set appropriately, and req.isCustomHost is set to true.
* If not found, a 'Domain not recognized' error is thrown, showing an error page.
*/
public get extractOrg(): RequestHandler {
return this._extractOrg.bind(this);
}
// Extract org info in a request. This applies to the low-level IncomingMessage type (rather
// than express.Request that derives from it) to be usable with websocket requests too.
public async getOrgInfo(req: IncomingMessage): Promise<RequestOrgInfo> {
const host = req.headers.host || '';
const hostname = host.split(':')[0]; // Strip out port (ignores IPv6 but is OK for us).
const info = await this.getOrgInfoFromParts(hostname, req.url!);
// "Organization" header is used in proxying to doc worker, so respect it if
// no org info found in url.
if (!info.org && req.headers.organization) {
info.org = req.headers.organization as string;
}
return info;
}
// Extract org, isCustomHost, and the URL with /o/ORG stripped away. Throws ApiError for
// mismatching org or invalid custom domain. Hostname should not include port.
public async getOrgInfoFromParts(hostname: string, urlPath: string): Promise<RequestOrgInfo> {
// Extract the org from the host and URL path.
const parts = extractOrgParts(hostname, urlPath);
// If the server is configured to serve a single hard-wired org, respect that.
const singleOrg = getKnownOrg();
if (singleOrg) {
return {org: singleOrg, url: parts.pathRemainder, isCustomHost: false};
}
// Fake the protocol; it doesn't matter for parsing out the hostname.
if (this._isNativeDomain(hostname)) {
if (parts.mismatch) {
throw new ApiError(`Wrong org for this domain: ` +
`'${parts.orgFromPath}' does not match '${parts.orgFromHost}'`, 400);
}
return {org: parts.subdomain || '', url: parts.pathRemainder, isCustomHost: false};
} else {
// Otherwise check for a custom host.
const org = await mapGetOrSet(this._host2org, hostname, async () => {
const o = await this._dbManager.connection.manager.findOne(Organization, {host: hostname});
return o && o.domain || undefined;
});
if (!org) { throw new ApiError(`Domain not recognized: ${hostname}`, 404); }
// Strip any stray /o/.... that has been added to a url with a custom host.
// TODO: it would eventually be cleaner to make sure we don't make those
// additions in the first place.
// To check for mismatch, compare to org, since orgFromHost is not expected to match.
if (parts.orgFromPath && parts.orgFromPath !== org) {
throw new ApiError(`Wrong org for this domain: ` +
`'${parts.orgFromPath}' does not match '${org}'`, 400);
}
return {org, isCustomHost: true, url: parts.pathRemainder};
}
}
public async addOrgInfo(req: Request): Promise<RequestWithOrg> {
return Object.assign(req, await this.getOrgInfo(req));
}
/**
* Use app.use(hosts.redirectHost) to ensure (by redirecting if necessary)
* that the domain in the url matches the preferred domain for the current org.
* Expects that the extractOrg has been used first.
*/
public get redirectHost(): RequestHandler {
return this._redirectHost.bind(this);
}
public close() {
this._host2org.clear();
this._org2host.clear();
}
private async _extractOrg(req: Request, resp: Response, next: NextFunction) {
try {
await this.addOrgInfo(req);
return next();
} catch (err) {
return resp.status(err.status || 500).send({error: err.message});
}
}
private async _redirectHost(req: Request, resp: Response, next: NextFunction) {
const {org} = req as RequestWithOrg;
if (org && this._isNativeDomain(req.hostname) && !this._dbManager.isMergedOrg(org)) {
// Check if the org has a preferred host.
const orgHost = await mapGetOrSet(this._org2host, org, async () => {
const o = await this._dbManager.connection.manager.findOne(Organization, {domain: org});
return o && o.host || undefined;
});
if (orgHost && orgHost !== req.hostname) {
const url = new URL(`${req.protocol}://${req.get('host')}${req.path}`);
url.hostname = orgHost; // assigning hostname rather than host preserves port.
return resp.redirect(url.href);
}
}
return next();
}
private _isNativeDomain(hostname: string) {
return !this._baseDomain || !isCustomHost(hostname, this._baseDomain);
}
}

138
app/server/lib/gristSessions.ts Normal file
View File

@@ -0,0 +1,138 @@
import * as session from '@gristlabs/express-session';
import {parseSubdomain} from 'app/common/gristUrls';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {GristServer} from 'app/server/lib/GristServer';
import {Sessions} from 'app/server/lib/Sessions';
import {promisifyAll} from 'bluebird';
import * as express from 'express';
import * as path from 'path';
import * as shortUUID from "short-uuid";
export const cookieName = process.env.GRIST_SESSION_COOKIE || 'grist_sid';
export const COOKIE_MAX_AGE = 90 * 24 * 60 * 60 * 1000; // 90 days in milliseconds
// RedisStore and SqliteStore are expected to provide a set/get interface for sessions.
export interface SessionStore {
getAsync(sid: string): Promise<any>;
setAsync(sid: string, session: any): Promise<void>;
}
/**
*
* A V1 session. A session can be associated with a number of users.
* There may be a preferred association between users and organizations:
* specifically, if from the url we can tell that we are showing material
* for a given organization, we should pick a user that has access to that
* organization.
*
* This interface plays no role at all yet! Working on refactoring existing
* sessions step by step to get closer to this.
*
*/
export interface IGristSession {
// V1 Hosted Grist - known available users.
users: Array<{
userId?: number;
}>;
// V1 Hosted Grist - known user/org relationships.
orgs: Array<{
orgId: number;
userId: number;
}>;
}
function createSessionStoreFactory(sessionsDB: string): () => SessionStore {
if (process.env.REDIS_URL) {
// Note that ./build excludes this module from the electron build.
const RedisStore = require('connect-redis')(session);
promisifyAll(RedisStore.prototype);
return () => new RedisStore({
url: process.env.REDIS_URL,
});
} else {
const SQLiteStore = require('@gristlabs/connect-sqlite3')(session);
promisifyAll(SQLiteStore.prototype);
return () => new SQLiteStore({
dir: path.dirname(sessionsDB),
db: path.basename(sessionsDB), // SQLiteStore no longer appends a .db suffix.
table: 'sessions'
});
}
}
export function getAllowedOrgForSessionID(sessionID: string): {org: string, host: string}|null {
if (sessionID.startsWith('c-') && sessionID.includes('@')) {
const [, org, host] = sessionID.split('@');
if (!host) { throw new Error('Invalid session ID'); }
return {org, host};
}
// Otherwise sessions start with 'g-', but we also accept older sessions without a prefix.
return null;
}
/**
* Set up Grist Sessions, either in a sqlite db or via redis.
* @param instanceRoot: path to storage area in case we need to make a sqlite db.
*/
export function initGristSessions(instanceRoot: string, server: GristServer) {
// TODO: We may need to evaluate the usage of space in the SQLite store grist-sessions.db
// since entries are created on the first get request.
const sessionsDB: string = path.join(instanceRoot, 'grist-sessions.db');
// The extra step with the creator function is used in server.js to create a new session store
// after unpausing the server.
const sessionStoreCreator = createSessionStoreFactory(sessionsDB);
const sessionStore = sessionStoreCreator();
const adaptDomain = process.env.GRIST_ADAPT_DOMAIN === 'true';
const fixedDomain = process.env.GRIST_SESSION_DOMAIN || process.env.GRIST_DOMAIN;
const getCookieDomain = (req: express.Request) => {
const mreq = req as RequestWithOrg;
if (mreq.isCustomHost) {
// For custom hosts, omit the domain to make it a "host-only" cookie, to avoid it being
// included into subdomain requests (since we would not control all the subdomains).
return undefined;
}
if (adaptDomain) {
const reqDomain = parseSubdomain(req.get('host'));
if (reqDomain.base) { return reqDomain.base.split(':')[0]; }
}
return fixedDomain;
};
// Use a separate session IDs for custom domains than for native ones. Because a custom domain
// cookie could be stolen (with some effort) by the custom domain's owner, we limit the damage
// by only honoring custom-domain cookies for requests to that domain.
const generateId = (req: RequestWithOrg) => {
const uid = shortUUID.generate();
return req.isCustomHost ? `c-${uid}@${req.org}@${req.get('host')}` : `g-${uid}`;
};
const sessionSecret = server.create.sessionSecret();
const sessionMiddleware = session({
secret: sessionSecret,
resave: false,
saveUninitialized: false,
name: cookieName,
requestDomain: getCookieDomain,
genid: generateId,
cookie: {
// We do not initially set max-age, leaving the cookie as a
// session cookie until there's a successful login. On the
// redis back-end, the session associated with the cookie will
// persist for 24 hours if there is no successful login. Once
// there is a successful login, max-age will be set to
// COOKIE_MAX_AGE, making the cookie a persistent cookie. The
// session associated with the cookie will receive an updated
// time-to-live, so that it persists for COOKIE_MAX_AGE.
},
store: sessionStore
});
const sessions = new Sessions(sessionSecret, sessionStore, server);
return {sessions, sessionSecret, sessionStore, sessionMiddleware, sessionStoreCreator};
}

51
app/server/lib/guessExt.ts Normal file
View File

@@ -0,0 +1,51 @@
import {fromFile} from 'file-type';
import {extension, lookup} from 'mime-types';
import * as path from 'path';
/**
* Get our best guess of the file extension, based on its original extension (as received from the
* user), mimeType (as reported by the browser upload, or perhaps some API), and the file
* contents.
*
* The resulting extension is used to choose a parser for imports, and to present the file back
* to the user for attachments.
*/
export async function guessExt(filePath: string, fileName: string, mimeType: string|null): Promise<string> {
const origExt = path.extname(fileName).toLowerCase(); // Has the form ".xls"
let mimeExt = extension(mimeType); // Has the form "xls"
mimeExt = mimeExt ? "." + mimeExt : null; // Use the more comparable form ".xls"
if (mimeExt === ".json") {
// It's common for JSON APIs to specify MIME type, but origExt might come from a URL with
// periods that don't indicate a meaningful extension. Trust mime-type here.
return mimeExt;
}
if (origExt === ".csv" || origExt === ".xls") {
// File type detection doesn't work for these, and mime type can't be trusted. E.g. Windows
// may report "application/vnd.ms-excel" for .csv files. See
// https://github.com/ManifoldScholar/manifold/issues/2409#issuecomment-545152220
return origExt;
}
// If extension and mime type agree, let's call it a day.
if (origExt && (origExt === mimeExt || lookup(origExt.slice(1)) === mimeType)) {
return origExt;
}
// If not, let's take a look at the file contents.
const detected = await fromFile(filePath);
const detectedExt = detected ? "." + detected.ext : null;
if (detectedExt) {
// For the types for which detection works, we think we should prefer it.
return detectedExt;
}
if (mimeExt === '.txt' || mimeExt === '.bin') {
// text/plain (txt) and application/octet-stream (bin) are too generic, only use them if we
// don't have anything better.
return origExt || mimeExt;
}
// In other cases, it's a tough call.
return origExt || mimeExt;
}

48
app/server/lib/idUtils.ts Normal file
View File

@@ -0,0 +1,48 @@
import {ForkResult} from 'app/common/ActiveDocAPI';
import {buildUrlId, parseUrlId} from 'app/common/gristUrls';
import {padStart} from 'app/common/gutil';
import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
import * as shortUUID from 'short-uuid';
// make an id that is a standard UUID compressed into fewer characters.
export function makeId(): string {
// Generate a flickr-style id, by converting a regular uuid interpreted
// as a hex number (without dashes) into a number expressed in a bigger
// base. That number is encoded as characters chosen for url safety and
// lack of confusability. The character encoding zero is '1'. We pad the
// result so that the length of the id remains consistent, since there is
// routing that depends on the id length exceeding a minimum threshold.
return padStart(shortUUID.generate(), 22, '1');
}
/**
* Construct an id for a fork, given the userId, whether the user is the anonymous user,
* and the id of a reference document (the trunk).
* If the userId is null, the user will be treated as the anonymous user.
*/
export function makeForkIds(options: { userId: number|null, isAnonymous: boolean,
trunkDocId: string, trunkUrlId: string }): ForkResult {
const forkId = makeId();
const forkUserId = options.isAnonymous ? undefined :
(options.userId !== null ? options.userId : undefined);
// TODO: we will want to support forks of forks, but for now we do not -
// forks are always forks of the trunk.
const docId = parseUrlId(options.trunkDocId).trunkId;
const urlId = parseUrlId(options.trunkUrlId).trunkId;
return {
docId: buildUrlId({trunkId: docId, forkId, forkUserId}),
urlId: buildUrlId({trunkId: urlId, forkId, forkUserId}),
};
}
// For importing, we can assign any worker to the job. As a hack, we reuse the document
// assignment mechanism. To spread the work around a bit if we have several doc workers,
// we use a fake document id between import0 and import9.
// This method takes a DocWorkerMap to allow for something smarter in future.
export function getAssignmentId(docWorkerMap: IDocWorkerMap, docId: string): string {
let assignmentId = docId;
if (assignmentId === 'import') {
assignmentId = `import${Math.round(Math.random() * 10)}`;
}
return assignmentId;
}

89
app/server/lib/log.ts Normal file
View File

@@ -0,0 +1,89 @@
/**
* Configures grist logging. This is merely a customization of the 'winston' logging module,
* and all winston methods are available. Additionally provides log.timestamp() function.
* Usage:
* var log = require('./lib/log');
* log.info(...);
*/
import {timeFormat} from 'app/common/timeFormat';
import * as winston from 'winston';
interface LogWithTimestamp extends winston.LoggerInstance {
timestamp(): string;
// We'd like to log raw json, for convenience of parsing downstream.
// We have a customization that interferes with meta arguments, and
// existing log messages that depend on that customization. For
// clarity then, we just add "raw" flavors of the primary level
// methods that pass their object argument through to winston.
rawError(msg: string, meta: ILogMeta): void;
rawInfo(msg: string, meta: ILogMeta): void;
rawWarn(msg: string, meta: ILogMeta): void;
rawDebug(msg: string, meta: ILogMeta): void;
origLog(level: string, msg: string, ...args: any[]): void;
}
/**
* Hack winston to provide a saner behavior with regard to its optional arguments. Winston allows
* two optional arguments at the end: "meta" (if object) and "callback" (if function). We don't
* use them, but we do use variable number of arguments as in log.info("foo %s", foo). If foo is
* an object, winston dumps it in an ugly way, not at all as intended. We fix by always appending
* {} to the end of the arguments, so that winston sees an empty meta object.
* We can add support for callback if ever needed.
*/
const origLog = winston.Logger.prototype.log;
winston.Logger.prototype.log = function(level: string, msg: string, ...args: any[]) {
return origLog.call(this, level, msg, ...args, {});
};
const rawLog = new (winston.Logger)();
const log: LogWithTimestamp = Object.assign(rawLog, {
timestamp,
/**
* Versions of log.info etc that take a meta parameter. For
* winston, logs are streams of info objects. Info objects
* have two mandatory fields, level and message. They can
* have other fields, called "meta" fields. When logging
* in json, those fields are added directly to the json,
* rather than stringified into the message field, which
* is what we want and why we are adding these variants.
*/
rawError: (msg: string, meta: ILogMeta) => origLog.call(log, 'error', msg, meta),
rawInfo: (msg: string, meta: ILogMeta) => origLog.call(log, 'info', msg, meta),
rawWarn: (msg: string, meta: ILogMeta) => origLog.call(log, 'warn', msg, meta),
rawDebug: (msg: string, meta: ILogMeta) => origLog.call(log, 'debug', msg, meta),
origLog,
});
/**
* Returns the current timestamp as a string in the same format as used in logging.
*/
function timestamp() {
return timeFormat("A", new Date());
}
const fileTransportOptions = {
stream: process.stderr,
level: 'debug',
timestamp: log.timestamp,
colorize: true,
json: process.env.GRIST_HOSTED_VERSION ? true : false
};
// Configure logging to use console and simple timestamps.
log.add(winston.transports.File, fileTransportOptions);
// Also update the default logger to use the same format.
winston.remove(winston.transports.Console);
winston.add(winston.transports.File, fileTransportOptions);
// It's a little tricky to export a type when the top-level export is an object.
// tslint:disable-next-line:no-namespace
declare namespace log { // eslint-disable-line @typescript-eslint/no-namespace
interface ILogMeta {
[key: string]: any;
}
}
type ILogMeta = log.ILogMeta;
export = log;

79
app/server/lib/manifest.ts Normal file
View File

@@ -0,0 +1,79 @@
import {BarePlugin} from 'app/plugin/PluginManifest';
import PluginManifestTI from 'app/plugin/PluginManifest-ti';
import * as fse from 'fs-extra';
import * as yaml from 'js-yaml';
import * as path from 'path';
import {createCheckers} from "ts-interface-checker";
const manifestChecker = createCheckers(PluginManifestTI).BarePlugin;
/**
* Validate the manifest and generate appropriate errors.
*/
// TODO: should validate that the resources referenced within the manifest are located within the
// plugin folder
// TODO: Need a comprehensive test that triggers every notices;
function isValidManifest(manifest: any, notices: string[]): boolean {
if (!manifest) {
notices.push("missing manifest");
return false;
}
try {
manifestChecker.check(manifest);
} catch (e) {
notices.push(`Invalid manifest: ${e.message}`);
return false;
}
try {
manifestChecker.strictCheck(manifest);
} catch (e) {
notices.push(`WARNING: ${e.message}` );
/* but don't fail */
}
if (Object.keys(manifest.contributions).length === 0) {
notices.push("WARNING: no valid contributions");
}
return true;
}
/**
* A ManifestError is an error caused by a wrongly formatted manifest or missing manifest. The
* `notices` property holds a user-friendly description of the error(s).
*/
export class ManifestError extends Error {
constructor(public notices: string[], message: string = "") {
super(message);
}
}
/**
* Parse the manifest. Look first for a Yaml manifest and then if missing for a Json manifest.
*/
export async function readManifest(pluginPath: string): Promise<BarePlugin> {
const notices: string[] = [];
const manifest = await _readManifest(pluginPath);
if (isValidManifest(manifest, notices)) {
return manifest as BarePlugin;
}
throw new ManifestError(notices);
}
async function _readManifest(pluginPath: string): Promise<object> {
try {
return yaml.safeLoad(await readManifestFile("yml"));
} catch (e) {
if (e instanceof yaml.YAMLException) {
throw new Error('error parsing yaml manifest: ' + e.message);
}
}
try {
return JSON.parse(await readManifestFile("json"));
} catch (e) {
if (e instanceof SyntaxError) {
throw new Error('error parsing json manifest' + e.message);
}
throw new Error('cannot read manifest file: ' + e.message);
}
async function readManifestFile(fileExtension: string): Promise<string> {
return await fse.readFile(path.join(pluginPath, "manifest." + fileExtension), "utf8");
}
}

46
app/server/lib/places.ts Normal file
View File

@@ -0,0 +1,46 @@
/**
* Utilities related to the layout of the application and where parts are stored.
*/
import * as path from 'path';
/**
* codeRoot is the directory containing ./app with all the JS code.
*/
export const codeRoot = path.dirname(path.dirname(path.dirname(__dirname)));
/**
* Returns the appRoot, i.e. the directory containing ./sandbox, ./node_modules, ./ormconfig.js,
* etc.
*/
export function getAppRoot(): string {
if (codeRoot.endsWith('/_build/core')) { return path.dirname(path.dirname(codeRoot)); }
return codeRoot.endsWith('/_build') ? path.dirname(codeRoot) : codeRoot;
}
/**
* When packaged as an electron application, most files are stored in a .asar
* archive. Most, but not all. This method takes the "application root"
* which is that .asar file in packaged form, and returns a directory where
* remaining files are available on the regular filesystem.
*/
export function getUnpackedAppRoot(appRoot: string): string {
return path.resolve(path.dirname(appRoot), path.basename(appRoot, '.asar'));
}
/**
* Return the correct root for a given subdirectory.
*/
export function getAppRootFor(appRoot: string, subdirectory: string): string {
if (['sandbox', 'plugins', 'public-api'].includes(subdirectory)) {
return getUnpackedAppRoot(appRoot);
}
return appRoot;
}
/**
* Return the path to a given subdirectory, from the correct appRoot.
*/
export function getAppPathTo(appRoot: string, subdirectory: string): string {
return path.resolve(getAppRootFor(appRoot, subdirectory), subdirectory);
}

204
app/server/lib/requestUtils.ts Normal file
View File

@@ -0,0 +1,204 @@
import {ApiError} from 'app/common/ApiError';
import {DEFAULT_HOME_SUBDOMAIN, parseSubdomain} from 'app/common/gristUrls';
import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import * as log from 'app/server/lib/log';
import {Request, Response} from 'express';
import {URL} from 'url';
// log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)
const shouldLogApiDetails = Boolean(process.env.GRIST_HOSTED_VERSION);
// Offset to https ports in dev/testing environment.
export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
parseInt(process.env.GRIST_TEST_HTTPS_OFFSET, 10) : undefined;
// Database fields that we permit in entities but don't want to cross the api.
const INTERNAL_FIELDS = new Set(['apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId',
'stripeCustomerId', 'stripeSubscriptionId', 'stripePlanId',
'stripeProductId', 'userId', 'isFirstTimeUser']);
/**
* Adapt a home-server or doc-worker URL to match the hostname in the request URL. For custom
* domains and when GRIST_SERVE_SAME_ORIGIN is set, we replace the full hostname; otherwise just
* the base of the hostname. The changes to url are made in-place.
*
* For dev purposes, port is kept but possibly adjusted for TEST_HTTPS_OFFSET. Note that if port
* is different from req's port, it is not considered same-origin for CORS purposes, but would
* still receive cookies.
*/
export function adaptServerUrl(url: URL, req: RequestWithOrg): void {
const reqBaseDomain = parseSubdomain(req.hostname).base;
if (process.env.GRIST_SERVE_SAME_ORIGIN === 'true' || req.isCustomHost) {
url.hostname = req.hostname;
} else if (reqBaseDomain) {
const subdomain: string|undefined = parseSubdomain(url.hostname).org || DEFAULT_HOME_SUBDOMAIN;
url.hostname = `${subdomain}${reqBaseDomain}`;
}
// In dev/test environment we can turn on a flag to adjust URLs to use https.
if (TEST_HTTPS_OFFSET && url.port && url.protocol === 'http:') {
url.port = String(parseInt(url.port, 10) + TEST_HTTPS_OFFSET);
url.protocol = 'https:';
}
}
/**
* Returns true for requests from permitted origins. For such requests, an
* "Access-Control-Allow-Origin" header is added to the response. Vary: Origin
* is also set to reflect the fact that the headers are a function of the origin,
* to prevent inappropriate caching on the browser's side.
*/
export function trustOrigin(req: Request, resp: Response): boolean {
// TODO: We may want to consider changing allowed origin values in the future.
// Note that the request origin is undefined for non-CORS requests.
const origin = req.get('origin');
if (!origin) { return true; } // Not a CORS request.
if (!allowHost(req, new URL(origin))) { return false; }
// For a request to a custom domain, the full hostname must match.
resp.header("Access-Control-Allow-Origin", origin);
resp.header("Vary", "Origin");
return true;
}
// Returns whether req satisfies the given allowedHost. Unless req is to a custom domain, it is
// enough if only the base domains match. Differing ports are allowed, which helps in dev/testing.
export function allowHost(req: Request, allowedHost: string|URL) {
const mreq = req as RequestWithOrg;
const proto = req.protocol;
const actualUrl = new URL(`${proto}://${req.get('host')}`);
const allowedUrl = (typeof allowedHost === 'string') ? new URL(`${proto}://${allowedHost}`) : allowedHost;
if (mreq.isCustomHost) {
// For a request to a custom domain, the full hostname must match.
return actualUrl.hostname === allowedUrl.hostname;
} else {
// For requests to a native subdomains, only the base domain needs to match.
const allowedDomain = parseSubdomain(allowedUrl.hostname);
const actualDomain = parseSubdomain(actualUrl.hostname);
return (actualDomain.base === allowedDomain.base);
}
}
export function isParameterOn(parameter: any): boolean {
return ['1', 'on', 'true'].includes(String(parameter).toLowerCase());
}
/**
* Get Scope from request, and make sure it has everything needed for a document.
*/
export function getDocScope(req: Request): DocScope {
const scope = getScope(req);
if (!scope.urlId) { throw new Error('document required'); }
return scope as DocScope;
}
/**
* Extract information included in the request that may restrict the scope of
* that request. Not all requests will support all restrictions.
*
* - userId - Mandatory. Produced by authentication middleware.
* Information returned and actions taken will be limited by what
* that user has access to.
*
* - org - Optional. Extracted by middleware. Limits
* information/action to the given org. Not every endpoint
* respects this limit. Possible exceptions include endpoints for
* listing orgs a user has access to, and endpoints with an org id
* encoded in them.
*
* - urlId - Optional. Embedded as "did" (or "docId") path parameter in endpoints related
* to documents. Specifies which document the request pertains to. Can
* be a urlId or a docId.
*
* - includeSupport - Optional. Embedded as "includeSupport" query parameter.
* Just a few endpoints support this, it is a very specific "hack" for including
* an example workspace in org listings.
*
* - showRemoved - Optional. Embedded as "showRemoved" query parameter.
* Supported by many endpoints. When absent, request is limited
* to docs/workspaces that have not been removed. When present, request
* is limited to docs/workspaces that have been removed.
*/
export function getScope(req: Request): Scope {
const urlId = req.params.did || req.params.docId;
const userId = getUserId(req);
const org = (req as RequestWithOrg).org;
const {specialPermit} = (req as RequestWithLogin);
const includeSupport = isParameterOn(req.query.includeSupport);
const showRemoved = isParameterOn(req.query.showRemoved);
return {urlId, userId, org, includeSupport, showRemoved, specialPermit};
}
// Return a JSON response reflecting the output of a query.
// Filter out keys we don't want crossing the api.
// Set req to null to not log any information about request.
export async function sendReply<T>(req: Request|null, res: Response, result: QueryResult<T>) {
const data = pruneAPIResult(result.data || null);
if (shouldLogApiDetails && req) {
const mreq = req as RequestWithLogin;
log.rawDebug('api call', {
url: req.url,
userId: mreq.userId,
email: mreq.user && mreq.user.loginEmail,
org: mreq.org,
params: req.params,
body: req.body,
result: data,
});
}
if (result.status === 200) {
return res.json(data);
} else {
return res.status(result.status).json({error: result.errMessage});
}
}
export async function sendOkReply<T>(req: Request|null, res: Response, result?: T) {
return sendReply(req, res, {status: 200, data: result});
}
export function pruneAPIResult<T>(data: T): T {
// TODO: This can be optimized by pruning data recursively without serializing in between. But
// it's fairly fast even with serializing (on the order of 15usec/kb).
const output = JSON.stringify(data,
(key: string, value: any) => {
// Do not include removedAt field if it is not set. It is not relevant to regular
// situations where the user is working with non-deleted resources.
if (key === 'removedAt' && value === null) { return undefined; }
return INTERNAL_FIELDS.has(key) ? undefined : value;
});
return JSON.parse(output);
}
/**
* Access the canonical docId associated with the request. Must have already authorized.
*/
export function getDocId(req: Request) {
const mreq = req as RequestWithLogin;
// We should always have authorized by now.
if (!mreq.docAuth || !mreq.docAuth.docId) { throw new ApiError(`unknown document`, 500); }
return mreq.docAuth.docId;
}
export function optStringParam(p: any): string|undefined {
if (typeof p === 'string') { return p; }
return undefined;
}
export function stringParam(p: any): string {
if (typeof p === 'string') { return p; }
throw new Error(`parameter should be a string: ${p}`);
}
export function integerParam(p: any): number {
if (typeof p === 'number') { return Math.floor(p); }
if (typeof p === 'string') { return parseInt(p, 10); }
throw new Error(`parameter should be an integer: ${p}`);
}
export interface RequestWithGristInfo extends Request {
gristInfo?: string;
}

58
app/server/lib/sandboxUtil.js Normal file
View File

@@ -0,0 +1,58 @@
/**
* Various utilities and constants for communicating with the python sandbox.
*/
var MemBuffer = require('app/common/MemBuffer');
var log = require('./log');
/**
* SandboxError is an error type for reporting errors forwarded from the sandbox.
*/
function SandboxError(message) {
// Poorly documented node feature, required to make the derived error keep a proper stack trace.
Error.captureStackTrace(this, this.constructor);
this.name = 'SandboxError';
this.message = "[Sandbox] " + (message || 'Python reported an error');
}
SandboxError.prototype = new Error();
// We need to set the .constructor property for Error.captureStackTrace to work correctly.
SandboxError.prototype.constructor = SandboxError;
exports.SandboxError = SandboxError;
/**
* Special msgCode values that precede msgBody to indicate what kind of message it is.
* These all cost one byte. If we needed more, we should probably switch to a number (5 bytes)
* CALL = call to the other side. The data must be an array of [func_name, arguments...]
* DATA = data must be a value to return to a call from the other side
* EXC = data must be an exception to return to a call from the other side
*/
exports.CALL = null;
exports.DATA = true;
exports.EXC = false;
/**
* Returns a function that takes data buffers and logs them to log.info() with the given prefix.
* The logged output is line-oriented, so that the prefix is only inserted at the start of a line.
* Binary data is encoded as with JSON.stringify.
*/
function makeLinePrefixer(prefix, logMeta) {
var partial = '';
return data => {
partial += MemBuffer.arrayToString(data);
var newline;
while ((newline = partial.indexOf("\n")) !== -1) {
var line = partial.slice(0, newline);
partial = partial.slice(newline + 1);
// Escape some parts of the string by serializing it to JSON (without the quotes).
log.rawInfo(prefix + JSON.stringify(line).slice(1, -1).replace(/\\"/g, '"')
.replace(/\\\\/g, '\\'),
logMeta);
}
};
}
exports.makeLinePrefixer = makeLinePrefixer;

105
app/server/lib/sendAppPage.ts Normal file
View File

@@ -0,0 +1,105 @@
import {GristLoadConfig} from 'app/common/gristUrls';
import {isAnonymousUser} from 'app/server/lib/Authorizer';
import {RequestWithOrg} from 'app/server/lib/extractOrg';
import {GristServer} from 'app/server/lib/GristServer';
import * as express from 'express';
import * as fse from 'fs-extra';
import * as path from 'path';
export interface ISendAppPageOptions {
path: string; // Ignored if .content is present (set to "" for clarity).
content?: string;
status: number;
config: Partial<GristLoadConfig>;
tag?: string; // If present, override version tag.
// If present, enable Google Tag Manager on this page (if GOOGLE_TAG_MANAGER_ID env var is set).
// Used on the welcome page to track sign-ups. We don't intend to use it for in-app analytics.
// Set to true to insert tracker unconditionally; false to omit it; "anon" to insert
// it only when the user is not logged in.
googleTagManager?: true | false | 'anon';
}
export function makeGristConfig(homeUrl: string|null, extra: Partial<GristLoadConfig>,
baseDomain?: string, req?: express.Request
): GristLoadConfig {
// .invalid is a TLD the IETF promises will never exist.
const pluginUrl = process.env.APP_UNTRUSTED_URL || 'plugins.invalid';
const pathOnly = (process.env.GRIST_ORG_IN_PATH === "true") ||
(homeUrl && new URL(homeUrl).hostname === 'localhost') || false;
const mreq = req as RequestWithOrg|undefined;
return {
homeUrl,
org: process.env.GRIST_SINGLE_ORG || (mreq && mreq.org),
baseDomain,
singleOrg: process.env.GRIST_SINGLE_ORG,
pathOnly,
supportAnon: shouldSupportAnon(),
pluginUrl,
stripeAPIKey: process.env.STRIPE_PUBLIC_API_KEY,
helpScoutBeaconId: process.env.HELP_SCOUT_BEACON_ID,
maxUploadSizeImport: (Number(process.env.GRIST_MAX_UPLOAD_IMPORT_MB) * 1024 * 1024) || undefined,
maxUploadSizeAttachment: (Number(process.env.GRIST_MAX_UPLOAD_ATTACHMENT_MB) * 1024 * 1024) || undefined,
timestampMs: Date.now(),
...extra,
};
}
/**
* Send a simple template page, read from file at pagePath (relative to static/), with certain
* placeholders replaced.
*/
export function makeSendAppPage(opts: {
server: GristServer|null, staticDir: string, tag: string, testLogin?: boolean,
baseDomain?: string
}) {
const {server, staticDir, tag, testLogin} = opts;
return async (req: express.Request, resp: express.Response, options: ISendAppPageOptions) => {
// .invalid is a TLD the IETF promises will never exist.
const config = makeGristConfig(server ? server.getHomeUrl(req) : null, options.config,
opts.baseDomain, req);
// We could cache file contents in memory, but the filesystem does caching too, and compared
// to that, the performance gain is unlikely to be meaningful. So keep it simple here.
const fileContent = options.content || await fse.readFile(path.join(staticDir, options.path), 'utf8');
const needTagManager = (options.googleTagManager === 'anon' && isAnonymousUser(req)) ||
options.googleTagManager === true;
const tagManagerSnippet = needTagManager ? getTagManagerSnippet() : '';
const staticOrigin = process.env.APP_STATIC_URL || "";
const staticBaseUrl = `${staticOrigin}/v/${options.tag || tag}/`;
const warning = testLogin ? "<div class=\"dev_warning\">Authentication is not enforced</div>" : "";
const content = fileContent
.replace("<!-- INSERT WARNING -->", warning)
.replace("<!-- INSERT BASE -->", `<base href="${staticBaseUrl}">` + tagManagerSnippet)
.replace("<!-- INSERT CONFIG -->", `<script>window.gristConfig = ${JSON.stringify(config)};</script>`);
resp.status(options.status).type('html').send(content);
};
}
function shouldSupportAnon() {
// Enable UI for anonymous access if a flag is explicitly set in the environment
return process.env.GRIST_SUPPORT_ANON === "true";
}
/**
* Returns the Google Tag Manager snippet to insert into <head> of the page, if
* GOOGLE_TAG_MANAGER_ID env var is set to a non-empty value. Otherwise returns the empty string.
*/
function getTagManagerSnippet() {
// Note also that we only insert the snippet for the <head>. The second recommended part (for
// <body>) is for <noscript> scenario, which doesn't apply to the Grist app (such visits, if
// any, wouldn't work and shouldn't be counted for any metrics we care about).
const tagId = process.env.GOOGLE_TAG_MANAGER_ID;
if (!tagId) { return ""; }
return `
<!-- Google Tag Manager -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
})(window,document,'script','dataLayer','${tagId}');</script>
<!-- End Google Tag Manager -->
`;
}

116
app/server/lib/serverUtils.ts Normal file
View File

@@ -0,0 +1,116 @@
import * as bluebird from 'bluebird';
import {ChildProcess} from 'child_process';
import * as net from 'net';
import * as path from 'path';
import {ConnectionOptions} from 'typeorm';
/**
* Promisify a node-style callback function. E.g.
* fromCallback(cb => someAsyncFunc(someArgs, cb));
* This is merely a type-checked version of bluebird.fromCallback().
* (Note that providing it using native Promises is also easy, but bluebird's big benefit is
* support of long stack traces (when enabled for debugging).
*/
type NodeCallback<T> = (err: Error|undefined|null, value?: T) => void;
type NodeCallbackFunc<T> = (cb: NodeCallback<T>) => void;
const _fromCallback = bluebird.fromCallback;
export function fromCallback<T>(nodeFunc: NodeCallbackFunc<T>): Promise<T> {
return _fromCallback(nodeFunc);
}
/**
* Finds and returns a promise for the first available TCP port.
* @param {Number} firstPort: First port number to check, defaults to 8000.
* @param {Number} optCount: Number of ports to check, defaults to 200.
* @returns Promise<Number>: Promise for an available port.
*/
export function getAvailablePort(firstPort: number = 8000, optCount: number = 200) {
const lastPort = firstPort + optCount - 1;
function checkNext(port: number): Promise<number> {
if (port > lastPort) {
throw new Error("No available ports between " + firstPort + " and " + lastPort);
}
return new bluebird((resolve: (p: number) => void, reject: (e: Error) => void) => {
const server = net.createServer();
server.on('error', reject);
server.on('close', () => resolve(port));
server.listen(port, 'localhost', () => server.close());
})
.catch(() => checkNext(port + 1));
}
return bluebird.try(() => checkNext(firstPort));
}
/**
* Promisified version of net.connect(). Takes the same arguments, and returns a Promise for the
* connected socket. (Types are specified as in @types/node.)
*/
export function connect(options: { port: number, host?: string, localAddress?: string, localPort?: string,
family?: number, allowHalfOpen?: boolean; }): Promise<net.Socket>;
export function connect(port: number, host?: string): Promise<net.Socket>;
export function connect(path: string): Promise<net.Socket>; // tslint:disable-line:unified-signatures
export function connect(arg: any, ...moreArgs: any[]): Promise<net.Socket> {
return new Promise((resolve, reject) => {
const s = net.connect(arg, ...moreArgs, () => resolve(s));
s.on('error', reject);
});
}
/**
* Returns whether the path `inner` is contained within the directory `outer`.
*/
export function isPathWithin(outer: string, inner: string): boolean {
const rel = path.relative(outer, inner);
const index = rel.indexOf(path.sep);
const firstDir = index < 0 ? rel : rel.slice(0, index);
return firstDir !== "..";
}
/**
* Returns a promise that's resolved when `child` exits, or rejected if it could not be started.
* The promise resolves to the numeric exit code, or the string signal that terminated the child.
*
* Note that this must be called synchronously after creating the ChildProcess, since a delay may
* cause the 'error' or 'exit' message from the child to be missed, and the resulting exitPromise
* would then hang forever.
*/
export function exitPromise(child: ChildProcess): Promise<number|string> {
return new Promise((resolve, reject) => {
// Called if process could not be spawned, or could not be killed(!), or sending a message failed.
child.on('error', reject);
child.on('exit', (code: number, signal: string) => resolve(signal || code));
});
}
/**
* Resolves to true if promise is still pending after msec milliseconds have passed. Otherwise
* returns false, including when promise is rejected.
*/
export function timeoutReached<T>(msec: number, promise: Promise<T>): Promise<boolean> {
const timedOut = {};
// Be careful to clean up the timer after ourselves, so it doesn't remain in the event loop.
let timer: NodeJS.Timer;
const delayPromise = new Promise<any>((resolve) => (timer = setTimeout(() => resolve(timedOut), msec)));
return Promise.race([promise, delayPromise])
.then((res) => { clearTimeout(timer); return res === timedOut; })
.catch(() => false);
}
/**
* Get database url in DATABASE_URL format popularized by heroku, suitable for
* use by psql, sqlalchemy, etc.
*/
export function getDatabaseUrl(options: ConnectionOptions, includeCredentials: boolean): string {
if (options.type === 'sqlite') {
return `sqlite://${options.database}`;
} else if (options.type === 'postgres') {
const pass = options.password ? `:${options.password}` : '';
const creds = includeCredentials && options.username ? `${options.username}${pass}@` : '';
const port = options.port ? `:${options.port}` : '';
return `postgres://${creds}${options.host}${port}/${options.database}`;
} else {
return `${options.type}://?`;
}
}

60
app/server/lib/shortDesc.ts Normal file
View File

@@ -0,0 +1,60 @@
import defaults = require('lodash/defaults');
import identity = require('lodash/identity');
import {inspect} from 'util';
function truncateString(s: string|Uint8Array, maxLen: number, optStringMapper?: (arg: any) => string): string {
const m: (arg: any) => string = optStringMapper || identity;
return s.length <= maxLen ? m(s) : m(s.slice(0, maxLen)) + "... (" + s.length + " length)";
}
function formatUint8Array(array: Uint8Array): string {
const s = Buffer.from(array).toString('binary');
// eslint-disable-next-line no-control-regex
return s.replace(/[\x00-\x1f\x7f-\xff]/g, '?');
}
interface DescLimits {
maxArrayLength: number;
maxObjectKeys: number;
maxStringLength: number;
maxBufferLength: number;
}
const defaultLimits: DescLimits = {
maxArrayLength: 5,
maxObjectKeys: 10,
maxStringLength: 80,
maxBufferLength: 80,
};
/**
* Produce a human-readable concise description of the object as a string. Similar to
* util.inspect(), but more concise and more readable.
* @param {Object} optLimits: Optional limits on how much of a value to include. Supports
* maxArrayLength, maxObjectKeys, maxStringLength, maxBufferLength.
*/
export function shortDesc(topObj: any, optLimits?: DescLimits): string {
const lim = defaults(optLimits || {}, defaultLimits);
function _shortDesc(obj: any): string {
if (Array.isArray(obj)) {
return "[" +
obj.slice(0, lim.maxArrayLength).map(_shortDesc).join(", ") +
(obj.length > lim.maxArrayLength ? ", ... (" + obj.length + " items)" : "") +
"]";
} else if (obj instanceof Uint8Array) {
return "b'" + truncateString(obj, lim.maxBufferLength, formatUint8Array) + "'";
} else if (obj && typeof obj === 'object' && !Buffer.isBuffer(obj)) {
const keys = Object.keys(obj);
return "{" + keys.slice(0, lim.maxObjectKeys).map(function(key) {
return key + ": " + _shortDesc(obj[key]);
}).join(", ") +
(keys.length > lim.maxObjectKeys ? ", ... (" + keys.length + " keys)" : "") +
"}";
} else if (typeof obj === 'string') {
return inspect(truncateString(obj, lim.maxStringLength));
} else {
return inspect(obj);
}
}
return _shortDesc(topObj);
}

116
app/server/lib/shutdown.js Normal file
View File

@@ -0,0 +1,116 @@
/**
* Module for managing graceful shutdown.
*/
var Promise = require('bluebird');
var log = require('./log');
var cleanupHandlers = [];
var signalsHandled = {};
/**
* Adds a handler that should be run on shutdown.
* @param {Object} context The context with which to run the method, and which can be used to
* remove cleanup handlers.
* @param {Function} method The method to run. It may return a promise, which will be waited on.
* @param {Number} timeout Timeout in ms, for how long to wait for the returned promise. Required
* because it's no good for a cleanup handler to block the shutdown process indefinitely.
* @param {String} name A title to show in log messages to distinguish one handler from another.
*/
function addCleanupHandler(context, method, timeout = 1000, name = 'unknown') {
cleanupHandlers.push({
context,
method,
timeout,
name
});
}
exports.addCleanupHandler = addCleanupHandler;
/**
* Removes all cleanup handlers with the given context.
* @param {Object} context The context with which once or more cleanup handlers were added.
*/
function removeCleanupHandlers(context) {
// Maybe there should be gutil.removeFilter(func) which does this in-place.
cleanupHandlers = cleanupHandlers.filter(function(handler) {
return handler.context !== context;
});
}
exports.removeCleanupHandlers = removeCleanupHandlers;
var _cleanupHandlersPromise = null;
/**
* Internal helper which runs all cleanup handlers, with the right contexts and timeouts,
* waits for them, and reports and swallows any errors. It returns a promise that should always be
* fulfilled.
*/
function runCleanupHandlers() {
if (!_cleanupHandlersPromise) {
// Switch out cleanupHandlers, to leave an empty array at the end.
var handlers = cleanupHandlers;
cleanupHandlers = [];
_cleanupHandlersPromise = Promise.all(handlers.map(function(handler) {
return Promise.try(handler.method.bind(handler.context)).timeout(handler.timeout)
.catch(function(err) {
log.warn(`Cleanup error for '${handler.name}' handler: ` + err);
});
}));
}
return _cleanupHandlersPromise;
}
/**
* Internal helper to exit on a signal. It runs the cleanup handlers, and then re-sends the same
* signal, which will no longer get caught.
*/
function signalExit(signal) {
var prog = 'grist[' + process.pid + ']';
log.info("Server %s got signal %s; cleaning up (%d handlers)",
prog, signal, cleanupHandlers.length);
function dup() {
log.info("Server %s ignoring duplicate signal %s", prog, signal);
}
process.on(signal, dup);
return runCleanupHandlers()
.finally(function() {
log.info("Server %s exiting on %s", prog, signal);
process.removeListener(signal, dup);
delete signalsHandled[signal];
process.kill(process.pid, signal);
});
}
/**
* For the given signals, run cleanup handlers (which may be asynchronous) before re-sending the
* signals to the process. This should only be used for signals that normally kill the process.
* E.g. cleanupOnSignals('SIGINT', 'SIGTERM', 'SIGUSR2');
*/
function cleanupOnSignals(varSignalNames) {
for (var i = 0; i < arguments.length; i++) {
var signal = arguments[i];
if (signalsHandled[signal]) { continue; }
signalsHandled[signal] = true;
process.once(signal, signalExit.bind(null, signal));
}
}
exports.cleanupOnSignals = cleanupOnSignals;
/**
* Run cleanup handlers and exit the process with the given exit code (0 if omitted).
*/
function exit(optExitCode) {
var prog = 'grist[' + process.pid + ']';
var code = optExitCode || 0;
log.info("Server %s cleaning up", prog);
return runCleanupHandlers()
.finally(function() {
log.info("Server %s exiting with code %s", prog, code);
process.exit(code);
});
}
exports.exit = exit;

425
app/server/lib/uploads.ts Normal file
View File

@@ -0,0 +1,425 @@
import {ApiError} from 'app/common/ApiError';
import {InactivityTimer} from 'app/common/InactivityTimer';
import {FileUploadResult, UPLOAD_URL_PATH, UploadResult} from 'app/common/uploads';
import {getAuthorizedUserId, getTransitiveHeaders, getUserId, isSingleUserMode,
RequestWithLogin} from 'app/server/lib/Authorizer';
import {expressWrap} from 'app/server/lib/expressWrap';
import {RequestWithGrist} from 'app/server/lib/FlexServer';
import {GristServer} from 'app/server/lib/GristServer';
import {guessExt} from 'app/server/lib/guessExt';
import * as log from 'app/server/lib/log';
import {optStringParam, trustOrigin} from 'app/server/lib/requestUtils';
import {isPathWithin} from 'app/server/lib/serverUtils';
import * as shutdown from 'app/server/lib/shutdown';
import {fromCallback} from 'bluebird';
import * as contentDisposition from 'content-disposition';
import {Application, Request, RequestHandler, Response} from 'express';
import * as fse from 'fs-extra';
import pick = require('lodash/pick');
import * as multiparty from 'multiparty';
import fetch, {Response as FetchResponse} from 'node-fetch';
import * as path from 'path';
import * as tmp from 'tmp';
// After some time of inactivity, clean up the upload. We give an hour, which seems generous,
// except that if one is toying with import options, and leaves the upload in an open browser idle
// for an hour, it will get cleaned up. TODO Address that; perhaps just with some UI messages.
const INACTIVITY_CLEANUP_MS = 60 * 60 * 1000; // an hour, very generously.
// A hook for dependency injection.
export const Deps = {fetch, INACTIVITY_CLEANUP_MS};
// An optional UploadResult, with parameters.
export interface FormResult {
upload?: UploadResult;
parameters?: {[key: string]: string};
}
/**
* Adds an upload route to the given express app, listening for POST requests at UPLOAD_URL_PATH.
*/
export function addUploadRoute(server: GristServer, expressApp: Application, ...handlers: RequestHandler[]): void {
// When doing a cross-origin post, the browser will check for access with options prior to posting.
// We need to reassure it that the request will be accepted before it will go ahead and post.
expressApp.options([`/${UPLOAD_URL_PATH}`, '/copy'], async (req: Request, res: Response) => {
if (!trustOrigin(req, res)) { return res.status(500).send(); }
res.header("Access-Control-Allow-Credentials", "true");
res.status(200).send();
});
expressApp.post(`/${UPLOAD_URL_PATH}`, ...handlers, expressWrap(async (req: Request, res: Response) => {
try {
if (!trustOrigin(req, res)) {
throw new Error('Unrecognized origin');
}
res.header("Access-Control-Allow-Credentials", "true");
const uploadResult: UploadResult = await handleUpload(req, res);
res.status(200).send(JSON.stringify(uploadResult));
} catch (err) {
req.resume();
log.error("Error uploading file", err);
// Respond with a JSON error like jsonErrorHandler does for API calls,
// to make it easier for the caller to parse it.
res.status(err.status || 500).json({error: err.message || 'internal error'});
}
}));
// Like upload, but copy data from a document already known to us.
expressApp.post(`/copy`, ...handlers, expressWrap(async (req: Request, res: Response) => {
if (!trustOrigin(req, res)) {
throw new Error('Unrecognized origin');
}
res.header("Access-Control-Allow-Credentials", "true");
const docId = optStringParam(req.query.doc);
const name = optStringParam(req.query.name);
if (!docId) { throw new Error('doc must be specified'); }
const accessId = makeAccessId(req, getAuthorizedUserId(req));
const uploadResult: UploadResult = await fetchDoc(server.getHomeUrl(req), docId, req, accessId,
req.query.template === '1');
if (name) {
globalUploadSet.changeUploadName(uploadResult.uploadId, accessId, name);
}
res.status(200).send(JSON.stringify(uploadResult));
}));
}
/**
* Create a FileUploadInfo for the given file.
*/
export async function getFileUploadInfo(filePath: string): Promise<FileUploadInfo> {
return {
absPath: filePath,
origName: path.basename(filePath),
size: (await fse.stat(filePath)).size,
ext: path.extname(filePath).toLowerCase(),
};
}
/**
* Implementation of the express /upload route.
*/
export async function handleUpload(req: Request, res: Response): Promise<UploadResult> {
const {upload} = await handleOptionalUpload(req, res);
if (!upload) { throw new ApiError('missing payload', 400); }
return upload;
}
/**
* Process form data that may contain an upload, returning that upload (if present)
* and any parameters.
*/
export async function handleOptionalUpload(req: Request, res: Response): Promise<FormResult> {
const {tmpDir, cleanupCallback} = await createTmpDir({});
const mreq = req as RequestWithLogin;
const meta = {
org: mreq.org,
email: mreq.user && mreq.user.loginEmail,
userId: mreq.userId,
};
log.rawDebug(`Prepared to receive upload into tmp dir ${tmpDir}`, meta);
// Note that we don't limit upload sizes here, since this endpoint doesn't know what kind of
// upload it is, and some uploads are unlimited (e.g. uploading .grist files). Limits are
// checked in the client, and should be enforced on the server where an upload is processed.
const form = new multiparty.Form({uploadDir: tmpDir});
const [formFields, formFiles] = await fromCallback((cb: any) => form.parse(req, cb),
{multiArgs: true});
// 'upload' is the name of the form field containing file data.
let upload: UploadResult|undefined;
if (formFiles.upload) {
const uploadedFiles: FileUploadInfo[] = [];
for (const file of formFiles.upload) {
const mimeType = file.headers['content-type'];
log.rawDebug(`Received file ${file.originalFilename} (${file.size} bytes)`, meta);
uploadedFiles.push({
absPath: file.path,
origName: file.originalFilename,
size: file.size,
ext: await guessExt(file.path, file.originalFilename, mimeType),
});
}
const accessId = makeAccessId(req, getUserId(req));
const uploadId = globalUploadSet.registerUpload(uploadedFiles, tmpDir, cleanupCallback, accessId);
const files: FileUploadResult[] = uploadedFiles.map(f => pick(f, ['origName', 'size', 'ext']));
log.rawDebug(`Created uploadId ${uploadId} in tmp dir ${tmpDir}`, meta);
upload = {uploadId, files};
}
const parameters: {[key: string]: string} = {};
for (const key of Object.keys(formFields)) {
parameters[key] = formFields[key][0];
}
return {upload, parameters};
}
/**
* Represents a single uploaded file on the server side. Only the FileUploadResult part is exposed
* to the browser for information purposes.
*/
export interface FileUploadInfo extends FileUploadResult {
absPath: string; // Absolute path to the file on disk.
}
/**
* Represents a complete upload on the server side. It may be a temporary directory containing a
* list of files (not subdirectories), or a collection of non-temporary files. The
* cleanupCallback() is responsible for removing the temporary directory. It should be a no-op for
* non-temporary files.
*/
export interface UploadInfo {
uploadId: number; // ID of the upload
files: FileUploadInfo[]; // List of all files included in the upload.
tmpDir: string|null; // Temporary directory to remove, containing this upload.
// If present, all files must be direct children of this directory.
cleanupCallback: CleanupCB; // Callback to clean up this upload, including removing tmpDir.
cleanupTimer: InactivityTimer;
accessId: string|null; // Optional identifier for access control purposes.
}
type CleanupCB = () => void|Promise<void>;
export class UploadSet {
private _uploads: Map<number, UploadInfo> = new Map();
private _nextId: number = 0;
/**
* Register a new upload.
*/
public registerUpload(files: FileUploadInfo[], tmpDir: string|null, cleanupCallback: CleanupCB,
accessId: string|null): number {
const uploadId = this._nextId++;
const cleanupTimer = new InactivityTimer(() => this.cleanup(uploadId), Deps.INACTIVITY_CLEANUP_MS);
this._uploads.set(uploadId, {uploadId, files, tmpDir, cleanupCallback, cleanupTimer, accessId});
cleanupTimer.ping();
return uploadId;
}
/**
* Returns full info for the given uploadId, if authorized.
*/
public getUploadInfo(uploadId: number, accessId: string|null): UploadInfo {
const info = this._getUploadInfoWithoutAuthorization(uploadId);
if (info.accessId !== accessId) {
throw new ApiError('access denied', 403);
}
return info;
}
/**
* Clean up a particular upload.
*/
public async cleanup(uploadId: number): Promise<void> {
log.debug("UploadSet: cleaning up uploadId %s", uploadId);
const info = this._getUploadInfoWithoutAuthorization(uploadId);
info.cleanupTimer.disable();
this._uploads.delete(uploadId);
await info.cleanupCallback();
}
/**
* Clean up all uploads in this UploadSet. It may be used again after this call (it's called
* multiple times in tests).
*/
public async cleanupAll(): Promise<void> {
log.info("UploadSet: cleaning up all %d uploads in set", this._uploads.size);
const uploads = Array.from(this._uploads.values());
this._uploads.clear();
this._nextId = 0;
for (const info of uploads) {
try {
info.cleanupTimer.disable();
await info.cleanupCallback();
} catch (err) {
log.warn(`Error cleaning upload ${info.uploadId}: ${err}`);
}
}
}
/**
* Changes the name of an uploaded file. It is an error to use if the upload set has more than one
* file and it will throw.
*/
public changeUploadName(uploadId: number, accessId: string|null, name: string) {
const info = this.getUploadInfo(uploadId, accessId);
if (info.files.length > 1) {
throw new Error("UploadSet.changeUploadName cannot operate on multiple files");
}
info.files[0].origName = name;
}
/**
* Returns full info for the given uploadId, without checking authorization.
*/
private _getUploadInfoWithoutAuthorization(uploadId: number): UploadInfo {
const info = this._uploads.get(uploadId);
if (!info) { throw new ApiError(`Unknown upload ${uploadId}`, 404); }
// If the upload is being used, reschedule the inactivity timeout.
info.cleanupTimer.ping();
return info;
}
}
// Maintains uploads created on this host.
export const globalUploadSet: UploadSet = new UploadSet();
// Registers a handler to clean up on exit. We do this intentionally: even though module `tmp` has
// its own logic to clean up, that logic isn't triggered when the server is killed with a signal.
shutdown.addCleanupHandler(null, () => globalUploadSet.cleanupAll());
/**
* Moves this upload to a new directory. A new temporary subdirectory is created there first. If
* the upload contained temporary files, those are moved; if non-temporary files, those are
* copied. Aside from new file locations, the rest of the upload info stays unchanged.
*
* In any case, the previous cleanupCallback is run, and a new one created for the new tmpDir.
*
* This is used specifically for placing uploads into a location accessible by sandboxed code.
*/
export async function moveUpload(uploadInfo: UploadInfo, newDir: string): Promise<void> {
if (uploadInfo.tmpDir && isPathWithin(newDir, uploadInfo.tmpDir)) {
// Upload is already within newDir.
return;
}
log.debug("UploadSet: moving uploadId %s to %s", uploadInfo.uploadId, newDir);
const {tmpDir, cleanupCallback} = await createTmpDir({dir: newDir});
const move: boolean = Boolean(uploadInfo.tmpDir);
const files: FileUploadInfo[] = [];
for (const f of uploadInfo.files) {
const absPath = path.join(tmpDir, path.basename(f.absPath));
await (move ? fse.move(f.absPath, absPath) : fse.copy(f.absPath, absPath));
files.push({...f, absPath});
}
try {
await uploadInfo.cleanupCallback();
} catch (err) {
// This is unexpected, but if the move succeeded, let's warn but not fail on cleanup error.
log.warn(`Error cleaning upload ${uploadInfo.uploadId} after move: ${err}`);
}
Object.assign(uploadInfo, {files, tmpDir, cleanupCallback});
}
interface TmpDirResult {
tmpDir: string;
cleanupCallback: CleanupCB;
}
/**
* Helper to create a temporary directory. It's a simple wrapper around tmp.dir, but replaces the
* cleanup callback with an asynchronous version.
*/
export async function createTmpDir(options: tmp.Options): Promise<TmpDirResult> {
const fullOptions = {prefix: 'grist-upload-', unsafeCleanup: true, ...options};
const [tmpDir, tmpCleanup]: [string, CleanupCB] = await fromCallback(
(cb: any) => tmp.dir(fullOptions, cb), {multiArgs: true});
async function cleanupCallback() {
// Using fs-extra is better because it's asynchronous.
await fse.remove(tmpDir);
try {
// Still call the original callback, so that `tmp` module doesn't keep remembering about
// this directory and doesn't try to delete it again on exit.
tmpCleanup();
} catch (err) {
// OK if it fails because the dir is already removed.
}
}
return {tmpDir, cleanupCallback};
}
/**
* Register a new upload with resource fetched from a public url. Returns corresponding UploadInfo.
*/
export async function fetchURL(url: string, accessId: string|null): Promise<UploadResult> {
return _fetchURL(url, accessId, path.basename(url));
}
/**
* Register a new upload with resource fetched from a url, optionally including credentials in request.
* Returns corresponding UploadInfo.
*/
async function _fetchURL(url: string, accessId: string|null, fileName: string,
headers?: {[key: string]: string}): Promise<UploadResult> {
const response: FetchResponse = await Deps.fetch(url, {headers});
await _checkForError(response);
if (fileName === '') {
const disposition = response.headers.get('content-disposition') || '';
fileName = contentDisposition.parse(disposition).parameters.filename || 'document.grist';
}
const mimeType = response.headers.get('content-type');
const {tmpDir, cleanupCallback} = await createTmpDir({});
// Any name will do for the single file in tmpDir, but note that fileName may not be valid.
const destPath = path.join(tmpDir, 'upload-content');
await new Promise((resolve, reject) => {
const dest = fse.createWriteStream(destPath, {autoClose: true});
response.body.on('error', reject);
dest.on('error', reject);
dest.on('finish', resolve);
response.body.pipe(dest);
});
const uploadedFile: FileUploadInfo = {
absPath: path.resolve(destPath),
origName: fileName,
size: (await fse.stat(destPath)).size,
ext: await guessExt(destPath, fileName, mimeType),
};
log.debug(`done fetching url: ${url} to ${destPath}`);
const uploadId = globalUploadSet.registerUpload([uploadedFile], tmpDir, cleanupCallback, accessId);
return {uploadId, files: [pick(uploadedFile, ['origName', 'size', 'ext'])]};
}
/**
* Fetches a Grist doc potentially managed by a different doc worker. Passes on credentials
* supplied in the current request.
*/
async function fetchDoc(homeUrl: string, docId: string, req: Request, accessId: string|null,
template: boolean): Promise<UploadResult> {
// Prepare headers that preserve credentials of current user.
const headers = getTransitiveHeaders(req);
// Find the doc worker responsible for the document we wish to copy.
const fetchUrl = new URL(`/api/worker/${docId}`, homeUrl);
const response: FetchResponse = await Deps.fetch(fetchUrl.href, {headers});
await _checkForError(response);
const {docWorkerUrl} = await response.json();
// Download the document, in full or as a template.
const url = `${docWorkerUrl}download?doc=${docId}&template=${Number(template)}`;
return _fetchURL(url, accessId, '', headers);
}
// Re-issue failures as exceptions.
async function _checkForError(response: FetchResponse) {
if (response.ok) { return; }
const body = await response.json().catch(() => ({}));
throw new ApiError(body.error || response.statusText, response.status, body.details);
}
/**
* Create an access identifier, combining the userId supplied with the host of the
* doc worker. Returns null if userId is null or in standalone mode.
* Adding host information makes workers sharing a process more useful models of
* full-blown isolated workers.
*/
export function makeAccessId(worker: string|Request|GristServer, userId: number|null): string|null {
if (isSingleUserMode()) { return null; }
if (userId === null) { return null; }
let host: string;
if (typeof worker === 'string') {
host = worker;
} else if ('getHost' in worker) {
host = worker.getHost();
} else {
const gristServer = (worker as RequestWithGrist).gristServer;
if (!gristServer) { throw new Error('Problem accessing server with upload'); }
host = gristServer.getHost();
}
return `${userId}:${host}`;
}