Archives/gristlabs_grist-core
1
0
Fork 0
mirror of https://github.com/gristlabs/grist-core.git synced 2026-03-02 04:09:24 +00:00
Code Issues Projects Releases Wiki Activity

support other SQLite wrappers, and various hooks needed by grist-static (#516)

Browse Source
This commit is contained in:
Paul Fitzpatrick
2023-05-23 15:17:28 -04:00
committed by GitHub
parent bd474a382f
commit 7be0ee289d
42 changed files with 684 additions and 249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
app/server/lib/ActionHistoryImpl.ts
View File

@@ -314,7 +314,7 @@ export class ActionHistoryImpl implements ActionHistory {
} finally {
if (tip) {
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name = "local_sent"`,
WHERE name = 'local_sent'`,
tip);
}
}
@@ -336,7 +336,7 @@ export class ActionHistoryImpl implements ActionHistory {
}
}
await this._db.run(`UPDATE _gristsys_ActionHistoryBranch SET actionRef = ?
WHERE name = "shared"`,
WHERE name = 'shared'`,
candidate.id);
if (candidates.length === 1) {
this._haveLocalSent = false;
@@ -405,9 +405,10 @@ export class ActionHistoryImpl implements ActionHistory {
}
public async getActions(actionNums: number[]): Promise<Array<LocalActionBundle|undefined>> {
const actions = await this._db.all(`SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory
where actionNum in (${actionNums.map(x => '?').join(',')})`,
actionNums);
const actions = await this._db.all(
`SELECT actionHash, actionNum, body FROM _gristsys_ActionHistory
where actionNum in (${actionNums.map(x => '?').join(',')})`,
...actionNums);
return reportTimeTaken("getActions", () => {
const actionsByActionNum = keyBy(actions, 'actionNum');
return actionNums
@@ -516,7 +517,7 @@ export class ActionHistoryImpl implements ActionHistory {
FROM _gristsys_ActionHistoryBranch as Branch
LEFT JOIN _gristsys_ActionHistory as History
ON History.id = Branch.actionRef
WHERE name in ("shared", "local_sent", "local_unsent")`);
WHERE name in ('shared', 'local_sent', 'local_unsent')`);
const bits = mapValues(keyBy(rows, 'name'), this._asActionIdentifiers);
const missing = { actionHash: null, actionRef: null, actionNum: null } as ActionIdentifiers;
return {

33
app/server/lib/ActiveDoc.ts
View File

@@ -169,7 +169,10 @@ const UPDATE_DATA_SIZE_DELAY = {delayMs: 5 * 60 * 1000, varianceMs: 30 * 1000};
const LOG_DOCUMENT_METRICS_DELAY = {delayMs: 60 * 60 * 1000, varianceMs: 30 * 1000};
// A hook for dependency injection.
export const Deps = {ACTIVEDOC_TIMEOUT};
export const Deps = {
ACTIVEDOC_TIMEOUT,
ACTIVEDOC_TIMEOUT_ACTION: 'shutdown' as 'shutdown'|'ignore',
};
interface UpdateUsageOptions {
// Whether usage should be synced to the home database. Defaults to true.
@@ -242,7 +245,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
// Timer for shutting down the ActiveDoc a bit after all clients are gone.
private _inactivityTimer = new InactivityTimer(() => {
this._log.debug(null, 'inactivity timeout');
return this.shutdown();
return this._onInactive();
}, Deps.ACTIVEDOC_TIMEOUT * 1000);
private _recoveryMode: boolean = false;
private _shuttingDown: boolean = false;
@@ -1509,8 +1512,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
*/
public async getUsersForViewAs(docSession: OptDocSession): Promise<PermissionDataWithExtraUsers> {
// Make sure we have rights to view access rules.
const db = this.getHomeDbManager();
if (!db || !await this._granularAccess.hasAccessRulesPermission(docSession)) {
if (!await this._granularAccess.hasAccessRulesPermission(docSession)) {
throw new Error('Cannot list ACL users');
}
@@ -1525,12 +1527,15 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
// Collect users the document is shared with.
const userId = getDocSessionUserId(docSession);
if (!userId) { throw new Error('Cannot determine user'); }
const access = db.unwrapQueryResult(
await db.getDocAccess({userId, urlId: this.docName}, {
flatten: true, excludeUsersWithoutAccess: true,
}));
result.users = access.users;
result.users.forEach(user => isShared.add(normalizeEmail(user.email)));
const db = this.getHomeDbManager();
if (db) {
const access = db.unwrapQueryResult(
await db.getDocAccess({userId, urlId: this.docName}, {
flatten: true, excludeUsersWithoutAccess: true,
}));
result.users = access.users;
result.users.forEach(user => isShared.add(normalizeEmail(user.email)));
}
// Collect users from user attribute tables. Omit duplicates with users the document is
// shared with.
@@ -2048,7 +2053,7 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
documentSettings.engine = (pythonVersion === '2') ? 'python2' : 'python3';
}
await this.docStorage.run('UPDATE _grist_DocInfo SET timezone = ?, documentSettings = ?',
[timezone, JSON.stringify(documentSettings)]);
timezone, JSON.stringify(documentSettings));
}
private _makeInfo(docSession: OptDocSession, options: ApplyUAOptions = {}) {
@@ -2657,6 +2662,12 @@ export class ActiveDoc extends EventEmitter implements AssistanceDoc {
}
return this._attachmentColumns;
}
private async _onInactive() {
if (Deps.ACTIVEDOC_TIMEOUT_ACTION === 'shutdown') {
await this.shutdown();
}
}
}
// Helper to initialize a sandbox action bundle with no values.

76
app/server/lib/DocStorage.ts
View File

@@ -7,7 +7,6 @@
*/
import * as sqlite3 from '@gristlabs/sqlite3';
import {LocalActionBundle} from 'app/common/ActionBundle';
import {BulkColValues, DocAction, TableColValues, TableDataAction, toTableDataAction} from 'app/common/DocActions';
import * as gristTypes from 'app/common/gristTypes';
@@ -23,12 +22,12 @@ import log from 'app/server/lib/log';
import assert from 'assert';
import * as bluebird from 'bluebird';
import * as fse from 'fs-extra';
import {RunResult} from 'sqlite3';
import * as _ from 'underscore';
import * as util from 'util';
import uuidv4 from "uuid/v4";
import {OnDemandStorage} from './OnDemandActions';
import {ISQLiteDB, MigrationHooks, OpenMode, quoteIdent, ResultRow, SchemaInfo, SQLiteDB} from './SQLiteDB';
import {ISQLiteDB, MigrationHooks, OpenMode, PreparedStatement, quoteIdent,
ResultRow, RunResult, SchemaInfo, SQLiteDB} from 'app/server/lib/SQLiteDB';
import chunk = require('lodash/chunk');
import cloneDeep = require('lodash/cloneDeep');
import groupBy = require('lodash/groupBy');
@@ -447,7 +446,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
* Converts an array of columns to an array of rows (suitable to use as sqlParams), encoding all
* values as needed, according to an array of Grist type strings (must be parallel to columns).
*/
private static _encodeColumnsToRows(types: string[], valueColumns: any[]): any[] {
private static _encodeColumnsToRows(types: string[], valueColumns: any[]): any[][] {
const marshaller = new marshal.Marshaller({version: 2});
const rows = _.unzip(valueColumns);
for (const row of rows) {
@@ -734,8 +733,11 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
})
.catch(err => {
// This replicates previous logic for _updateMetadata.
if (err.message.startsWith('SQLITE_ERROR: no such table')) {
// It matches errors from node-sqlite3 and better-sqlite3
if (err.message.startsWith('SQLITE_ERROR: no such table') ||
err.message.startsWith('no such table:')) {
err.message = `NO_METADATA_ERROR: ${this.docName} has no metadata`;
if (!err.cause) { err.cause = {}; }
err.cause.code = 'NO_METADATA_ERROR';
}
throw err;
@@ -781,7 +783,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
.then(() => true)
// If UNIQUE constraint failed, this ident must already exists, so return false.
.catch(err => {
if (/^SQLITE_CONSTRAINT: UNIQUE constraint failed/.test(err.message)) {
if (/^(SQLITE_CONSTRAINT: )?UNIQUE constraint failed/.test(err.message)) {
return false;
}
throw err;
@@ -879,7 +881,7 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
}
whereParts = whereParts.concat(query.wheres ?? []);
const sql = this._getSqlForQuery(query, whereParts);
return this._getDB().allMarshal(sql, params);
return this._getDB().allMarshal(sql, ...params);
}
/**
@@ -1125,16 +1127,12 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
if (numChunks > 0) {
debuglog("DocStorage.BulkRemoveRecord: splitting " + rowIds.length +
" deletes into chunks of size " + chunkSize);
await this.prepare(preSql + chunkParams + postSql)
.then(function(stmt) {
return bluebird.Promise.each(_.range(0, numChunks * chunkSize, chunkSize), function(index: number) {
debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1));
return bluebird.Promise.fromCallback((cb: any) => stmt.run(rowIds.slice(index, index + chunkSize), cb));
})
.then(function() {
return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb));
});
});
const stmt = await this.prepare(preSql + chunkParams + postSql);
for (const index of _.range(0, numChunks * chunkSize, chunkSize)) {
debuglog("DocStorage.BulkRemoveRecord: chunk delete " + index + "-" + (index + chunkSize - 1));
await stmt.run(rowIds.slice(index, index + chunkSize));
}
await stmt.finalize();
}
if (numLeftovers > 0) {
@@ -1433,8 +1431,8 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
return this._markAsChanged(this._getDB().exec(sql));
}
public prepare(sql: string, ...args: any[]): Promise<sqlite3.Statement> {
return this._getDB().prepare(sql, ...args);
public prepare(sql: string): Promise<PreparedStatement> {
return this._getDB().prepare(sql);
}
public get(sql: string, ...args: any[]): Promise<ResultRow|undefined> {
@@ -1545,7 +1543,16 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
name LIKE 'sqlite_%' OR
name LIKE '_gristsys_%'
);
`);
`).catch(e => {
if (String(e).match(/no such table: dbstat/)) {
// We are using a version of SQLite that doesn't have
// dbstat compiled in. But it would be sad to disable
// Grist entirely just because we can't track byte-count.
// So return NaN in this case.
return {totalSize: NaN};
}
throw e;
});
return result!.totalSize;
}
@@ -1576,19 +1583,15 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
/**
* Internal helper for applying Bulk Update or Add Record sql
*/
private async _applyMaybeBulkUpdateOrAddSql(sql: string, sqlParams: any[]): Promise<void> {
private async _applyMaybeBulkUpdateOrAddSql(sql: string, sqlParams: any[][]): Promise<void> {
if (sqlParams.length === 1) {
await this.run(sql, sqlParams[0]);
await this.run(sql, ...sqlParams[0]);
} else {
return this.prepare(sql)
.then(function(stmt) {
return bluebird.Promise.each(sqlParams, function(param: string) {
return bluebird.Promise.fromCallback((cb: any) => stmt.run(param, cb));
})
.then(function() {
return bluebird.Promise.fromCallback((cb: any) => stmt.finalize(cb));
});
});
const stmt = await this.prepare(sql);
for (const param of sqlParams) {
await stmt.run(...param);
}
await stmt.finalize();
}
}
@@ -1613,9 +1616,9 @@ export class DocStorage implements ISQLiteDB, OnDemandStorage {
}
const oldGristType = this._getGristType(tableId, colId);
const oldSqlType = colInfo.type || 'BLOB';
const oldDefault = colInfo.dflt_value;
const oldDefault = fixDefault(colInfo.dflt_value);
const newSqlType = newColType ? DocStorage._getSqlType(newColType) : oldSqlType;
const newDefault = newColType ? DocStorage._formattedDefault(newColType) : oldDefault;
const newDefault = fixDefault(newColType ? DocStorage._formattedDefault(newColType) : oldDefault);
const newInfo = {name: newColId, type: newSqlType, dflt_value: newDefault};
// Check if anything actually changed, and only rebuild the table then.
if (Object.keys(newInfo).every(p => ((newInfo as any)[p] === colInfo[p]))) {
@@ -1832,3 +1835,10 @@ export interface IndexInfo extends IndexColumns {
export async function createAttachmentsIndex(db: ISQLiteDB) {
await db.exec(`CREATE INDEX _grist_Attachments_fileIdent ON _grist_Attachments(fileIdent)`);
}
// Old docs may have incorrect quotes in their schema for default values
// that node-sqlite3 may tolerate but not other wrappers. Patch such
// material as we run into it.
function fixDefault(def: string) {
return (def === '""') ? "''" : def;
}

13
app/server/lib/GranularAccess.ts
View File

@@ -1879,11 +1879,10 @@ export class GranularAccess implements GranularAccessForBundle {
* tables or examples.
*/
private async _getViewAsUser(linkParameters: Record<string, string>): Promise<UserOverride> {
// Look up user information in database.
if (!this._homeDbManager) { throw new Error('database required'); }
// Look up user information in database, if available
const dbUser = linkParameters.aclAsUserId ?
(await this._homeDbManager.getUser(integerParam(linkParameters.aclAsUserId, 'aclAsUserId'))) :
(await this._homeDbManager.getExistingUserByLogin(linkParameters.aclAsUser));
(await this._homeDbManager?.getUser(integerParam(linkParameters.aclAsUserId, 'aclAsUserId'))) :
(await this._homeDbManager?.getExistingUserByLogin(linkParameters.aclAsUser));
// If this is one of example users we will pretend that it doesn't exist, otherwise we would
// end up using permissions of the real user.
const isExampleUser = this.getExampleViewAsUsers().some(e => e.email === dbUser?.loginEmail);
@@ -1905,13 +1904,13 @@ export class GranularAccess implements GranularAccessForBundle {
};
}
}
const docAuth = userExists ? await this._homeDbManager.getDocAuthCached({
const docAuth = userExists ? await this._homeDbManager?.getDocAuthCached({
urlId: this._docId,
userId: dbUser.id
}) : null;
const access = docAuth?.access || null;
const user = userExists ? this._homeDbManager.makeFullUser(dbUser) : null;
return { access, user };
const user = userExists ? this._homeDbManager?.makeFullUser(dbUser) : null;
return { access, user: user || null };
}
/**

11
app/server/lib/ICreate.ts
View File

@@ -7,7 +7,8 @@ import {IBilling} from 'app/server/lib/IBilling';
import {INotifier} from 'app/server/lib/INotifier';
import {ISandbox, ISandboxCreationOptions} from 'app/server/lib/ISandbox';
import {IShell} from 'app/server/lib/IShell';
import {createSandbox} from 'app/server/lib/NSandbox';
import {createSandbox, SpawnFn} from 'app/server/lib/NSandbox';
import {SqliteVariant} from 'app/server/lib/SqliteCommon';
export interface ICreate {
@@ -31,6 +32,8 @@ export interface ICreate {
// static page.
getExtraHeadHtml?(): string;
getStorageOptions?(name: string): ICreateStorageOptions|undefined;
getSqliteVariant?(): SqliteVariant;
getSandboxVariants?(): Record<string, SpawnFn>;
}
export interface ICreateActiveDocOptions {
@@ -62,6 +65,8 @@ export function makeSimpleCreator(opts: {
sandboxFlavor?: string,
shell?: IShell,
getExtraHeadHtml?: () => string,
getSqliteVariant?: () => SqliteVariant,
getSandboxVariants?: () => Record<string, SpawnFn>,
}): ICreate {
const {sessionSecret, storage, notifier, billing} = opts;
return {
@@ -121,6 +126,8 @@ export function makeSimpleCreator(opts: {
},
getStorageOptions(name: string) {
return storage?.find(s => s.name === name);
}
},
getSqliteVariant: opts.getSqliteVariant,
getSandboxVariants: opts.getSandboxVariants,
};
}

205
app/server/lib/NSandbox.ts
View File

@@ -3,6 +3,7 @@
*/
import {arrayToString} from 'app/common/arrayToString';
import * as marshal from 'app/common/marshal';
import {create} from 'app/server/lib/create';
import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox';
import log from 'app/server/lib/log';
import {getAppRoot, getAppRootFor, getUnpackedAppRoot} from 'app/server/lib/places';
@@ -69,12 +70,18 @@ export interface ISandboxOptions {
* We interact with sandboxes as a separate child process. Data engine work is done
* across standard input and output streams from and to this process. We also monitor
* and control resource utilization via a distinct control interface.
*
* More recently, a sandbox may not be a separate OS process, but (for
* example) a web worker. In this case, a pair of callbacks (getData and
* sendData) replace pipes.
*/
interface SandboxProcess {
child: ChildProcess;
export interface SandboxProcess {
child?: ChildProcess;
control: ISandboxControl;
dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data
dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data
getData?: (cb: (data: any) => void) => void; // use a callback instead of a pipe to get data
sendData?: (data: any) => void; // use a callback instead of a pipe to send data
}
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
@@ -88,7 +95,7 @@ const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR;
export class NSandbox implements ISandbox {
public readonly childProc: ChildProcess;
public readonly childProc?: ChildProcess;
private _control: ISandboxControl;
private _logTimes: boolean;
private _exportedFunctions: {[name: string]: SandboxMethod};
@@ -101,8 +108,9 @@ export class NSandbox implements ISandbox {
private _isWriteClosed = false;
private _logMeta: log.ILogMeta;
private _streamToSandbox: Writable;
private _streamToSandbox?: Writable;
private _streamFromSandbox: Stream;
private _dataToSandbox?: (data: any) => void;
private _lastStderr: Uint8Array; // Record last error line seen.
// Create a unique subdirectory for each sandbox process so they can be replayed separately
@@ -129,52 +137,26 @@ export class NSandbox implements ISandbox {
this._control = sandboxProcess.control;
this.childProc = sandboxProcess.child;
this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
this._logMeta = {sandboxPid: this.childProc?.pid, ...options.logMeta};
if (options.minimalPipeMode) {
log.rawDebug("3-pipe Sandbox started", this._logMeta);
if (sandboxProcess.dataToSandboxDescriptor) {
this._streamToSandbox =
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
if (this.childProc) {
if (options.minimalPipeMode) {
this._initializeMinimalPipeMode(sandboxProcess);
} else {
this._streamToSandbox = this.childProc.stdin!;
}
if (sandboxProcess.dataFromSandboxDescriptor) {
this._streamFromSandbox =
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
} else {
this._streamFromSandbox = this.childProc.stdout!;
this._initializeFivePipeMode(sandboxProcess);
}
} else {
log.rawDebug("5-pipe Sandbox started", this._logMeta);
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
throw new Error('cannot override file descriptors in 5 pipe mode');
// No child process. In this case, there should be a callback for
// receiving and sending data.
if (!sandboxProcess.getData) {
throw new Error('no way to get data from sandbox');
}
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
if (!sandboxProcess.sendData) {
throw new Error('no way to send data to sandbox');
}
sandboxProcess.getData((data) => this._onSandboxData(data));
this._dataToSandbox = sandboxProcess.sendData;
}
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
this.childProc.stderr!.on('data', data => {
this._lastStderr = data;
sandboxStderrLogger(data);
});
this.childProc.on('close', this._onExit.bind(this));
this.childProc.on('error', this._onError.bind(this));
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
this._streamFromSandbox.on('end', () => this._onSandboxClose());
this._streamFromSandbox.on('error', (err) => {
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
this._onSandboxClose();
});
this._streamToSandbox.on('error', (err) => {
if (!this._isWriteClosed) {
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
}
});
// On shutdown, shutdown the child process cleanly, and wait for it to exit.
shutdown.addCleanupHandler(this, this.shutdown);
@@ -203,9 +185,9 @@ export class NSandbox implements ISandbox {
const result = await new Promise<void>((resolve, reject) => {
if (this._isWriteClosed) { resolve(); }
this.childProc.on('error', reject);
this.childProc.on('close', resolve);
this.childProc.on('exit', resolve);
this.childProc?.on('error', reject);
this.childProc?.on('close', resolve);
this.childProc?.on('exit', resolve);
this._close();
}).finally(() => this._control.close());
@@ -244,6 +226,82 @@ export class NSandbox implements ISandbox {
log.rawDebug('Sandbox memory', {memory, ...this._logMeta});
}
/**
* Get ready to communicate with a sandbox process using stdin,
* stdout, and stderr.
*/
private _initializeMinimalPipeMode(sandboxProcess: SandboxProcess) {
log.rawDebug("3-pipe Sandbox started", this._logMeta);
if (!this.childProc) {
throw new Error('child process required');
}
if (sandboxProcess.dataToSandboxDescriptor) {
this._streamToSandbox =
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
} else {
this._streamToSandbox = this.childProc.stdin!;
}
if (sandboxProcess.dataFromSandboxDescriptor) {
this._streamFromSandbox =
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
} else {
this._streamFromSandbox = this.childProc.stdout!;
}
this._initializeStreamEvents();
}
/**
* Get ready to communicate with a sandbox process using stdin,
* stdout, and stderr, and two extra FDs. This was a nice way
* to have a clean, separate data channel, when supported.
*/
private _initializeFivePipeMode(sandboxProcess: SandboxProcess) {
log.rawDebug("5-pipe Sandbox started", this._logMeta);
if (!this.childProc) {
throw new Error('child process required');
}
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
throw new Error('cannot override file descriptors in 5 pipe mode');
}
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
this._initializeStreamEvents();
}
/**
* Set up logging and events on streams to/from a sandbox.
*/
private _initializeStreamEvents() {
if (!this.childProc) {
throw new Error('child process required');
}
if (!this._streamToSandbox) {
throw new Error('expected streamToSandbox to be configured');
}
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
this.childProc.stderr!.on('data', data => {
this._lastStderr = data;
sandboxStderrLogger(data);
});
this.childProc.on('close', this._onExit.bind(this));
this.childProc.on('error', this._onError.bind(this));
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
this._streamFromSandbox.on('end', () => this._onSandboxClose());
this._streamFromSandbox.on('error', (err) => {
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
this._onSandboxClose();
});
this._streamToSandbox.on('error', (err) => {
if (!this._isWriteClosed) {
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
}
});
}
private async _pyCallWait(funcName: string, startTime: number): Promise<any> {
try {
return await new Promise((resolve, reject) => {
@@ -263,7 +321,7 @@ export class NSandbox implements ISandbox {
this._control.prepareToClose();
if (!this._isWriteClosed) {
// Close the pipe to the sandbox, which should cause the sandbox to exit cleanly.
this._streamToSandbox.end();
this._streamToSandbox?.end();
this._isWriteClosed = true;
}
}
@@ -298,10 +356,17 @@ export class NSandbox implements ISandbox {
if (this._recordBuffersDir) {
fs.appendFileSync(path.resolve(this._recordBuffersDir, "input"), buf);
}
return this._streamToSandbox.write(buf);
if (this._streamToSandbox) {
return this._streamToSandbox.write(buf);
} else {
if (!this._dataToSandbox) {
throw new Error('no way to send data to sandbox');
}
this._dataToSandbox(buf);
return true;
}
}
/**
* Process a buffer of data received from the sandbox process.
*/
@@ -422,18 +487,26 @@ function isFlavor(flavor: string): flavor is keyof typeof spawners {
* It is ignored by other flavors.
*/
export class NSandboxCreator implements ISandboxCreator {
private _flavor: keyof typeof spawners;
private _flavor: string;
private _spawner: SpawnFn;
private _command?: string;
private _preferredPythonVersion?: string;
public constructor(options: {
defaultFlavor: keyof typeof spawners,
defaultFlavor: string,
command?: string,
preferredPythonVersion?: string,
}) {
const flavor = options.defaultFlavor;
if (!isFlavor(flavor)) {
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
const variants = create.getSandboxVariants?.();
if (!variants?.[flavor]) {
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
} else {
this._spawner = variants[flavor];
}
} else {
this._spawner = spawners[flavor];
}
this._flavor = flavor;
this._command = options.command;
@@ -463,12 +536,12 @@ export class NSandboxCreator implements ISandboxCreator {
importDir: options.importMount,
...options.sandboxOptions,
};
return new NSandbox(translatedOptions, spawners[this._flavor]);
return new NSandbox(translatedOptions, this._spawner);
}
}
// A function that takes sandbox options and starts a sandbox process.
type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
export type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
/**
* Helper function to run a nacl sandbox. It takes care of most arguments, similarly to
@@ -750,7 +823,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess {
...getWrappingEnv(options),
};
const command = findPython(options.command, options.preferredPythonVersion);
const realPath = fs.realpathSync(command);
const realPath = realpathSync(command);
log.rawDebug("macSandboxExec found a python", {...options.logMeta, command: realPath});
// Prepare sandbox profile
@@ -868,11 +941,11 @@ function getAbsolutePaths(options: ISandboxOptions) {
// Get path to sandbox directory - this is a little idiosyncratic to work well
// in grist-core. It is important to use real paths since we may be viewing
// the file system through a narrow window in a container.
const sandboxDir = path.join(fs.realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
const sandboxDir = path.join(realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
'..');
// Copy plugin options, and then make them absolute.
if (options.importDir) {
options.importDir = fs.realpathSync(options.importDir);
options.importDir = realpathSync(options.importDir);
}
return {
sandboxDir,
@@ -976,9 +1049,6 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
const flavor = parts[parts.length - 1];
const version = parts.length === 2 ? parts[0] : '*';
if (preferredPythonVersion === version || version === '*' || !preferredPythonVersion) {
if (!isFlavor(flavor)) {
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
}
const creator = new NSandboxCreator({
defaultFlavor: flavor,
command: process.env['GRIST_SANDBOX' + (preferredPythonVersion||'')] ||
@@ -990,3 +1060,16 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
}
throw new Error('Failed to create a sandbox');
}
/**
* The realpath function may not be available, just return the
* path unchanged if it is not. Specifically, this happens when
* compiled for use in a browser environment.
*/
function realpathSync(src: string) {
try {
return fs.realpathSync(src);
} catch (e) {
return src;
}
}

136
app/server/lib/SQLiteDB.ts
View File

@@ -69,23 +69,24 @@
import {ErrorWithCode} from 'app/common/ErrorWithCode';
import {timeFormat} from 'app/common/timeFormat';
import {create} from 'app/server/lib/create';
import * as docUtils from 'app/server/lib/docUtils';
import log from 'app/server/lib/log';
import {fromCallback} from 'app/server/lib/serverUtils';
import * as sqlite3 from '@gristlabs/sqlite3';
import {MinDB, MinRunResult, PreparedStatement, ResultRow,
SqliteVariant, Statement} from 'app/server/lib/SqliteCommon';
import {NodeSqliteVariant} from 'app/server/lib/SqliteNode';
import assert from 'assert';
import {each} from 'bluebird';
import * as fse from 'fs-extra';
import {RunResult} from 'sqlite3';
import fromPairs = require('lodash/fromPairs');
import isEqual = require('lodash/isEqual');
import noop = require('lodash/noop');
import range = require('lodash/range');
// Describes the result of get() and all() database methods.
export interface ResultRow {
[column: string]: any;
export type {PreparedStatement, ResultRow, Statement};
export type RunResult = MinRunResult;
function getVariant(): SqliteVariant {
return create.getSqliteVariant?.() || new NodeSqliteVariant();
}
// Describes how to create a new DB or migrate an old one. Any changes to the DB must be reflected
@@ -136,7 +137,7 @@ export interface ISQLiteDB {
run(sql: string, ...params: any[]): Promise<RunResult>;
get(sql: string, ...params: any[]): Promise<ResultRow|undefined>;
all(sql: string, ...params: any[]): Promise<ResultRow[]>;
prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement>;
prepare(sql: string, ...params: any[]): Promise<PreparedStatement>;
execTransaction<T>(callback: () => Promise<T>): Promise<T>;
runAndGetId(sql: string, ...params: any[]): Promise<number>;
requestVacuum(): Promise<boolean>;
@@ -196,18 +197,11 @@ export class SQLiteDB implements ISQLiteDB {
*/
public static async openDBRaw(dbPath: string,
mode: OpenMode = OpenMode.OPEN_CREATE): Promise<SQLiteDB> {
const sqliteMode: number =
// tslint:disable-next-line:no-bitwise
(mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) |
(mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0);
let _db: sqlite3.Database;
await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); });
limitAttach(_db!, 0); // Outside of VACUUM, we don't allow ATTACH.
const minDb: MinDB = await getVariant().opener(dbPath, mode);
if (SQLiteDB._addOpens(dbPath, 1) > 1) {
log.warn("SQLiteDB[%s] avoid opening same DB more than once", dbPath);
}
return new SQLiteDB(_db!, dbPath);
return new SQLiteDB(minDb, dbPath);
}
/**
@@ -261,12 +255,29 @@ export class SQLiteDB implements ISQLiteDB {
private _migrationError: Error|null = null;
private _needVacuum: boolean = false;
private constructor(private _db: sqlite3.Database, private _dbPath: string) {
// Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow
// This isn't enough for transactions, which we serialize explicitly.
this._db.serialize();
private constructor(protected _db: MinDB, private _dbPath: string) {
}
public async all(sql: string, ...args: any[]): Promise<ResultRow[]> {
const result = await this._db.all(sql, ...args);
return result;
}
public run(sql: string, ...args: any[]): Promise<MinRunResult> {
return this._db.run(sql, ...args);
}
public exec(sql: string): Promise<void> {
return this._db.exec(sql);
}
public prepare(sql: string): Promise<PreparedStatement> {
return this._db.prepare(sql);
}
public get(sql: string, ...args: any[]): Promise<ResultRow|undefined> {
return this._db.get(sql, ...args);
}
/**
* If a DB was migrated on open, this will be set to the path of the pre-migration backup copy.
@@ -285,40 +296,8 @@ export class SQLiteDB implements ISQLiteDB {
// The following methods mirror https://github.com/mapbox/node-sqlite3/wiki/API, but return
// Promises. We use fromCallback() rather than use promisify, to get better type-checking.
public exec(sql: string): Promise<void> {
return fromCallback(cb => this._db.exec(sql, cb));
}
public run(sql: string, ...params: any[]): Promise<RunResult> {
return new Promise((resolve, reject) => {
function callback(this: RunResult, err: Error | null) {
if (err) {
reject(err);
} else {
resolve(this);
}
}
this._db.run(sql, ...params, callback);
});
}
public get(sql: string, ...params: any[]): Promise<ResultRow|undefined> {
return fromCallback(cb => this._db.get(sql, ...params, cb));
}
public all(sql: string, ...params: any[]): Promise<ResultRow[]> {
return fromCallback(cb => this._db.all(sql, ...params, cb));
}
public allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
// allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib.
return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb));
}
public prepare(sql: string, ...params: any[]): Promise<sqlite3.Statement> {
let stmt: sqlite3.Statement;
// The original interface is a little strange; we resolve to Statement if prepare() succeeded.
return fromCallback(cb => { stmt = this._db.prepare(sql, ...params, cb); }).then(() => stmt);
public async allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
return this._db.allMarshal(sql, ...params);
}
/**
@@ -336,11 +315,11 @@ export class SQLiteDB implements ISQLiteDB {
}
public async vacuum(): Promise<void> {
limitAttach(this._db, 1); // VACUUM implementation uses ATTACH.
await this._db.limitAttach(1); // VACUUM implementation uses ATTACH.
try {
await this.exec("VACUUM");
} finally {
limitAttach(this._db, 0); // Outside of VACUUM, we don't allow ATTACH.
await this._db.limitAttach(0); // Outside of VACUUM, we don't allow ATTACH.
}
}
@@ -348,25 +327,24 @@ export class SQLiteDB implements ISQLiteDB {
* Run each of the statements in turn. Each statement is either a string, or an array of arguments
* to db.run, e.g. [sqlString, [params...]].
*/
public runEach(...statements: Array<string | [string, any[]]>): Promise<void> {
return each(statements,
async (stmt: any) => {
try {
return await (Array.isArray(stmt) ?
this.run(stmt[0], ...stmt[1]) :
this.exec(stmt)
);
} catch (err) {
log.warn(`SQLiteDB: Failed to run ${stmt}`);
throw err;
public async runEach(...statements: Array<string | [string, any[]]>): Promise<void> {
for (const stmt of statements) {
try {
if (Array.isArray(stmt)) {
await this.run(stmt[0], ...stmt[1]);
} else {
await this.exec(stmt);
}
} catch (err) {
log.warn(`SQLiteDB: Failed to run ${stmt}`);
throw err;
}
);
}
}
public close(): Promise<void> {
return fromCallback(cb => this._db.close(cb))
.then(() => { SQLiteDB._addOpens(this._dbPath, -1); });
public async close(): Promise<void> {
await this._db.close();
SQLiteDB._addOpens(this._dbPath, -1);
}
/**
@@ -375,8 +353,7 @@ export class SQLiteDB implements ISQLiteDB {
* is only useful if the sql is actually an INSERT operation, but we don't check this.
*/
public async runAndGetId(sql: string, ...params: any[]): Promise<number> {
const result = await this.run(sql, ...params);
return result.lastID;
return this._db.runAndGetId(sql, ...params);
}
/**
@@ -567,12 +544,3 @@ export function quoteIdent(ident: string): string {
assert(/^[\w.]+$/.test(ident), `SQL identifier is not valid: ${ident}`);
return `"${ident}"`;
}
/**
* Limit the number of ATTACHed databases permitted.
*/
export function limitAttach(db: sqlite3.Database, maxAttach: number) {
// Pardon the casts, types are out of date.
const SQLITE_LIMIT_ATTACHED = (sqlite3 as any).LIMIT_ATTACHED;
(db as any).configure('limit', SQLITE_LIMIT_ATTACHED, maxAttach);
}

126
app/server/lib/SqliteCommon.ts Normal file
View File

@@ -0,0 +1,126 @@
import { Marshaller } from 'app/common/marshal';
import { OpenMode, quoteIdent } from 'app/server/lib/SQLiteDB';
/**
* Code common to SQLite wrappers.
*/
/**
* It is important that Statement exists - but we don't expect
* anything of it.
*/
// eslint-disable-next-line @typescript-eslint/no-empty-interface
export interface Statement {}
export interface MinDB {
exec(sql: string): Promise<void>;
run(sql: string, ...params: any[]): Promise<MinRunResult>;
get(sql: string, ...params: any[]): Promise<ResultRow|undefined>;
all(sql: string, ...params: any[]): Promise<ResultRow[]>;
prepare(sql: string, ...params: any[]): Promise<PreparedStatement>;
runAndGetId(sql: string, ...params: any[]): Promise<number>;
close(): Promise<void>;
allMarshal(sql: string, ...params: any[]): Promise<Buffer>;
/**
* Limit the number of ATTACHed databases permitted.
*/
limitAttach(maxAttach: number): Promise<void>;
}
export interface MinRunResult {
changes: number;
}
// Describes the result of get() and all() database methods.
export interface ResultRow {
[column: string]: any;
}
export interface PreparedStatement {
run(...params: any[]): Promise<MinRunResult>;
finalize(): Promise<void>;
columns(): string[];
}
export interface SqliteVariant {
opener(dbPath: string, mode: OpenMode): Promise<MinDB>;
}
/**
* A crude implementation of Grist marshalling.
* There is a fork of node-sqlite3 that has Grist
* marshalling built in, at:
* https://github.com/gristlabs/node-sqlite3
* If using a version of SQLite without this built
* in, another option is to add custom functions
* to do it. This object has the initialize, step,
* and finalize callbacks typically needed to add
* a custom aggregration function.
*/
export const gristMarshal = {
initialize(): GristMarshalIntermediateValue {
return {};
},
step(accum: GristMarshalIntermediateValue, ...row: any[]) {
if (!accum.names || !accum.values) {
accum.names = row.map(value => String(value));
accum.values = row.map(() => []);
} else {
for (const [i, v] of row.entries()) {
accum.values[i].push(v);
}
}
return accum;
},
finalize(accum: GristMarshalIntermediateValue) {
const marshaller = new Marshaller({version: 2, keysAreBuffers: true});
const result: Record<string, Array<any>> = {};
if (accum.names && accum.values) {
for (const [i, name] of accum.names.entries()) {
result[name] = accum.values[i];
}
}
marshaller.marshal(result);
return marshaller.dumpAsBuffer();
}
};
/**
* An intermediate value used during an aggregation.
*/
interface GristMarshalIntermediateValue {
// The names of the columns, once known.
names?: string[];
// Values stored in the columns.
// There is one element in the outermost array per column.
// That element contains a list of values stored in that column.
values?: Array<Array<any>>;
}
/**
* Run Grist marshalling as a SQLite query, assuming
* a custom aggregation has been added as "grist_marshal".
* The marshalled result needs to contain the column
* identifiers embedded in it. This is a little awkward
* to organize - hence the hacky UNION here. This is
* for compatibility with the existing marshalling method,
* which could be replaced instead.
*/
export async function allMarshalQuery(db: MinDB, sql: string, ...params: any[]): Promise<Buffer> {
const statement = await db.prepare(sql);
const columns = statement.columns();
const quotedColumnList = columns.map(quoteIdent).join(',');
const query = await db.all(`select grist_marshal(${quotedColumnList}) as buf FROM ` +
`(select ${quotedColumnList} UNION ALL select * from (` + sql + '))', ..._fixParameters(params));
return query[0].buf;
}
/**
* Booleans need to be cast to 1 or 0 for SQLite.
* The node-sqlite3 wrapper does this automatically, but other
* wrappers do not.
*/
function _fixParameters(params: any[]) {
return params.map(p => p === true ? 1 : (p === false ? 0 : p));
}

104
app/server/lib/SqliteNode.ts Normal file
View File

@@ -0,0 +1,104 @@
import * as sqlite3 from '@gristlabs/sqlite3';
import { fromCallback } from 'app/server/lib/serverUtils';
import { MinDB, PreparedStatement, ResultRow, SqliteVariant } from 'app/server/lib/SqliteCommon';
import { OpenMode, RunResult } from 'app/server/lib/SQLiteDB';
export class NodeSqliteVariant implements SqliteVariant {
public opener(dbPath: string, mode: OpenMode): Promise<MinDB> {
return NodeSqlite3DatabaseAdapter.opener(dbPath, mode);
}
}
export class NodeSqlite3PreparedStatement implements PreparedStatement {
public constructor(private _statement: sqlite3.Statement) {
}
public async run(...params: any[]): Promise<RunResult> {
return fromCallback(cb => this._statement.run(...params, cb));
}
public async finalize() {
await fromCallback(cb => this._statement.finalize(cb));
}
public columns(): string[] {
// This method is only needed if marshalling is not built in -
// and node-sqlite3 has marshalling built in.
throw new Error('not available (but should not be needed)');
}
}
export class NodeSqlite3DatabaseAdapter implements MinDB {
public static async opener(dbPath: string, mode: OpenMode): Promise<any> {
const sqliteMode: number =
// tslint:disable-next-line:no-bitwise
(mode === OpenMode.OPEN_READONLY ? sqlite3.OPEN_READONLY : sqlite3.OPEN_READWRITE) |
(mode === OpenMode.OPEN_CREATE || mode === OpenMode.CREATE_EXCL ? sqlite3.OPEN_CREATE : 0);
let _db: sqlite3.Database;
await fromCallback(cb => { _db = new sqlite3.Database(dbPath, sqliteMode, cb); });
const result = new NodeSqlite3DatabaseAdapter(_db!);
await result.limitAttach(0); // Outside of VACUUM, we don't allow ATTACH.
return result;
}
public constructor(protected _db: sqlite3.Database) {
// Default database to serialized execution. See https://github.com/mapbox/node-sqlite3/wiki/Control-Flow
// This isn't enough for transactions, which we serialize explicitly.
this._db.serialize();
}
public async exec(sql: string): Promise<void> {
return fromCallback(cb => this._db.exec(sql, cb));
}
public async run(sql: string, ...params: any[]): Promise<RunResult> {
return new Promise((resolve, reject) => {
function callback(this: RunResult, err: Error | null) {
if (err) {
reject(err);
} else {
resolve(this);
}
}
this._db.run(sql, ...params, callback);
});
}
public async get(sql: string, ...params: any[]): Promise<ResultRow|undefined> {
return fromCallback(cb => this._db.get(sql, ...params, cb));
}
public async all(sql: string, ...params: any[]): Promise<ResultRow[]> {
return fromCallback(cb => this._db.all(sql, params, cb));
}
public async prepare(sql: string): Promise<PreparedStatement> {
let stmt: sqlite3.Statement|undefined;
// The original interface is a little strange; we resolve to Statement if prepare() succeeded.
await fromCallback(cb => { stmt = this._db.prepare(sql, cb); }).then(() => stmt);
if (!stmt) { throw new Error('could not prepare statement'); }
return new NodeSqlite3PreparedStatement(stmt);
}
public async close() {
this._db.close();
}
public async allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
// allMarshal isn't in the typings, because it is our addition to our fork of sqlite3 JS lib.
return fromCallback(cb => (this._db as any).allMarshal(sql, ...params, cb));
}
public async runAndGetId(sql: string, ...params: any[]): Promise<number> {
const result = await this.run(sql, ...params);
return (result as any).lastID;
}
public async limitAttach(maxAttach: number) {
const SQLITE_LIMIT_ATTACHED = (sqlite3 as any).LIMIT_ATTACHED;
// Cast because types out of date.
(this._db as any).configure('limit', SQLITE_LIMIT_ATTACHED, maxAttach);
}
}

7
app/server/lib/WorkCoordinator.ts
View File

@@ -60,7 +60,12 @@ export class WorkCoordinator {
private _maybeSchedule() {
if (this._isStepScheduled && !this._isStepRunning) {
setImmediate(this._tryNextStepCB);
try {
setImmediate(this._tryNextStepCB);
} catch (e) {
// setImmediate may not be available outside node.
setTimeout(this._tryNextStepCB, 0);
}
}
}
}

1
app/server/lib/requestUtils.ts
View File

@@ -8,7 +8,6 @@ import {RequestWithGrist} from 'app/server/lib/GristServer';
import log from 'app/server/lib/log';
import {Permit} from 'app/server/lib/Permit';
import {Request, Response} from 'express';
import {URL} from 'url';
import _ from 'lodash';
// log api details outside of dev environment (when GRIST_HOSTED_VERSION is set)

2
app/server/lib/shutdown.js
View File

@@ -3,8 +3,8 @@
*/
var log = require('app/server/lib/log');
var Promise = require('bluebird');
var log = require('./log');
var cleanupHandlers = [];