mirror of
https://github.com/gristlabs/grist-core.git
synced 2026-03-02 04:09:24 +00:00
support other SQLite wrappers, and various hooks needed by grist-static (#516)
This commit is contained in:
Paul Fitzpatrick
GitHub
@@ -3,6 +3,7 @@
|
||||
*/
|
||||
import {arrayToString} from 'app/common/arrayToString';
|
||||
import * as marshal from 'app/common/marshal';
|
||||
import {create} from 'app/server/lib/create';
|
||||
import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox';
|
||||
import log from 'app/server/lib/log';
|
||||
import {getAppRoot, getAppRootFor, getUnpackedAppRoot} from 'app/server/lib/places';
|
||||
@@ -69,12 +70,18 @@ export interface ISandboxOptions {
|
||||
* We interact with sandboxes as a separate child process. Data engine work is done
|
||||
* across standard input and output streams from and to this process. We also monitor
|
||||
* and control resource utilization via a distinct control interface.
|
||||
*
|
||||
* More recently, a sandbox may not be a separate OS process, but (for
|
||||
* example) a web worker. In this case, a pair of callbacks (getData and
|
||||
* sendData) replace pipes.
|
||||
*/
|
||||
interface SandboxProcess {
|
||||
child: ChildProcess;
|
||||
export interface SandboxProcess {
|
||||
child?: ChildProcess;
|
||||
control: ISandboxControl;
|
||||
dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data
|
||||
dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data
|
||||
getData?: (cb: (data: any) => void) => void; // use a callback instead of a pipe to get data
|
||||
sendData?: (data: any) => void; // use a callback instead of a pipe to send data
|
||||
}
|
||||
|
||||
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
|
||||
@@ -88,7 +95,7 @@ const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR;
|
||||
|
||||
export class NSandbox implements ISandbox {
|
||||
|
||||
public readonly childProc: ChildProcess;
|
||||
public readonly childProc?: ChildProcess;
|
||||
private _control: ISandboxControl;
|
||||
private _logTimes: boolean;
|
||||
private _exportedFunctions: {[name: string]: SandboxMethod};
|
||||
@@ -101,8 +108,9 @@ export class NSandbox implements ISandbox {
|
||||
private _isWriteClosed = false;
|
||||
|
||||
private _logMeta: log.ILogMeta;
|
||||
private _streamToSandbox: Writable;
|
||||
private _streamToSandbox?: Writable;
|
||||
private _streamFromSandbox: Stream;
|
||||
private _dataToSandbox?: (data: any) => void;
|
||||
private _lastStderr: Uint8Array; // Record last error line seen.
|
||||
|
||||
// Create a unique subdirectory for each sandbox process so they can be replayed separately
|
||||
@@ -129,52 +137,26 @@ export class NSandbox implements ISandbox {
|
||||
this._control = sandboxProcess.control;
|
||||
this.childProc = sandboxProcess.child;
|
||||
|
||||
this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
|
||||
this._logMeta = {sandboxPid: this.childProc?.pid, ...options.logMeta};
|
||||
|
||||
if (options.minimalPipeMode) {
|
||||
log.rawDebug("3-pipe Sandbox started", this._logMeta);
|
||||
if (sandboxProcess.dataToSandboxDescriptor) {
|
||||
this._streamToSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
|
||||
if (this.childProc) {
|
||||
if (options.minimalPipeMode) {
|
||||
this._initializeMinimalPipeMode(sandboxProcess);
|
||||
} else {
|
||||
this._streamToSandbox = this.childProc.stdin!;
|
||||
}
|
||||
if (sandboxProcess.dataFromSandboxDescriptor) {
|
||||
this._streamFromSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
|
||||
} else {
|
||||
this._streamFromSandbox = this.childProc.stdout!;
|
||||
this._initializeFivePipeMode(sandboxProcess);
|
||||
}
|
||||
} else {
|
||||
log.rawDebug("5-pipe Sandbox started", this._logMeta);
|
||||
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
|
||||
throw new Error('cannot override file descriptors in 5 pipe mode');
|
||||
// No child process. In this case, there should be a callback for
|
||||
// receiving and sending data.
|
||||
if (!sandboxProcess.getData) {
|
||||
throw new Error('no way to get data from sandbox');
|
||||
}
|
||||
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
|
||||
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
|
||||
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
|
||||
if (!sandboxProcess.sendData) {
|
||||
throw new Error('no way to send data to sandbox');
|
||||
}
|
||||
sandboxProcess.getData((data) => this._onSandboxData(data));
|
||||
this._dataToSandbox = sandboxProcess.sendData;
|
||||
}
|
||||
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
|
||||
this.childProc.stderr!.on('data', data => {
|
||||
this._lastStderr = data;
|
||||
sandboxStderrLogger(data);
|
||||
});
|
||||
|
||||
this.childProc.on('close', this._onExit.bind(this));
|
||||
this.childProc.on('error', this._onError.bind(this));
|
||||
|
||||
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
|
||||
this._streamFromSandbox.on('end', () => this._onSandboxClose());
|
||||
this._streamFromSandbox.on('error', (err) => {
|
||||
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
|
||||
this._onSandboxClose();
|
||||
});
|
||||
|
||||
this._streamToSandbox.on('error', (err) => {
|
||||
if (!this._isWriteClosed) {
|
||||
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
|
||||
}
|
||||
});
|
||||
|
||||
// On shutdown, shutdown the child process cleanly, and wait for it to exit.
|
||||
shutdown.addCleanupHandler(this, this.shutdown);
|
||||
@@ -203,9 +185,9 @@ export class NSandbox implements ISandbox {
|
||||
|
||||
const result = await new Promise<void>((resolve, reject) => {
|
||||
if (this._isWriteClosed) { resolve(); }
|
||||
this.childProc.on('error', reject);
|
||||
this.childProc.on('close', resolve);
|
||||
this.childProc.on('exit', resolve);
|
||||
this.childProc?.on('error', reject);
|
||||
this.childProc?.on('close', resolve);
|
||||
this.childProc?.on('exit', resolve);
|
||||
this._close();
|
||||
}).finally(() => this._control.close());
|
||||
|
||||
@@ -244,6 +226,82 @@ export class NSandbox implements ISandbox {
|
||||
log.rawDebug('Sandbox memory', {memory, ...this._logMeta});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ready to communicate with a sandbox process using stdin,
|
||||
* stdout, and stderr.
|
||||
*/
|
||||
private _initializeMinimalPipeMode(sandboxProcess: SandboxProcess) {
|
||||
log.rawDebug("3-pipe Sandbox started", this._logMeta);
|
||||
if (!this.childProc) {
|
||||
throw new Error('child process required');
|
||||
}
|
||||
if (sandboxProcess.dataToSandboxDescriptor) {
|
||||
this._streamToSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
|
||||
} else {
|
||||
this._streamToSandbox = this.childProc.stdin!;
|
||||
}
|
||||
if (sandboxProcess.dataFromSandboxDescriptor) {
|
||||
this._streamFromSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
|
||||
} else {
|
||||
this._streamFromSandbox = this.childProc.stdout!;
|
||||
}
|
||||
this._initializeStreamEvents();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get ready to communicate with a sandbox process using stdin,
|
||||
* stdout, and stderr, and two extra FDs. This was a nice way
|
||||
* to have a clean, separate data channel, when supported.
|
||||
*/
|
||||
private _initializeFivePipeMode(sandboxProcess: SandboxProcess) {
|
||||
log.rawDebug("5-pipe Sandbox started", this._logMeta);
|
||||
if (!this.childProc) {
|
||||
throw new Error('child process required');
|
||||
}
|
||||
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
|
||||
throw new Error('cannot override file descriptors in 5 pipe mode');
|
||||
}
|
||||
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
|
||||
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
|
||||
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
|
||||
this._initializeStreamEvents();
|
||||
}
|
||||
|
||||
/**
|
||||
* Set up logging and events on streams to/from a sandbox.
|
||||
*/
|
||||
private _initializeStreamEvents() {
|
||||
if (!this.childProc) {
|
||||
throw new Error('child process required');
|
||||
}
|
||||
if (!this._streamToSandbox) {
|
||||
throw new Error('expected streamToSandbox to be configured');
|
||||
}
|
||||
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
|
||||
this.childProc.stderr!.on('data', data => {
|
||||
this._lastStderr = data;
|
||||
sandboxStderrLogger(data);
|
||||
});
|
||||
|
||||
this.childProc.on('close', this._onExit.bind(this));
|
||||
this.childProc.on('error', this._onError.bind(this));
|
||||
|
||||
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
|
||||
this._streamFromSandbox.on('end', () => this._onSandboxClose());
|
||||
this._streamFromSandbox.on('error', (err) => {
|
||||
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
|
||||
this._onSandboxClose();
|
||||
});
|
||||
|
||||
this._streamToSandbox.on('error', (err) => {
|
||||
if (!this._isWriteClosed) {
|
||||
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async _pyCallWait(funcName: string, startTime: number): Promise<any> {
|
||||
try {
|
||||
return await new Promise((resolve, reject) => {
|
||||
@@ -263,7 +321,7 @@ export class NSandbox implements ISandbox {
|
||||
this._control.prepareToClose();
|
||||
if (!this._isWriteClosed) {
|
||||
// Close the pipe to the sandbox, which should cause the sandbox to exit cleanly.
|
||||
this._streamToSandbox.end();
|
||||
this._streamToSandbox?.end();
|
||||
this._isWriteClosed = true;
|
||||
}
|
||||
}
|
||||
@@ -298,10 +356,17 @@ export class NSandbox implements ISandbox {
|
||||
if (this._recordBuffersDir) {
|
||||
fs.appendFileSync(path.resolve(this._recordBuffersDir, "input"), buf);
|
||||
}
|
||||
return this._streamToSandbox.write(buf);
|
||||
if (this._streamToSandbox) {
|
||||
return this._streamToSandbox.write(buf);
|
||||
} else {
|
||||
if (!this._dataToSandbox) {
|
||||
throw new Error('no way to send data to sandbox');
|
||||
}
|
||||
this._dataToSandbox(buf);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Process a buffer of data received from the sandbox process.
|
||||
*/
|
||||
@@ -422,18 +487,26 @@ function isFlavor(flavor: string): flavor is keyof typeof spawners {
|
||||
* It is ignored by other flavors.
|
||||
*/
|
||||
export class NSandboxCreator implements ISandboxCreator {
|
||||
private _flavor: keyof typeof spawners;
|
||||
private _flavor: string;
|
||||
private _spawner: SpawnFn;
|
||||
private _command?: string;
|
||||
private _preferredPythonVersion?: string;
|
||||
|
||||
public constructor(options: {
|
||||
defaultFlavor: keyof typeof spawners,
|
||||
defaultFlavor: string,
|
||||
command?: string,
|
||||
preferredPythonVersion?: string,
|
||||
}) {
|
||||
const flavor = options.defaultFlavor;
|
||||
if (!isFlavor(flavor)) {
|
||||
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
||||
const variants = create.getSandboxVariants?.();
|
||||
if (!variants?.[flavor]) {
|
||||
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
||||
} else {
|
||||
this._spawner = variants[flavor];
|
||||
}
|
||||
} else {
|
||||
this._spawner = spawners[flavor];
|
||||
}
|
||||
this._flavor = flavor;
|
||||
this._command = options.command;
|
||||
@@ -463,12 +536,12 @@ export class NSandboxCreator implements ISandboxCreator {
|
||||
importDir: options.importMount,
|
||||
...options.sandboxOptions,
|
||||
};
|
||||
return new NSandbox(translatedOptions, spawners[this._flavor]);
|
||||
return new NSandbox(translatedOptions, this._spawner);
|
||||
}
|
||||
}
|
||||
|
||||
// A function that takes sandbox options and starts a sandbox process.
|
||||
type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
|
||||
export type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
|
||||
|
||||
/**
|
||||
* Helper function to run a nacl sandbox. It takes care of most arguments, similarly to
|
||||
@@ -750,7 +823,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess {
|
||||
...getWrappingEnv(options),
|
||||
};
|
||||
const command = findPython(options.command, options.preferredPythonVersion);
|
||||
const realPath = fs.realpathSync(command);
|
||||
const realPath = realpathSync(command);
|
||||
log.rawDebug("macSandboxExec found a python", {...options.logMeta, command: realPath});
|
||||
|
||||
// Prepare sandbox profile
|
||||
@@ -868,11 +941,11 @@ function getAbsolutePaths(options: ISandboxOptions) {
|
||||
// Get path to sandbox directory - this is a little idiosyncratic to work well
|
||||
// in grist-core. It is important to use real paths since we may be viewing
|
||||
// the file system through a narrow window in a container.
|
||||
const sandboxDir = path.join(fs.realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
|
||||
const sandboxDir = path.join(realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
|
||||
'..');
|
||||
// Copy plugin options, and then make them absolute.
|
||||
if (options.importDir) {
|
||||
options.importDir = fs.realpathSync(options.importDir);
|
||||
options.importDir = realpathSync(options.importDir);
|
||||
}
|
||||
return {
|
||||
sandboxDir,
|
||||
@@ -976,9 +1049,6 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
|
||||
const flavor = parts[parts.length - 1];
|
||||
const version = parts.length === 2 ? parts[0] : '*';
|
||||
if (preferredPythonVersion === version || version === '*' || !preferredPythonVersion) {
|
||||
if (!isFlavor(flavor)) {
|
||||
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
||||
}
|
||||
const creator = new NSandboxCreator({
|
||||
defaultFlavor: flavor,
|
||||
command: process.env['GRIST_SANDBOX' + (preferredPythonVersion||'')] ||
|
||||
@@ -990,3 +1060,16 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
|
||||
}
|
||||
throw new Error('Failed to create a sandbox');
|
||||
}
|
||||
|
||||
/**
|
||||
* The realpath function may not be available, just return the
|
||||
* path unchanged if it is not. Specifically, this happens when
|
||||
* compiled for use in a browser environment.
|
||||
*/
|
||||
function realpathSync(src: string) {
|
||||
try {
|
||||
return fs.realpathSync(src);
|
||||
} catch (e) {
|
||||
return src;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user