|
|
|
|
@ -3,6 +3,7 @@
|
|
|
|
|
*/
|
|
|
|
|
import {arrayToString} from 'app/common/arrayToString';
|
|
|
|
|
import * as marshal from 'app/common/marshal';
|
|
|
|
|
import {create} from 'app/server/lib/create';
|
|
|
|
|
import {ISandbox, ISandboxCreationOptions, ISandboxCreator} from 'app/server/lib/ISandbox';
|
|
|
|
|
import log from 'app/server/lib/log';
|
|
|
|
|
import {getAppRoot, getAppRootFor, getUnpackedAppRoot} from 'app/server/lib/places';
|
|
|
|
|
@ -69,12 +70,18 @@ export interface ISandboxOptions {
|
|
|
|
|
* We interact with sandboxes as a separate child process. Data engine work is done
|
|
|
|
|
* across standard input and output streams from and to this process. We also monitor
|
|
|
|
|
* and control resource utilization via a distinct control interface.
|
|
|
|
|
*
|
|
|
|
|
* More recently, a sandbox may not be a separate OS process, but (for
|
|
|
|
|
* example) a web worker. In this case, a pair of callbacks (getData and
|
|
|
|
|
* sendData) replace pipes.
|
|
|
|
|
*/
|
|
|
|
|
interface SandboxProcess {
|
|
|
|
|
child: ChildProcess;
|
|
|
|
|
export interface SandboxProcess {
|
|
|
|
|
child?: ChildProcess;
|
|
|
|
|
control: ISandboxControl;
|
|
|
|
|
dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data
|
|
|
|
|
dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data
|
|
|
|
|
getData?: (cb: (data: any) => void) => void; // use a callback instead of a pipe to get data
|
|
|
|
|
sendData?: (data: any) => void; // use a callback instead of a pipe to send data
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
|
|
|
|
|
@ -88,7 +95,7 @@ const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR;
|
|
|
|
|
|
|
|
|
|
export class NSandbox implements ISandbox {
|
|
|
|
|
|
|
|
|
|
public readonly childProc: ChildProcess;
|
|
|
|
|
public readonly childProc?: ChildProcess;
|
|
|
|
|
private _control: ISandboxControl;
|
|
|
|
|
private _logTimes: boolean;
|
|
|
|
|
private _exportedFunctions: {[name: string]: SandboxMethod};
|
|
|
|
|
@ -101,8 +108,9 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
private _isWriteClosed = false;
|
|
|
|
|
|
|
|
|
|
private _logMeta: log.ILogMeta;
|
|
|
|
|
private _streamToSandbox: Writable;
|
|
|
|
|
private _streamToSandbox?: Writable;
|
|
|
|
|
private _streamFromSandbox: Stream;
|
|
|
|
|
private _dataToSandbox?: (data: any) => void;
|
|
|
|
|
private _lastStderr: Uint8Array; // Record last error line seen.
|
|
|
|
|
|
|
|
|
|
// Create a unique subdirectory for each sandbox process so they can be replayed separately
|
|
|
|
|
@ -129,52 +137,26 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
this._control = sandboxProcess.control;
|
|
|
|
|
this.childProc = sandboxProcess.child;
|
|
|
|
|
|
|
|
|
|
this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
|
|
|
|
|
this._logMeta = {sandboxPid: this.childProc?.pid, ...options.logMeta};
|
|
|
|
|
|
|
|
|
|
if (options.minimalPipeMode) {
|
|
|
|
|
log.rawDebug("3-pipe Sandbox started", this._logMeta);
|
|
|
|
|
if (sandboxProcess.dataToSandboxDescriptor) {
|
|
|
|
|
this._streamToSandbox =
|
|
|
|
|
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
|
|
|
|
|
} else {
|
|
|
|
|
this._streamToSandbox = this.childProc.stdin!;
|
|
|
|
|
}
|
|
|
|
|
if (sandboxProcess.dataFromSandboxDescriptor) {
|
|
|
|
|
this._streamFromSandbox =
|
|
|
|
|
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
|
|
|
|
|
if (this.childProc) {
|
|
|
|
|
if (options.minimalPipeMode) {
|
|
|
|
|
this._initializeMinimalPipeMode(sandboxProcess);
|
|
|
|
|
} else {
|
|
|
|
|
this._streamFromSandbox = this.childProc.stdout!;
|
|
|
|
|
this._initializeFivePipeMode(sandboxProcess);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
log.rawDebug("5-pipe Sandbox started", this._logMeta);
|
|
|
|
|
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
|
|
|
|
|
throw new Error('cannot override file descriptors in 5 pipe mode');
|
|
|
|
|
// No child process. In this case, there should be a callback for
|
|
|
|
|
// receiving and sending data.
|
|
|
|
|
if (!sandboxProcess.getData) {
|
|
|
|
|
throw new Error('no way to get data from sandbox');
|
|
|
|
|
}
|
|
|
|
|
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
|
|
|
|
|
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
|
|
|
|
|
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
|
|
|
|
|
}
|
|
|
|
|
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
|
|
|
|
|
this.childProc.stderr!.on('data', data => {
|
|
|
|
|
this._lastStderr = data;
|
|
|
|
|
sandboxStderrLogger(data);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
this.childProc.on('close', this._onExit.bind(this));
|
|
|
|
|
this.childProc.on('error', this._onError.bind(this));
|
|
|
|
|
|
|
|
|
|
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
|
|
|
|
|
this._streamFromSandbox.on('end', () => this._onSandboxClose());
|
|
|
|
|
this._streamFromSandbox.on('error', (err) => {
|
|
|
|
|
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
|
|
|
|
|
this._onSandboxClose();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
this._streamToSandbox.on('error', (err) => {
|
|
|
|
|
if (!this._isWriteClosed) {
|
|
|
|
|
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
|
|
|
|
|
if (!sandboxProcess.sendData) {
|
|
|
|
|
throw new Error('no way to send data to sandbox');
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
sandboxProcess.getData((data) => this._onSandboxData(data));
|
|
|
|
|
this._dataToSandbox = sandboxProcess.sendData;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// On shutdown, shutdown the child process cleanly, and wait for it to exit.
|
|
|
|
|
shutdown.addCleanupHandler(this, this.shutdown);
|
|
|
|
|
@ -203,9 +185,9 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
|
|
|
|
|
const result = await new Promise<void>((resolve, reject) => {
|
|
|
|
|
if (this._isWriteClosed) { resolve(); }
|
|
|
|
|
this.childProc.on('error', reject);
|
|
|
|
|
this.childProc.on('close', resolve);
|
|
|
|
|
this.childProc.on('exit', resolve);
|
|
|
|
|
this.childProc?.on('error', reject);
|
|
|
|
|
this.childProc?.on('close', resolve);
|
|
|
|
|
this.childProc?.on('exit', resolve);
|
|
|
|
|
this._close();
|
|
|
|
|
}).finally(() => this._control.close());
|
|
|
|
|
|
|
|
|
|
@ -244,6 +226,82 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
log.rawDebug('Sandbox memory', {memory, ...this._logMeta});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get ready to communicate with a sandbox process using stdin,
|
|
|
|
|
* stdout, and stderr.
|
|
|
|
|
*/
|
|
|
|
|
private _initializeMinimalPipeMode(sandboxProcess: SandboxProcess) {
|
|
|
|
|
log.rawDebug("3-pipe Sandbox started", this._logMeta);
|
|
|
|
|
if (!this.childProc) {
|
|
|
|
|
throw new Error('child process required');
|
|
|
|
|
}
|
|
|
|
|
if (sandboxProcess.dataToSandboxDescriptor) {
|
|
|
|
|
this._streamToSandbox =
|
|
|
|
|
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
|
|
|
|
|
} else {
|
|
|
|
|
this._streamToSandbox = this.childProc.stdin!;
|
|
|
|
|
}
|
|
|
|
|
if (sandboxProcess.dataFromSandboxDescriptor) {
|
|
|
|
|
this._streamFromSandbox =
|
|
|
|
|
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
|
|
|
|
|
} else {
|
|
|
|
|
this._streamFromSandbox = this.childProc.stdout!;
|
|
|
|
|
}
|
|
|
|
|
this._initializeStreamEvents();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get ready to communicate with a sandbox process using stdin,
|
|
|
|
|
* stdout, and stderr, and two extra FDs. This was a nice way
|
|
|
|
|
* to have a clean, separate data channel, when supported.
|
|
|
|
|
*/
|
|
|
|
|
private _initializeFivePipeMode(sandboxProcess: SandboxProcess) {
|
|
|
|
|
log.rawDebug("5-pipe Sandbox started", this._logMeta);
|
|
|
|
|
if (!this.childProc) {
|
|
|
|
|
throw new Error('child process required');
|
|
|
|
|
}
|
|
|
|
|
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
|
|
|
|
|
throw new Error('cannot override file descriptors in 5 pipe mode');
|
|
|
|
|
}
|
|
|
|
|
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
|
|
|
|
|
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
|
|
|
|
|
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
|
|
|
|
|
this._initializeStreamEvents();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set up logging and events on streams to/from a sandbox.
|
|
|
|
|
*/
|
|
|
|
|
private _initializeStreamEvents() {
|
|
|
|
|
if (!this.childProc) {
|
|
|
|
|
throw new Error('child process required');
|
|
|
|
|
}
|
|
|
|
|
if (!this._streamToSandbox) {
|
|
|
|
|
throw new Error('expected streamToSandbox to be configured');
|
|
|
|
|
}
|
|
|
|
|
const sandboxStderrLogger = sandboxUtil.makeLinePrefixer('Sandbox stderr: ', this._logMeta);
|
|
|
|
|
this.childProc.stderr!.on('data', data => {
|
|
|
|
|
this._lastStderr = data;
|
|
|
|
|
sandboxStderrLogger(data);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
this.childProc.on('close', this._onExit.bind(this));
|
|
|
|
|
this.childProc.on('error', this._onError.bind(this));
|
|
|
|
|
|
|
|
|
|
this._streamFromSandbox.on('data', (data) => this._onSandboxData(data));
|
|
|
|
|
this._streamFromSandbox.on('end', () => this._onSandboxClose());
|
|
|
|
|
this._streamFromSandbox.on('error', (err) => {
|
|
|
|
|
log.rawError(`Sandbox error reading: ${err}`, this._logMeta);
|
|
|
|
|
this._onSandboxClose();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
this._streamToSandbox.on('error', (err) => {
|
|
|
|
|
if (!this._isWriteClosed) {
|
|
|
|
|
log.rawError(`Sandbox error writing: ${err}`, this._logMeta);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async _pyCallWait(funcName: string, startTime: number): Promise<any> {
|
|
|
|
|
try {
|
|
|
|
|
return await new Promise((resolve, reject) => {
|
|
|
|
|
@ -263,7 +321,7 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
this._control.prepareToClose();
|
|
|
|
|
if (!this._isWriteClosed) {
|
|
|
|
|
// Close the pipe to the sandbox, which should cause the sandbox to exit cleanly.
|
|
|
|
|
this._streamToSandbox.end();
|
|
|
|
|
this._streamToSandbox?.end();
|
|
|
|
|
this._isWriteClosed = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
@ -298,10 +356,17 @@ export class NSandbox implements ISandbox {
|
|
|
|
|
if (this._recordBuffersDir) {
|
|
|
|
|
fs.appendFileSync(path.resolve(this._recordBuffersDir, "input"), buf);
|
|
|
|
|
}
|
|
|
|
|
return this._streamToSandbox.write(buf);
|
|
|
|
|
if (this._streamToSandbox) {
|
|
|
|
|
return this._streamToSandbox.write(buf);
|
|
|
|
|
} else {
|
|
|
|
|
if (!this._dataToSandbox) {
|
|
|
|
|
throw new Error('no way to send data to sandbox');
|
|
|
|
|
}
|
|
|
|
|
this._dataToSandbox(buf);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Process a buffer of data received from the sandbox process.
|
|
|
|
|
*/
|
|
|
|
|
@ -422,18 +487,26 @@ function isFlavor(flavor: string): flavor is keyof typeof spawners {
|
|
|
|
|
* It is ignored by other flavors.
|
|
|
|
|
*/
|
|
|
|
|
export class NSandboxCreator implements ISandboxCreator {
|
|
|
|
|
private _flavor: keyof typeof spawners;
|
|
|
|
|
private _flavor: string;
|
|
|
|
|
private _spawner: SpawnFn;
|
|
|
|
|
private _command?: string;
|
|
|
|
|
private _preferredPythonVersion?: string;
|
|
|
|
|
|
|
|
|
|
public constructor(options: {
|
|
|
|
|
defaultFlavor: keyof typeof spawners,
|
|
|
|
|
defaultFlavor: string,
|
|
|
|
|
command?: string,
|
|
|
|
|
preferredPythonVersion?: string,
|
|
|
|
|
}) {
|
|
|
|
|
const flavor = options.defaultFlavor;
|
|
|
|
|
if (!isFlavor(flavor)) {
|
|
|
|
|
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
|
|
|
|
const variants = create.getSandboxVariants?.();
|
|
|
|
|
if (!variants?.[flavor]) {
|
|
|
|
|
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
|
|
|
|
} else {
|
|
|
|
|
this._spawner = variants[flavor];
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
this._spawner = spawners[flavor];
|
|
|
|
|
}
|
|
|
|
|
this._flavor = flavor;
|
|
|
|
|
this._command = options.command;
|
|
|
|
|
@ -463,12 +536,12 @@ export class NSandboxCreator implements ISandboxCreator {
|
|
|
|
|
importDir: options.importMount,
|
|
|
|
|
...options.sandboxOptions,
|
|
|
|
|
};
|
|
|
|
|
return new NSandbox(translatedOptions, spawners[this._flavor]);
|
|
|
|
|
return new NSandbox(translatedOptions, this._spawner);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A function that takes sandbox options and starts a sandbox process.
|
|
|
|
|
type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
|
|
|
|
|
export type SpawnFn = (options: ISandboxOptions) => SandboxProcess;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Helper function to run a nacl sandbox. It takes care of most arguments, similarly to
|
|
|
|
|
@ -750,7 +823,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess {
|
|
|
|
|
...getWrappingEnv(options),
|
|
|
|
|
};
|
|
|
|
|
const command = findPython(options.command, options.preferredPythonVersion);
|
|
|
|
|
const realPath = fs.realpathSync(command);
|
|
|
|
|
const realPath = realpathSync(command);
|
|
|
|
|
log.rawDebug("macSandboxExec found a python", {...options.logMeta, command: realPath});
|
|
|
|
|
|
|
|
|
|
// Prepare sandbox profile
|
|
|
|
|
@ -868,11 +941,11 @@ function getAbsolutePaths(options: ISandboxOptions) {
|
|
|
|
|
// Get path to sandbox directory - this is a little idiosyncratic to work well
|
|
|
|
|
// in grist-core. It is important to use real paths since we may be viewing
|
|
|
|
|
// the file system through a narrow window in a container.
|
|
|
|
|
const sandboxDir = path.join(fs.realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
|
|
|
|
|
const sandboxDir = path.join(realpathSync(path.join(process.cwd(), 'sandbox', 'grist')),
|
|
|
|
|
'..');
|
|
|
|
|
// Copy plugin options, and then make them absolute.
|
|
|
|
|
if (options.importDir) {
|
|
|
|
|
options.importDir = fs.realpathSync(options.importDir);
|
|
|
|
|
options.importDir = realpathSync(options.importDir);
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
sandboxDir,
|
|
|
|
|
@ -976,9 +1049,6 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
|
|
|
|
|
const flavor = parts[parts.length - 1];
|
|
|
|
|
const version = parts.length === 2 ? parts[0] : '*';
|
|
|
|
|
if (preferredPythonVersion === version || version === '*' || !preferredPythonVersion) {
|
|
|
|
|
if (!isFlavor(flavor)) {
|
|
|
|
|
throw new Error(`Unrecognized sandbox flavor: ${flavor}`);
|
|
|
|
|
}
|
|
|
|
|
const creator = new NSandboxCreator({
|
|
|
|
|
defaultFlavor: flavor,
|
|
|
|
|
command: process.env['GRIST_SANDBOX' + (preferredPythonVersion||'')] ||
|
|
|
|
|
@ -990,3 +1060,16 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
|
|
|
|
|
}
|
|
|
|
|
throw new Error('Failed to create a sandbox');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* The realpath function may not be available, just return the
|
|
|
|
|
* path unchanged if it is not. Specifically, this happens when
|
|
|
|
|
* compiled for use in a browser environment.
|
|
|
|
|
*/
|
|
|
|
|
function realpathSync(src: string) {
|
|
|
|
|
try {
|
|
|
|
|
return fs.realpathSync(src);
|
|
|
|
|
} catch (e) {
|
|
|
|
|
return src;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
Paul Fitzpatrick
11 months ago
committed by
GitHub