@ -3,6 +3,7 @@
* /
import { arrayToString } from 'app/common/arrayToString' ;
import * as marshal from 'app/common/marshal' ;
import { create } from 'app/server/lib/create' ;
import { ISandbox , ISandboxCreationOptions , ISandboxCreator } from 'app/server/lib/ISandbox' ;
import log from 'app/server/lib/log' ;
import { getAppRoot , getAppRootFor , getUnpackedAppRoot } from 'app/server/lib/places' ;
@ -69,12 +70,18 @@ export interface ISandboxOptions {
* We interact with sandboxes as a separate child process . Data engine work is done
* across standard input and output streams from and to this process . We also monitor
* and control resource utilization via a distinct control interface .
*
* More recently , a sandbox may not be a separate OS process , but ( for
* example ) a web worker . In this case , a pair of callbacks ( getData and
* sendData ) replace pipes .
* /
interface SandboxProcess {
child : ChildProcess ;
export interface SandboxProcess {
child ? : ChildProcess ;
control : ISandboxControl ;
dataToSandboxDescriptor? : number ; // override sandbox's 'stdin' for data
dataFromSandboxDescriptor? : number ; // override sandbox's 'stdout' for data
getData ? : ( cb : ( data : any ) = > void ) = > void ; // use a callback instead of a pipe to get data
sendData ? : ( data : any ) = > void ; // use a callback instead of a pipe to send data
}
type ResolveRejectPair = [ ( value? : any ) = > void , ( reason? : unknown ) = > void ] ;
@ -88,7 +95,7 @@ const recordBuffersRoot = process.env.RECORD_SANDBOX_BUFFERS_DIR;
export class NSandbox implements ISandbox {
public readonly childProc : ChildProcess ;
public readonly childProc ? : ChildProcess ;
private _control : ISandboxControl ;
private _logTimes : boolean ;
private _exportedFunctions : { [ name : string ] : SandboxMethod } ;
@ -101,8 +108,9 @@ export class NSandbox implements ISandbox {
private _isWriteClosed = false ;
private _logMeta : log.ILogMeta ;
private _streamToSandbox : Writable ;
private _streamToSandbox ? : Writable ;
private _streamFromSandbox : Stream ;
private _dataToSandbox ? : ( data : any ) = > void ;
private _lastStderr : Uint8Array ; // Record last error line seen.
// Create a unique subdirectory for each sandbox process so they can be replayed separately
@ -129,52 +137,26 @@ export class NSandbox implements ISandbox {
this . _control = sandboxProcess . control ;
this . childProc = sandboxProcess . child ;
this . _logMeta = { sandboxPid : this.childProc .pid, . . . options . logMeta } ;
this . _logMeta = { sandboxPid : this.childProc ? .pid, . . . options . logMeta } ;
if ( options . minimalPipeMode ) {
log . rawDebug ( "3-pipe Sandbox started" , this . _logMeta ) ;
if ( sandboxProcess . dataToSandboxDescriptor ) {
this . _streamToSandbox =
( this . childProc . stdio as Stream [ ] ) [ sandboxProcess . dataToSandboxDescriptor ] as Writable ;
} else {
this . _streamToSandbox = this . childProc . stdin ! ;
}
if ( sandboxProcess . dataFromSandboxDescriptor ) {
this . _streamFromSandbox =
( this . childProc . stdio as Stream [ ] ) [ sandboxProcess . dataFromSandboxDescriptor ] ;
if ( this . childProc ) {
if ( options . minimalPipeMode ) {
this . _initializeMinimalPipeMode ( sandboxProcess ) ;
} else {
this . _ streamFromSandbox = this . childProc . stdout ! ;
this . _initializeFivePipeMode ( sandboxProcess ) ;
}
} else {
log . rawDebug ( "5-pipe Sandbox started" , this . _logMeta ) ;
if ( sandboxProcess . dataFromSandboxDescriptor || sandboxProcess . dataToSandboxDescriptor ) {
throw new Error ( 'cannot override file descriptors in 5 pipe mode' ) ;
// No child process. In this case, there should be a callback for
// receiving and sending data.
if ( ! sandboxProcess . getData ) {
throw new Error ( 'no way to get data from sandbox' ) ;
}
this . _streamToSandbox = ( this . childProc . stdio as Stream [ ] ) [ 3 ] as Writable ;
this . _streamFromSandbox = ( this . childProc . stdio as Stream [ ] ) [ 4 ] ;
this . childProc . stdout ! . on ( 'data' , sandboxUtil . makeLinePrefixer ( 'Sandbox stdout: ' , this . _logMeta ) ) ;
}
const sandboxStderrLogger = sandboxUtil . makeLinePrefixer ( 'Sandbox stderr: ' , this . _logMeta ) ;
this . childProc . stderr ! . on ( 'data' , data = > {
this . _lastStderr = data ;
sandboxStderrLogger ( data ) ;
} ) ;
this . childProc . on ( 'close' , this . _onExit . bind ( this ) ) ;
this . childProc . on ( 'error' , this . _onError . bind ( this ) ) ;
this . _streamFromSandbox . on ( 'data' , ( data ) = > this . _onSandboxData ( data ) ) ;
this . _streamFromSandbox . on ( 'end' , ( ) = > this . _onSandboxClose ( ) ) ;
this . _streamFromSandbox . on ( 'error' , ( err ) = > {
log . rawError ( ` Sandbox error reading: ${ err } ` , this . _logMeta ) ;
this . _onSandboxClose ( ) ;
} ) ;
this . _streamToSandbox . on ( 'error' , ( err ) = > {
if ( ! this . _isWriteClosed ) {
log . rawError ( ` Sandbox error writing: ${ err } ` , this . _logMeta ) ;
if ( ! sandboxProcess . sendData ) {
throw new Error ( 'no way to send data to sandbox' ) ;
}
} ) ;
sandboxProcess . getData ( ( data ) = > this . _onSandboxData ( data ) ) ;
this . _dataToSandbox = sandboxProcess . sendData ;
}
// On shutdown, shutdown the child process cleanly, and wait for it to exit.
shutdown . addCleanupHandler ( this , this . shutdown ) ;
@ -203,9 +185,9 @@ export class NSandbox implements ISandbox {
const result = await new Promise < void > ( ( resolve , reject ) = > {
if ( this . _isWriteClosed ) { resolve ( ) ; }
this . childProc . on ( 'error' , reject ) ;
this . childProc . on ( 'close' , resolve ) ;
this . childProc . on ( 'exit' , resolve ) ;
this . childProc ? . on ( 'error' , reject ) ;
this . childProc ? . on ( 'close' , resolve ) ;
this . childProc ? . on ( 'exit' , resolve ) ;
this . _close ( ) ;
} ) . finally ( ( ) = > this . _control . close ( ) ) ;
@ -244,6 +226,82 @@ export class NSandbox implements ISandbox {
log . rawDebug ( 'Sandbox memory' , { memory , . . . this . _logMeta } ) ;
}
/ * *
* Get ready to communicate with a sandbox process using stdin ,
* stdout , and stderr .
* /
private _initializeMinimalPipeMode ( sandboxProcess : SandboxProcess ) {
log . rawDebug ( "3-pipe Sandbox started" , this . _logMeta ) ;
if ( ! this . childProc ) {
throw new Error ( 'child process required' ) ;
}
if ( sandboxProcess . dataToSandboxDescriptor ) {
this . _streamToSandbox =
( this . childProc . stdio as Stream [ ] ) [ sandboxProcess . dataToSandboxDescriptor ] as Writable ;
} else {
this . _streamToSandbox = this . childProc . stdin ! ;
}
if ( sandboxProcess . dataFromSandboxDescriptor ) {
this . _streamFromSandbox =
( this . childProc . stdio as Stream [ ] ) [ sandboxProcess . dataFromSandboxDescriptor ] ;
} else {
this . _streamFromSandbox = this . childProc . stdout ! ;
}
this . _initializeStreamEvents ( ) ;
}
/ * *
* Get ready to communicate with a sandbox process using stdin ,
* stdout , and stderr , and two extra FDs . This was a nice way
* to have a clean , separate data channel , when supported .
* /
private _initializeFivePipeMode ( sandboxProcess : SandboxProcess ) {
log . rawDebug ( "5-pipe Sandbox started" , this . _logMeta ) ;
if ( ! this . childProc ) {
throw new Error ( 'child process required' ) ;
}
if ( sandboxProcess . dataFromSandboxDescriptor || sandboxProcess . dataToSandboxDescriptor ) {
throw new Error ( 'cannot override file descriptors in 5 pipe mode' ) ;
}
this . _streamToSandbox = ( this . childProc . stdio as Stream [ ] ) [ 3 ] as Writable ;
this . _streamFromSandbox = ( this . childProc . stdio as Stream [ ] ) [ 4 ] ;
this . childProc . stdout ! . on ( 'data' , sandboxUtil . makeLinePrefixer ( 'Sandbox stdout: ' , this . _logMeta ) ) ;
this . _initializeStreamEvents ( ) ;
}
/ * *
* Set up logging and events on streams to / from a sandbox .
* /
private _initializeStreamEvents() {
if ( ! this . childProc ) {
throw new Error ( 'child process required' ) ;
}
if ( ! this . _streamToSandbox ) {
throw new Error ( 'expected streamToSandbox to be configured' ) ;
}
const sandboxStderrLogger = sandboxUtil . makeLinePrefixer ( 'Sandbox stderr: ' , this . _logMeta ) ;
this . childProc . stderr ! . on ( 'data' , data = > {
this . _lastStderr = data ;
sandboxStderrLogger ( data ) ;
} ) ;
this . childProc . on ( 'close' , this . _onExit . bind ( this ) ) ;
this . childProc . on ( 'error' , this . _onError . bind ( this ) ) ;
this . _streamFromSandbox . on ( 'data' , ( data ) = > this . _onSandboxData ( data ) ) ;
this . _streamFromSandbox . on ( 'end' , ( ) = > this . _onSandboxClose ( ) ) ;
this . _streamFromSandbox . on ( 'error' , ( err ) = > {
log . rawError ( ` Sandbox error reading: ${ err } ` , this . _logMeta ) ;
this . _onSandboxClose ( ) ;
} ) ;
this . _streamToSandbox . on ( 'error' , ( err ) = > {
if ( ! this . _isWriteClosed ) {
log . rawError ( ` Sandbox error writing: ${ err } ` , this . _logMeta ) ;
}
} ) ;
}
private async _pyCallWait ( funcName : string , startTime : number ) : Promise < any > {
try {
return await new Promise ( ( resolve , reject ) = > {
@ -263,7 +321,7 @@ export class NSandbox implements ISandbox {
this . _control . prepareToClose ( ) ;
if ( ! this . _isWriteClosed ) {
// Close the pipe to the sandbox, which should cause the sandbox to exit cleanly.
this . _streamToSandbox . end ( ) ;
this . _streamToSandbox ? . end ( ) ;
this . _isWriteClosed = true ;
}
}
@ -298,10 +356,17 @@ export class NSandbox implements ISandbox {
if ( this . _recordBuffersDir ) {
fs . appendFileSync ( path . resolve ( this . _recordBuffersDir , "input" ) , buf ) ;
}
return this . _streamToSandbox . write ( buf ) ;
if ( this . _streamToSandbox ) {
return this . _streamToSandbox . write ( buf ) ;
} else {
if ( ! this . _dataToSandbox ) {
throw new Error ( 'no way to send data to sandbox' ) ;
}
this . _dataToSandbox ( buf ) ;
return true ;
}
}
/ * *
* Process a buffer of data received from the sandbox process .
* /
@ -422,18 +487,26 @@ function isFlavor(flavor: string): flavor is keyof typeof spawners {
* It is ignored by other flavors .
* /
export class NSandboxCreator implements ISandboxCreator {
private _flavor : keyof typeof spawners ;
private _flavor : string ;
private _spawner : SpawnFn ;
private _command? : string ;
private _preferredPythonVersion? : string ;
public constructor ( options : {
defaultFlavor : keyof typeof spawners ,
defaultFlavor : string ,
command? : string ,
preferredPythonVersion? : string ,
} ) {
const flavor = options . defaultFlavor ;
if ( ! isFlavor ( flavor ) ) {
throw new Error ( ` Unrecognized sandbox flavor: ${ flavor } ` ) ;
const variants = create . getSandboxVariants ? . ( ) ;
if ( ! variants ? . [ flavor ] ) {
throw new Error ( ` Unrecognized sandbox flavor: ${ flavor } ` ) ;
} else {
this . _spawner = variants [ flavor ] ;
}
} else {
this . _spawner = spawners [ flavor ] ;
}
this . _flavor = flavor ;
this . _command = options . command ;
@ -463,12 +536,12 @@ export class NSandboxCreator implements ISandboxCreator {
importDir : options.importMount ,
. . . options . sandboxOptions ,
} ;
return new NSandbox ( translatedOptions , spawners [ this . _flavor ] ) ;
return new NSandbox ( translatedOptions , this . _spawner ) ;
}
}
// A function that takes sandbox options and starts a sandbox process.
type SpawnFn = ( options : ISandboxOptions ) = > SandboxProcess ;
export type SpawnFn = ( options : ISandboxOptions ) = > SandboxProcess ;
/ * *
* Helper function to run a nacl sandbox . It takes care of most arguments , similarly to
@ -750,7 +823,7 @@ function macSandboxExec(options: ISandboxOptions): SandboxProcess {
. . . getWrappingEnv ( options ) ,
} ;
const command = findPython ( options . command , options . preferredPythonVersion ) ;
const realPath = fs. realpathSync( command ) ;
const realPath = realpathSync( command ) ;
log . rawDebug ( "macSandboxExec found a python" , { . . . options . logMeta , command : realPath } ) ;
// Prepare sandbox profile
@ -868,11 +941,11 @@ function getAbsolutePaths(options: ISandboxOptions) {
// Get path to sandbox directory - this is a little idiosyncratic to work well
// in grist-core. It is important to use real paths since we may be viewing
// the file system through a narrow window in a container.
const sandboxDir = path . join ( fs. realpathSync( path . join ( process . cwd ( ) , 'sandbox' , 'grist' ) ) ,
const sandboxDir = path . join ( realpathSync( path . join ( process . cwd ( ) , 'sandbox' , 'grist' ) ) ,
'..' ) ;
// Copy plugin options, and then make them absolute.
if ( options . importDir ) {
options . importDir = fs. realpathSync( options . importDir ) ;
options . importDir = realpathSync( options . importDir ) ;
}
return {
sandboxDir ,
@ -976,9 +1049,6 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
const flavor = parts [ parts . length - 1 ] ;
const version = parts . length === 2 ? parts [ 0 ] : '*' ;
if ( preferredPythonVersion === version || version === '*' || ! preferredPythonVersion ) {
if ( ! isFlavor ( flavor ) ) {
throw new Error ( ` Unrecognized sandbox flavor: ${ flavor } ` ) ;
}
const creator = new NSandboxCreator ( {
defaultFlavor : flavor ,
command : process.env [ 'GRIST_SANDBOX' + ( preferredPythonVersion || '' ) ] ||
@ -990,3 +1060,16 @@ export function createSandbox(defaultFlavorSpec: string, options: ISandboxCreati
}
throw new Error ( 'Failed to create a sandbox' ) ;
}
/ * *
* The realpath function may not be available , just return the
* path unchanged if it is not . Specifically , this happens when
* compiled for use in a browser environment .
* /
function realpathSync ( src : string ) {
try {
return fs . realpathSync ( src ) ;
} catch ( e ) {
return src ;
}
}