mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
add a pyodide-based "sandbox" flavor (#437)
This adds a new `GRIST_SANDBOX_FLAVOR=pyodide` option where the version of Python used for the data engine is wasm, and so can be run by node like the rest of the back end. It still runs as a separate process. There are a few small version changes made to packages to avoid various awkwardnesses present in the current versions. All existing tests pass. This is very experimental. To use, you'll need something with a bash shell and make. First do: ``` cd sandbox/pyodide make setup # README.md and Makefile have details cd .. ``` Then running Grist as: ``` GRIST_SANDBOX_FLAVOR=pyodide yarn start ``` should work. Adding a formula with content: ``` import sys; return sys.version ``` should return a different Python version than other sandboxes. The motivation for this work is to have a form of sandboxing that will work on Windows for Grist Electron (for Linux we have gvisor/runsc, for Mac we have sandbox-exec, but I haven't found anything comparable for Windows). It also brings a back-end-free version of Grist a bit closer, for use-cases where that would make sense - such as serving a report (in the form of a Grist document) on a static site.
This commit is contained in:
Paul Fitzpatrick
GitHub
parent
a1259139f6
commit
66643a5e6b
@ -13,3 +13,4 @@
|
||||
!plugins
|
||||
!test
|
||||
!ext
|
||||
**/_build
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -73,3 +73,5 @@ jspm_packages/
|
||||
|
||||
# dotenv environment variables file
|
||||
.env
|
||||
|
||||
**/_build
|
||||
|
||||
@ -15,7 +15,7 @@ import {
|
||||
} from 'app/server/lib/SandboxControl';
|
||||
import * as sandboxUtil from 'app/server/lib/sandboxUtil';
|
||||
import * as shutdown from 'app/server/lib/shutdown';
|
||||
import {ChildProcess, spawn} from 'child_process';
|
||||
import {ChildProcess, fork, spawn} from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as _ from 'lodash';
|
||||
import * as path from 'path';
|
||||
@ -73,6 +73,8 @@ export interface ISandboxOptions {
|
||||
interface SandboxProcess {
|
||||
child: ChildProcess;
|
||||
control: ISandboxControl;
|
||||
dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data
|
||||
dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data
|
||||
}
|
||||
|
||||
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
|
||||
@ -131,10 +133,23 @@ export class NSandbox implements ISandbox {
|
||||
|
||||
if (options.minimalPipeMode) {
|
||||
log.rawDebug("3-pipe Sandbox started", this._logMeta);
|
||||
this._streamToSandbox = this.childProc.stdin!;
|
||||
this._streamFromSandbox = this.childProc.stdout!;
|
||||
if (sandboxProcess.dataToSandboxDescriptor) {
|
||||
this._streamToSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable;
|
||||
} else {
|
||||
this._streamToSandbox = this.childProc.stdin!;
|
||||
}
|
||||
if (sandboxProcess.dataFromSandboxDescriptor) {
|
||||
this._streamFromSandbox =
|
||||
(this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor];
|
||||
} else {
|
||||
this._streamFromSandbox = this.childProc.stdout!;
|
||||
}
|
||||
} else {
|
||||
log.rawDebug("5-pipe Sandbox started", this._logMeta);
|
||||
if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) {
|
||||
throw new Error('cannot override file descriptors in 5 pipe mode');
|
||||
}
|
||||
this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable;
|
||||
this._streamFromSandbox = (this.childProc.stdio as Stream[])[4];
|
||||
this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta));
|
||||
@ -206,10 +221,17 @@ export class NSandbox implements ISandbox {
|
||||
* @param args Arguments to pass to the given function.
|
||||
* @returns A promise for the return value from the Python function.
|
||||
*/
|
||||
public pyCall(funcName: string, ...varArgs: unknown[]): Promise<any> {
|
||||
public async pyCall(funcName: string, ...varArgs: unknown[]): Promise<any> {
|
||||
const startTime = Date.now();
|
||||
this._sendData(sandboxUtil.CALL, Array.from(arguments));
|
||||
return this._pyCallWait(funcName, startTime);
|
||||
const slowCallCheck = setTimeout(() => {
|
||||
// Log calls that take some time, can be a useful symptom of misconfiguration
|
||||
// (or just benign if the doc is big).
|
||||
log.rawWarn('Slow pyCall', {...this._logMeta, funcName});
|
||||
}, 10000);
|
||||
const result = await this._pyCallWait(funcName, startTime);
|
||||
clearTimeout(slowCallCheck);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -371,6 +393,7 @@ const spawners = {
|
||||
docker, // Run sandboxes in distinct docker containers.
|
||||
gvisor, // Gvisor's runsc sandbox.
|
||||
macSandboxExec, // Use "sandbox-exec" on Mac.
|
||||
pyodide, // Run data engine using pyodide.
|
||||
};
|
||||
|
||||
function isFlavor(flavor: string): flavor is keyof typeof spawners {
|
||||
@ -528,6 +551,44 @@ function unsandboxed(options: ISandboxOptions): SandboxProcess {
|
||||
return {child, control: new DirectProcessControl(child, options.logMeta)};
|
||||
}
|
||||
|
||||
function pyodide(options: ISandboxOptions): SandboxProcess {
|
||||
const paths = getAbsolutePaths(options);
|
||||
// We will fork with three regular pipes (stdin, stdout, stderr), then
|
||||
// ipc (mandatory for calling fork), and a replacement pipe for stdin
|
||||
// and for stdout.
|
||||
// The regular stdin always opens non-blocking in node, which is a pain
|
||||
// in this case, so we just use a different pipe. There's a different
|
||||
// problem with stdout, with the same solution.
|
||||
const spawnOptions = {
|
||||
stdio: ['ignore', 'ignore', 'pipe', 'ipc', 'pipe', 'pipe'] as Array<'pipe'|'ipc'>,
|
||||
env: {
|
||||
PYTHONPATH: paths.engine,
|
||||
IMPORTDIR: options.importDir,
|
||||
...getInsertedEnv(options),
|
||||
...getWrappingEnv(options),
|
||||
}
|
||||
};
|
||||
const base = getUnpackedAppRoot();
|
||||
const child = fork(path.join(base, 'sandbox', 'pyodide', 'pipe.js'),
|
||||
{cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
|
||||
return {
|
||||
child,
|
||||
control: new DirectProcessControl(child, options.logMeta),
|
||||
dataToSandboxDescriptor: 4, // Cannot use normal descriptor, node
|
||||
// makes it non-blocking. Can be worked around in linux and osx, but
|
||||
// for windows just using a different file descriptor seems simplest.
|
||||
// In the sandbox, calling async methods from emscripten code is
|
||||
// possible but would require more changes to the data engine code
|
||||
// than seems reasonable at this time. The top level sandbox.run
|
||||
// can be tweaked to step operations, which actually works for a
|
||||
// lot of things, but not for cases where the sandbox calls back
|
||||
// into node (e.g. for column type guessing). TLDR: just switching
|
||||
// to FD 4 and reading synchronously is more practical solution.
|
||||
dataFromSandboxDescriptor: 5, // There's an equally long but different
|
||||
// story about why stdout is a bit messed up under pyodide right now.
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to run python in gvisor's runsc, with multiple
|
||||
* sandboxes run within the same container. GRIST_SANDBOX should
|
||||
|
||||
@ -14,6 +14,32 @@ import marshal
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
class CarefulReader(object):
|
||||
"""
|
||||
Wrap a pipe when reading from Pyodide, to work around marshaling
|
||||
panicking if fewer bytes are read in a block than it was expecting.
|
||||
Just wait for more.
|
||||
"""
|
||||
|
||||
def __init__(self, file_):
|
||||
self._file = file_
|
||||
|
||||
def write(self, data):
|
||||
return self._file.write(data)
|
||||
|
||||
def read(self, size):
|
||||
return self._file.read(size)
|
||||
|
||||
def readinto(self, b):
|
||||
result = self._file.readinto(b)
|
||||
while result is not None and result < len(b):
|
||||
bview = memoryview(b)
|
||||
result += self._file.readinto(bview[result:])
|
||||
return result
|
||||
|
||||
def __getattr__(self, attr):
|
||||
return getattr(self._file, attr)
|
||||
|
||||
def log(msg):
|
||||
sys.stderr.write(str(msg) + "\n")
|
||||
sys.stderr.flush()
|
||||
@ -23,22 +49,25 @@ class Sandbox(object):
|
||||
This class works in conjunction with Sandbox.js to allow function calls
|
||||
between the Node process and this sandbox.
|
||||
|
||||
The sandbox provides two pipes (on fds 3 and 4) to send data to and from the sandboxed
|
||||
The sandbox provides two pipes to send data to and from the sandboxed
|
||||
process. Data on these is serialized using `marshal` module. All messages are comprised of a
|
||||
msgCode followed immediatedly by msgBody, with the following msgCodes:
|
||||
CALL = call to the other side. The data must be an array of [func_name, arguments...]
|
||||
DATA = data must be a value to return to a call from the other side
|
||||
EXC = data must be an exception to return to a call from the other side
|
||||
|
||||
Optionally, a callback can be supplied instead of an output pipe.
|
||||
"""
|
||||
|
||||
CALL = None
|
||||
DATA = True
|
||||
EXC = False
|
||||
|
||||
def __init__(self, external_input, external_output):
|
||||
def __init__(self, external_input, external_output, external_output_method=None):
|
||||
self._functions = {}
|
||||
self._external_input = external_input
|
||||
self._external_output = external_output
|
||||
self._external_output_method = external_output_method
|
||||
|
||||
@classmethod
|
||||
def connected_to_js_pipes(cls):
|
||||
@ -62,6 +91,14 @@ class Sandbox(object):
|
||||
sys.stdout = sys.stderr
|
||||
return Sandbox.connected_to_js_pipes()
|
||||
|
||||
@classmethod
|
||||
def use_pyodide(cls):
|
||||
import js # Get pyodide object.
|
||||
external_input = CarefulReader(sys.stdin.buffer)
|
||||
external_output_method = lambda data: js.sendFromSandbox(data)
|
||||
sys.stdout = sys.stderr
|
||||
return cls(external_input, None, external_output_method)
|
||||
|
||||
def _send_to_js(self, msgCode, msgBody):
|
||||
# (Note that marshal version 2 is the default; we specify it explicitly for clarity. The
|
||||
# difference with version 0 is that version 2 uses a faster binary format for floats.)
|
||||
@ -70,8 +107,14 @@ class Sandbox(object):
|
||||
# It's much better to ensure the whole blob is sent as one write. We marshal the resulting
|
||||
# buffer again so that the reader can quickly tell how many bytes to expect.
|
||||
buf = marshal.dumps((msgCode, msgBody), 2)
|
||||
marshal.dump(buf, self._external_output, 2)
|
||||
self._external_output.flush()
|
||||
if self._external_output:
|
||||
marshal.dump(buf, self._external_output, 2)
|
||||
self._external_output.flush()
|
||||
elif self._external_output_method:
|
||||
buf = marshal.dumps(buf, 2)
|
||||
self._external_output_method(buf)
|
||||
else:
|
||||
raise Exception('no data output method')
|
||||
|
||||
def call_external(self, name, *args):
|
||||
self._send_to_js(Sandbox.CALL, (name,) + args)
|
||||
@ -115,6 +158,8 @@ def get_default_sandbox():
|
||||
if default_sandbox is None:
|
||||
if os.environ.get('PIPE_MODE') == 'minimal':
|
||||
default_sandbox = Sandbox.use_common_pipes()
|
||||
elif os.environ.get('PIPE_MODE') == 'pyodide':
|
||||
default_sandbox = Sandbox.use_pyodide()
|
||||
else:
|
||||
default_sandbox = Sandbox.connected_to_js_pipes()
|
||||
return default_sandbox
|
||||
|
||||
@ -189,23 +189,21 @@ class TestRenames2(test_engine.EngineTestCase):
|
||||
|
||||
def test_renames_b(self):
|
||||
# Rename Games.name: affects People.Games_Won, Games.win4_game_name
|
||||
# TODO: win4_game_name isn't updated due to astroid avoidance of looking up the same attr on
|
||||
# the same class during inference.
|
||||
out_actions = self.apply_user_action(["RenameColumn", "Games", "name", "nombre"])
|
||||
self.assertPartialOutActions(out_actions, { "stored": [
|
||||
["RenameColumn", "Games", "name", "nombre"],
|
||||
["ModifyColumn", "People", "Games_Won", {
|
||||
"formula": "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))"
|
||||
}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [4, 12], {
|
||||
"colId": ["nombre", "Games_Won"],
|
||||
["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.win.win.win.nombre"}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [4, 12, 19], {
|
||||
"colId": ["nombre", "Games_Won", "win4_game_name"],
|
||||
"formula": [
|
||||
"", "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))"]
|
||||
}],
|
||||
["BulkUpdateRecord", "Games", [1, 2, 3, 4], {
|
||||
"win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"],
|
||||
["E", "AttributeError"], ["E", "AttributeError"]]
|
||||
}],
|
||||
"",
|
||||
"' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))",
|
||||
"$win.win.win.win.nombre"
|
||||
]
|
||||
}]
|
||||
]})
|
||||
|
||||
# Fix up things missed due to the TODOs above.
|
||||
@ -264,22 +262,16 @@ class TestRenames2(test_engine.EngineTestCase):
|
||||
|
||||
def test_renames_d(self):
|
||||
# Rename People.name: affects People.N, People.ParnerNames
|
||||
# TODO: win3_person_name ($win.win.win.name) does NOT get updated correctly with astroid
|
||||
# because of a limitation in astroid inference: it refuses to look up the same attr on the
|
||||
# same class during inference (in order to protect against too much recursion).
|
||||
# TODO: PartnerNames does NOT get updated correctly because astroid doesn't infer meanings of
|
||||
# lists very well.
|
||||
out_actions = self.apply_user_action(["RenameColumn", "People", "name", "nombre"])
|
||||
self.assertPartialOutActions(out_actions, { "stored": [
|
||||
["RenameColumn", "People", "name", "nombre"],
|
||||
["ModifyColumn", "People", "N", {"formula": "$nombre.upper()"}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [2, 11], {
|
||||
"colId": ["nombre", "N"],
|
||||
"formula": ["", "$nombre.upper()"]
|
||||
}],
|
||||
["BulkUpdateRecord", "Games", [1, 2, 3, 4], {
|
||||
"win3_person_name": [["E", "AttributeError"], ["E", "AttributeError"],
|
||||
["E", "AttributeError"], ["E", "AttributeError"]]
|
||||
["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.win.win.nombre"}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [2, 11, 18], {
|
||||
"colId": ["nombre", "N", "win3_person_name"],
|
||||
"formula": ["", "$nombre.upper()", "$win.win.win.nombre"]
|
||||
}],
|
||||
["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], {
|
||||
"PartnerNames": [["E", "AttributeError"], ["E", "AttributeError"],
|
||||
@ -287,8 +279,7 @@ class TestRenames2(test_engine.EngineTestCase):
|
||||
}],
|
||||
]})
|
||||
|
||||
# Fix up things missed due to the TODOs above.
|
||||
self.modify_column("Games", "win3_person_name", formula="$win.win.win.nombre")
|
||||
# Fix up things missed due to the TODO above.
|
||||
self.modify_column("People", "PartnerNames",
|
||||
formula=self.partner_names.replace("name", "nombre"))
|
||||
|
||||
@ -305,21 +296,14 @@ class TestRenames2(test_engine.EngineTestCase):
|
||||
self.assertPartialOutActions(out_actions, { "stored": [
|
||||
["RenameColumn", "People", "partner", "companero"],
|
||||
["ModifyColumn", "People", "partner4", {
|
||||
"formula": "$companero.companero.partner.partner"
|
||||
"formula": "$companero.companero.companero.companero"
|
||||
}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [14, 15], {
|
||||
"colId": ["companero", "partner4"],
|
||||
"formula": [self.partner, "$companero.companero.partner.partner"]
|
||||
}],
|
||||
["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], {
|
||||
"partner4": [["E", "AttributeError"], ["E", "AttributeError"],
|
||||
["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]]
|
||||
}],
|
||||
"formula": [self.partner, "$companero.companero.companero.companero"]
|
||||
}]
|
||||
]})
|
||||
|
||||
# Fix up things missed due to the TODOs above.
|
||||
self.modify_column("People", "partner4", formula="$companero.companero.companero.companero")
|
||||
|
||||
_replace_col_name(self.people_data, "partner", "companero")
|
||||
self.assertTableData("People", cols="subset", data=self.people_data)
|
||||
self.assertTableData("Games", cols="subset", data=self.games_data)
|
||||
@ -331,21 +315,13 @@ class TestRenames2(test_engine.EngineTestCase):
|
||||
self.assertPartialOutActions(out_actions, { "stored": [
|
||||
["RenameColumn", "People", "win", "pwin"],
|
||||
["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.pwin.win.name"}],
|
||||
# TODO: the omission of the 4th win's update is due to the same astroid bug mentioned above.
|
||||
["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.win.name"}],
|
||||
["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.pwin.name"}],
|
||||
["BulkUpdateRecord", "_grist_Tables_column", [16, 18, 19], {
|
||||
"colId": ["pwin", "win3_person_name", "win4_game_name"],
|
||||
"formula": ["Entries.lookupOne(person=$id, rank=1).game",
|
||||
"$win.pwin.win.name", "$win.pwin.win.win.name"]}],
|
||||
["BulkUpdateRecord", "Games", [1, 2, 3, 4], {
|
||||
"win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"],
|
||||
["E", "AttributeError"], ["E", "AttributeError"]]
|
||||
}],
|
||||
"$win.pwin.win.name", "$win.pwin.win.pwin.name"]}],
|
||||
]})
|
||||
|
||||
# Fix up things missed due to the TODOs above.
|
||||
self.modify_column("Games", "win4_game_name", formula="$win.pwin.win.pwin.name")
|
||||
|
||||
_replace_col_name(self.people_data, "win", "pwin")
|
||||
self.assertTableData("People", cols="subset", data=self.people_data)
|
||||
self.assertTableData("Games", cols="subset", data=self.games_data)
|
||||
|
||||
19
sandbox/pyodide/Makefile
Normal file
19
sandbox/pyodide/Makefile
Normal file
@ -0,0 +1,19 @@
|
||||
default:
|
||||
echo "Welcome to the pyodide sandbox"
|
||||
echo "make fetch_packages # gets python packages prepared earlier"
|
||||
echo "make build_packages # build python packages from scratch"
|
||||
echo "make save_packages # upload python packages to fetch later"
|
||||
echo "setup # get pyodide node package"
|
||||
|
||||
fetch_packages:
|
||||
node ./packages.js https://s3.amazonaws.com/grist-pynbox/pyodide/packages/ _build/packages/
|
||||
|
||||
build_packages:
|
||||
./build_packages.sh
|
||||
|
||||
save_packages:
|
||||
aws s3 sync _build/packages s3://grist-pynbox/pyodide/packages/
|
||||
|
||||
setup:
|
||||
./setup.sh
|
||||
make fetch_packages
|
||||
28
sandbox/pyodide/README.md
Normal file
28
sandbox/pyodide/README.md
Normal file
@ -0,0 +1,28 @@
|
||||
This is a collection of scripts for running a pyodide-based "sandbox" for
|
||||
Grist.
|
||||
|
||||
I put "sandbox" in quotes since pyodide isn't built with sandboxing
|
||||
in mind. It was written to run in a browser, where the browser does
|
||||
sandboxing. I don't know how much of node's API ends up being exposed
|
||||
to the "sandbox" - in previous versions of pyodide it seems the answer is
|
||||
"a lot". See the back-and-forth between dalcde and hoodmane in:
|
||||
https://github.com/pyodide/pyodide/issues/960
|
||||
See specifically:
|
||||
https://github.com/pyodide/pyodide/issues/960#issuecomment-752305257
|
||||
I looked at hiwire and its treatment of js globals has changed a
|
||||
lot. On the surface it looks like there is good control of what is
|
||||
exposed, but there may be other routes.
|
||||
|
||||
Still, some wasm-based solution is likely to be helpful, whether from
|
||||
pyodide or elsewhere, and this is good practice for that.
|
||||
|
||||
***
|
||||
|
||||
To run, we need specific versions of the Python packages that Grist uses
|
||||
to be prepared. It should suffice to do:
|
||||
|
||||
```
|
||||
make setup
|
||||
```
|
||||
|
||||
In this directory. See the `Makefile` for other options.
|
||||
30
sandbox/pyodide/build_packages.sh
Normal file
30
sandbox/pyodide/build_packages.sh
Normal file
@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "###############################################################"
|
||||
echo "## Get pyodide repository, for transpiling python packages"
|
||||
|
||||
if [[ ! -e _build/pyodide ]]; then
|
||||
cd _build
|
||||
git clone https://github.com/pyodide/pyodide
|
||||
cd ..
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "###############################################################"
|
||||
echo "## Prepare python packages"
|
||||
|
||||
cd _build/pyodide
|
||||
./run_docker make
|
||||
cp ../../../requirements3.txt .
|
||||
./run_docker pyodide build -r requirements3.txt --output-lockfile result.txt
|
||||
cat result.txt
|
||||
cd ../..
|
||||
|
||||
echo ""
|
||||
echo "###############################################################"
|
||||
echo "## Copy out python packages"
|
||||
|
||||
node ./packages.js _build/pyodide/dist/ _build/packages/
|
||||
117
sandbox/pyodide/packages.js
Normal file
117
sandbox/pyodide/packages.js
Normal file
@ -0,0 +1,117 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const fetch = require('node-fetch');
|
||||
|
||||
async function listLibs(src) {
|
||||
const txt = fs.readFileSync(path.join(__dirname, '..', 'requirements3.txt'), 'utf8');
|
||||
const libs = {};
|
||||
for (const line of txt.split(/\r?\n/)) {
|
||||
const raw = line.split('#')[0];
|
||||
if (!raw.includes('==')) { continue; }
|
||||
const [name, version] = line.split('==');
|
||||
libs[name] = version;
|
||||
}
|
||||
const hits = [];
|
||||
const misses = [];
|
||||
const toLoad = [];
|
||||
const material = fs.readdirSync(src);
|
||||
for (const [lib, version] of Object.entries(libs)) {
|
||||
const nlib = lib.replace(/-/g, '_');
|
||||
const info = {
|
||||
name: lib,
|
||||
standardName: nlib,
|
||||
version: version,
|
||||
}
|
||||
try {
|
||||
const found = material.filter(m => m.startsWith(`${nlib}-${version}-`));
|
||||
if (found.length !== 1) {
|
||||
throw new Error('did not find 1');
|
||||
}
|
||||
const fname = found[0];
|
||||
info.fullName = path.join(src, fname);
|
||||
info.fileName = fname;
|
||||
toLoad.push(info);
|
||||
hits.push(lib);
|
||||
} catch (e) {
|
||||
misses.push(info);
|
||||
}
|
||||
}
|
||||
return {
|
||||
available: toLoad,
|
||||
misses,
|
||||
};
|
||||
}
|
||||
exports.listLibs = listLibs;
|
||||
|
||||
async function findOnDisk(src, dest) {
|
||||
console.log(`Organizing packages on disk`, {src, dest});
|
||||
fs.mkdirSync(dest, {recursive: true});
|
||||
let libs = (await listLibs(src));
|
||||
for (const lib of libs.available) {
|
||||
fs.copyFileSync(lib.fullName, path.join(dest, lib.fileName));
|
||||
fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`),
|
||||
JSON.stringify({
|
||||
name: lib.name,
|
||||
version: lib.version,
|
||||
fileName: lib.fileName,
|
||||
}, null, 2));
|
||||
console.log("Copied", {
|
||||
content: path.join(dest, lib.fileName),
|
||||
meta: path.join(dest, `${lib.name}-${lib.version}.json`),
|
||||
});
|
||||
}
|
||||
libs = await listLibs(dest);
|
||||
console.log(`Cached`, {libs: libs.available.map(lib => lib.name)});
|
||||
console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)});
|
||||
}
|
||||
|
||||
async function findOnNet(src, dest) {
|
||||
console.log(`Caching packages on disk`, {src, dest});
|
||||
fs.mkdirSync(dest, {recursive: true});
|
||||
let libs = await listLibs(dest);
|
||||
console.log(`Cached`, {libs: libs.available.map(lib => lib.name)});
|
||||
for (const lib of libs.misses) {
|
||||
console.log('Fetching', lib);
|
||||
const url = new URL(src);
|
||||
url.pathname = url.pathname + lib.name + '-' + lib.version + '.json';
|
||||
const result = await fetch(url.href);
|
||||
if (result.status === 200) {
|
||||
const data = await result.json();
|
||||
const url2 = new URL(src);
|
||||
url2.pathname = url2.pathname + data.fileName;
|
||||
const result2 = await fetch(url2.href);
|
||||
if (result2.status === 200) {
|
||||
fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`),
|
||||
JSON.stringify(data, null, 2));
|
||||
fs.writeFileSync(path.join(dest, data.fileName),
|
||||
await result2.buffer());
|
||||
} else {
|
||||
console.error("No payload available", {lib});
|
||||
}
|
||||
} else {
|
||||
console.error("No metadata available", {lib});
|
||||
}
|
||||
}
|
||||
libs = await listLibs(dest);
|
||||
console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)});
|
||||
}
|
||||
|
||||
async function main(src, dest) {
|
||||
if (!src) {
|
||||
console.error('please supply a source');
|
||||
process.exit(1);
|
||||
}
|
||||
if (!dest) {
|
||||
console.error('please supply a destination');
|
||||
process.exit(1);
|
||||
}
|
||||
if (src.startsWith('http:') || src.startsWith('https:')) {
|
||||
await findOnNet(src, dest);
|
||||
return;
|
||||
}
|
||||
await findOnDisk(src, dest);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main(...process.argv.slice(2)).catch(e => console.error(e));
|
||||
}
|
||||
147
sandbox/pyodide/pipe.js
Normal file
147
sandbox/pyodide/pipe.js
Normal file
@ -0,0 +1,147 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const { loadPyodide } = require('./_build/worker/node_modules/pyodide');
|
||||
const { listLibs } = require('./packages');
|
||||
|
||||
const INCOMING_FD = 4;
|
||||
const OUTGOING_FD = 5;
|
||||
|
||||
class GristPipe {
|
||||
constructor() {
|
||||
this.pyodide = null;
|
||||
this.incomingBuffer = Buffer.alloc(65536);
|
||||
this.addedBlob = false;
|
||||
this.adminMode = false;
|
||||
}
|
||||
|
||||
async init() {
|
||||
const self = this;
|
||||
this.setAdminMode(true);
|
||||
this.pyodide = await loadPyodide({
|
||||
jsglobals: {
|
||||
Object: {},
|
||||
setTimeout: function(code, delay) {
|
||||
if (self.adminMode) {
|
||||
setTimeout(code, delay);
|
||||
// Seems to be OK not to return anything, so we don't.
|
||||
} else {
|
||||
throw new Error('setTimeout not available');
|
||||
}
|
||||
},
|
||||
sendFromSandbox: (data) => {
|
||||
return fs.writeSync(OUTGOING_FD, Buffer.from(data.toJs()));
|
||||
}
|
||||
},
|
||||
});
|
||||
this.setAdminMode(false);
|
||||
this.pyodide.setStdin({
|
||||
stdin: () => {
|
||||
const result = fs.readSync(INCOMING_FD, this.incomingBuffer, 0,
|
||||
this.incomingBuffer.byteLength);
|
||||
if (result > 0) {
|
||||
const buf = Buffer.allocUnsafe(result, 0, 0, result);
|
||||
this.incomingBuffer.copy(buf);
|
||||
return buf;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
});
|
||||
this.pyodide.setStderr({
|
||||
batched: (data) => {
|
||||
this.log("[py]", data);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async loadCode() {
|
||||
// Load python packages.
|
||||
const src = path.join(__dirname, '_build', 'packages');
|
||||
const lsty = (await listLibs(src)).available.map(item => item.fullName);
|
||||
await this.pyodide.loadPackage(lsty, {
|
||||
messageCallback: (msg) => this.log('[package]', msg),
|
||||
});
|
||||
|
||||
// Load Grist data engine code.
|
||||
// We mount it as /grist_src, copy to /grist, then unmount.
|
||||
// Note that path to source must be a realpath.
|
||||
const root = fs.realpathSync(path.join(__dirname, '../grist'));
|
||||
await this.pyodide.FS.mkdir("/grist_src");
|
||||
// careful, needs to be a realpath
|
||||
await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, { root }, "/grist_src");
|
||||
// Now want to copy /grist_src to /grist.
|
||||
// For some reason shutil.copytree doesn't work on Windows in this situation, so
|
||||
// we reimplement it crudely.
|
||||
await this.pyodide.runPython(`
|
||||
import os, shutil
|
||||
def copytree(src, dst):
|
||||
os.makedirs(dst, exist_ok=True)
|
||||
for item in os.listdir(src):
|
||||
s = os.path.join(src, item)
|
||||
d = os.path.join(dst, item)
|
||||
if os.path.isdir(s):
|
||||
copytree(s, d)
|
||||
else:
|
||||
shutil.copy2(s, d)
|
||||
copytree('/grist_src', '/grist')`);
|
||||
await this.pyodide.FS.unmount("/grist_src");
|
||||
await this.pyodide.FS.rmdir("/grist_src");
|
||||
}
|
||||
|
||||
async mountImportDirIfNeeded() {
|
||||
if (process.env.IMPORTDIR) {
|
||||
this.log("Setting up import from", process.env.IMPORTDIR);
|
||||
// Ideally would be read-only; don't see a way to do that,
|
||||
// other than copying like for Grist code.
|
||||
await this.pyodide.FS.mkdir("/import");
|
||||
await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, {
|
||||
root: process.env.IMPORTDIR,
|
||||
}, "/import");
|
||||
}
|
||||
}
|
||||
|
||||
async runCode() {
|
||||
await this.pyodide.runPython(`
|
||||
import sys
|
||||
sys.path.append('/')
|
||||
sys.path.append('/grist')
|
||||
import grist
|
||||
import main
|
||||
import os
|
||||
os.environ['PIPE_MODE'] = 'pyodide'
|
||||
os.environ['IMPORTDIR'] = '/import'
|
||||
main.main()
|
||||
`);
|
||||
}
|
||||
|
||||
setAdminMode(active) {
|
||||
this.adminMode = active;
|
||||
// Lack of Blob may result in a message on console.log that hurts us.
|
||||
if (active && !globalThis.Blob) {
|
||||
globalThis.Blob = String;
|
||||
this.addedBlob = true;
|
||||
}
|
||||
if (!active && this.addedBlob) {
|
||||
delete globalThis.Blob;
|
||||
this.addedBlob = false;
|
||||
}
|
||||
}
|
||||
|
||||
log(...args) {
|
||||
console.error("[pyodide sandbox]", ...args);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
const pipe = new GristPipe();
|
||||
await pipe.init();
|
||||
await pipe.loadCode();
|
||||
await pipe.mountImportDirIfNeeded();
|
||||
await pipe.runCode();
|
||||
} finally {
|
||||
process.stdin.removeAllListeners();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => console.error("[pyodide error]", err));
|
||||
15
sandbox/pyodide/setup.sh
Executable file
15
sandbox/pyodide/setup.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
echo ""
|
||||
echo "###############################################################"
|
||||
echo "## Get pyodide node package"
|
||||
|
||||
if [[ ! -e _build/worker ]]; then
|
||||
mkdir -p _build/worker
|
||||
cd _build/worker
|
||||
yarn init --yes
|
||||
yarn add pyodide@0.22.1
|
||||
cd ../..
|
||||
fi
|
||||
@ -12,7 +12,11 @@ jdcal==1.4.1
|
||||
et-xmlfile==1.0.1
|
||||
|
||||
# Different astroid version for python 3
|
||||
astroid==2.5.7
|
||||
astroid==2.14.2
|
||||
typing_extensions==4.4.0
|
||||
|
||||
# Different roman version for python 3
|
||||
roman==3.3
|
||||
|
||||
# Everything after this is the same for python 2 and 3
|
||||
asttokens==2.0.5
|
||||
@ -23,7 +27,6 @@ iso8601==0.1.12
|
||||
lazy_object_proxy==1.6.0
|
||||
phonenumberslite==8.12.57
|
||||
python_dateutil==2.8.2
|
||||
roman==2.0.0
|
||||
singledispatch==3.6.2
|
||||
six==1.16.0
|
||||
sortedcontainers==2.4.0
|
||||
|
||||
Loading…
Reference in New Issue
Block a user