diff --git a/.dockerignore b/.dockerignore index 1434837b..f2e04e8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ !plugins !test !ext +**/_build diff --git a/.gitignore b/.gitignore index d26d0f07..48663300 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ jspm_packages/ # dotenv environment variables file .env + +**/_build diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts index aff564f8..35e09ae1 100644 --- a/app/server/lib/NSandbox.ts +++ b/app/server/lib/NSandbox.ts @@ -15,7 +15,7 @@ import { } from 'app/server/lib/SandboxControl'; import * as sandboxUtil from 'app/server/lib/sandboxUtil'; import * as shutdown from 'app/server/lib/shutdown'; -import {ChildProcess, spawn} from 'child_process'; +import {ChildProcess, fork, spawn} from 'child_process'; import * as fs from 'fs'; import * as _ from 'lodash'; import * as path from 'path'; @@ -73,6 +73,8 @@ export interface ISandboxOptions { interface SandboxProcess { child: ChildProcess; control: ISandboxControl; + dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data + dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data } type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void]; @@ -131,10 +133,23 @@ export class NSandbox implements ISandbox { if (options.minimalPipeMode) { log.rawDebug("3-pipe Sandbox started", this._logMeta); - this._streamToSandbox = this.childProc.stdin!; - this._streamFromSandbox = this.childProc.stdout!; + if (sandboxProcess.dataToSandboxDescriptor) { + this._streamToSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable; + } else { + this._streamToSandbox = this.childProc.stdin!; + } + if (sandboxProcess.dataFromSandboxDescriptor) { + this._streamFromSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor]; + } else { + this._streamFromSandbox = this.childProc.stdout!; + } } else { log.rawDebug("5-pipe Sandbox started", this._logMeta); + if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) { + throw new Error('cannot override file descriptors in 5 pipe mode'); + } this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable; this._streamFromSandbox = (this.childProc.stdio as Stream[])[4]; this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta)); @@ -206,10 +221,17 @@ export class NSandbox implements ISandbox { * @param args Arguments to pass to the given function. * @returns A promise for the return value from the Python function. */ - public pyCall(funcName: string, ...varArgs: unknown[]): Promise { + public async pyCall(funcName: string, ...varArgs: unknown[]): Promise { const startTime = Date.now(); this._sendData(sandboxUtil.CALL, Array.from(arguments)); - return this._pyCallWait(funcName, startTime); + const slowCallCheck = setTimeout(() => { + // Log calls that take some time, can be a useful symptom of misconfiguration + // (or just benign if the doc is big). + log.rawWarn('Slow pyCall', {...this._logMeta, funcName}); + }, 10000); + const result = await this._pyCallWait(funcName, startTime); + clearTimeout(slowCallCheck); + return result; } /** @@ -371,6 +393,7 @@ const spawners = { docker, // Run sandboxes in distinct docker containers. gvisor, // Gvisor's runsc sandbox. macSandboxExec, // Use "sandbox-exec" on Mac. + pyodide, // Run data engine using pyodide. }; function isFlavor(flavor: string): flavor is keyof typeof spawners { @@ -528,6 +551,44 @@ function unsandboxed(options: ISandboxOptions): SandboxProcess { return {child, control: new DirectProcessControl(child, options.logMeta)}; } +function pyodide(options: ISandboxOptions): SandboxProcess { + const paths = getAbsolutePaths(options); + // We will fork with three regular pipes (stdin, stdout, stderr), then + // ipc (mandatory for calling fork), and a replacement pipe for stdin + // and for stdout. + // The regular stdin always opens non-blocking in node, which is a pain + // in this case, so we just use a different pipe. There's a different + // problem with stdout, with the same solution. + const spawnOptions = { + stdio: ['ignore', 'ignore', 'pipe', 'ipc', 'pipe', 'pipe'] as Array<'pipe'|'ipc'>, + env: { + PYTHONPATH: paths.engine, + IMPORTDIR: options.importDir, + ...getInsertedEnv(options), + ...getWrappingEnv(options), + } + }; + const base = getUnpackedAppRoot(); + const child = fork(path.join(base, 'sandbox', 'pyodide', 'pipe.js'), + {cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions}); + return { + child, + control: new DirectProcessControl(child, options.logMeta), + dataToSandboxDescriptor: 4, // Cannot use normal descriptor, node + // makes it non-blocking. Can be worked around in linux and osx, but + // for windows just using a different file descriptor seems simplest. + // In the sandbox, calling async methods from emscripten code is + // possible but would require more changes to the data engine code + // than seems reasonable at this time. The top level sandbox.run + // can be tweaked to step operations, which actually works for a + // lot of things, but not for cases where the sandbox calls back + // into node (e.g. for column type guessing). TLDR: just switching + // to FD 4 and reading synchronously is more practical solution. + dataFromSandboxDescriptor: 5, // There's an equally long but different + // story about why stdout is a bit messed up under pyodide right now. + }; +} + /** * Helper function to run python in gvisor's runsc, with multiple * sandboxes run within the same container. GRIST_SANDBOX should diff --git a/sandbox/grist/sandbox.py b/sandbox/grist/sandbox.py index fb4c4fb6..ff9727e2 100644 --- a/sandbox/grist/sandbox.py +++ b/sandbox/grist/sandbox.py @@ -14,6 +14,32 @@ import marshal import sys import traceback +class CarefulReader(object): + """ + Wrap a pipe when reading from Pyodide, to work around marshaling + panicking if fewer bytes are read in a block than it was expecting. + Just wait for more. + """ + + def __init__(self, file_): + self._file = file_ + + def write(self, data): + return self._file.write(data) + + def read(self, size): + return self._file.read(size) + + def readinto(self, b): + result = self._file.readinto(b) + while result is not None and result < len(b): + bview = memoryview(b) + result += self._file.readinto(bview[result:]) + return result + + def __getattr__(self, attr): + return getattr(self._file, attr) + def log(msg): sys.stderr.write(str(msg) + "\n") sys.stderr.flush() @@ -23,22 +49,25 @@ class Sandbox(object): This class works in conjunction with Sandbox.js to allow function calls between the Node process and this sandbox. - The sandbox provides two pipes (on fds 3 and 4) to send data to and from the sandboxed + The sandbox provides two pipes to send data to and from the sandboxed process. Data on these is serialized using `marshal` module. All messages are comprised of a msgCode followed immediatedly by msgBody, with the following msgCodes: CALL = call to the other side. The data must be an array of [func_name, arguments...] DATA = data must be a value to return to a call from the other side EXC = data must be an exception to return to a call from the other side + + Optionally, a callback can be supplied instead of an output pipe. """ CALL = None DATA = True EXC = False - def __init__(self, external_input, external_output): + def __init__(self, external_input, external_output, external_output_method=None): self._functions = {} self._external_input = external_input self._external_output = external_output + self._external_output_method = external_output_method @classmethod def connected_to_js_pipes(cls): @@ -62,6 +91,14 @@ class Sandbox(object): sys.stdout = sys.stderr return Sandbox.connected_to_js_pipes() + @classmethod + def use_pyodide(cls): + import js # Get pyodide object. + external_input = CarefulReader(sys.stdin.buffer) + external_output_method = lambda data: js.sendFromSandbox(data) + sys.stdout = sys.stderr + return cls(external_input, None, external_output_method) + def _send_to_js(self, msgCode, msgBody): # (Note that marshal version 2 is the default; we specify it explicitly for clarity. The # difference with version 0 is that version 2 uses a faster binary format for floats.) @@ -70,8 +107,14 @@ class Sandbox(object): # It's much better to ensure the whole blob is sent as one write. We marshal the resulting # buffer again so that the reader can quickly tell how many bytes to expect. buf = marshal.dumps((msgCode, msgBody), 2) - marshal.dump(buf, self._external_output, 2) - self._external_output.flush() + if self._external_output: + marshal.dump(buf, self._external_output, 2) + self._external_output.flush() + elif self._external_output_method: + buf = marshal.dumps(buf, 2) + self._external_output_method(buf) + else: + raise Exception('no data output method') def call_external(self, name, *args): self._send_to_js(Sandbox.CALL, (name,) + args) @@ -115,6 +158,8 @@ def get_default_sandbox(): if default_sandbox is None: if os.environ.get('PIPE_MODE') == 'minimal': default_sandbox = Sandbox.use_common_pipes() + elif os.environ.get('PIPE_MODE') == 'pyodide': + default_sandbox = Sandbox.use_pyodide() else: default_sandbox = Sandbox.connected_to_js_pipes() return default_sandbox diff --git a/sandbox/grist/test_renames2.py b/sandbox/grist/test_renames2.py index 57a58713..d453297f 100644 --- a/sandbox/grist/test_renames2.py +++ b/sandbox/grist/test_renames2.py @@ -189,23 +189,21 @@ class TestRenames2(test_engine.EngineTestCase): def test_renames_b(self): # Rename Games.name: affects People.Games_Won, Games.win4_game_name - # TODO: win4_game_name isn't updated due to astroid avoidance of looking up the same attr on - # the same class during inference. out_actions = self.apply_user_action(["RenameColumn", "Games", "name", "nombre"]) self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "Games", "name", "nombre"], ["ModifyColumn", "People", "Games_Won", { "formula": "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))" }], - ["BulkUpdateRecord", "_grist_Tables_column", [4, 12], { - "colId": ["nombre", "Games_Won"], + ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.win.win.win.nombre"}], + ["BulkUpdateRecord", "_grist_Tables_column", [4, 12, 19], { + "colId": ["nombre", "Games_Won", "win4_game_name"], "formula": [ - "", "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))"] - }], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "", + "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))", + "$win.win.win.win.nombre" + ] + }] ]}) # Fix up things missed due to the TODOs above. @@ -264,22 +262,16 @@ class TestRenames2(test_engine.EngineTestCase): def test_renames_d(self): # Rename People.name: affects People.N, People.ParnerNames - # TODO: win3_person_name ($win.win.win.name) does NOT get updated correctly with astroid - # because of a limitation in astroid inference: it refuses to look up the same attr on the - # same class during inference (in order to protect against too much recursion). # TODO: PartnerNames does NOT get updated correctly because astroid doesn't infer meanings of # lists very well. out_actions = self.apply_user_action(["RenameColumn", "People", "name", "nombre"]) self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "name", "nombre"], ["ModifyColumn", "People", "N", {"formula": "$nombre.upper()"}], - ["BulkUpdateRecord", "_grist_Tables_column", [2, 11], { - "colId": ["nombre", "N"], - "formula": ["", "$nombre.upper()"] - }], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win3_person_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] + ["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.win.win.nombre"}], + ["BulkUpdateRecord", "_grist_Tables_column", [2, 11, 18], { + "colId": ["nombre", "N", "win3_person_name"], + "formula": ["", "$nombre.upper()", "$win.win.win.nombre"] }], ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { "PartnerNames": [["E", "AttributeError"], ["E", "AttributeError"], @@ -287,8 +279,7 @@ class TestRenames2(test_engine.EngineTestCase): }], ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("Games", "win3_person_name", formula="$win.win.win.nombre") + # Fix up things missed due to the TODO above. self.modify_column("People", "PartnerNames", formula=self.partner_names.replace("name", "nombre")) @@ -305,21 +296,14 @@ class TestRenames2(test_engine.EngineTestCase): self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "partner", "companero"], ["ModifyColumn", "People", "partner4", { - "formula": "$companero.companero.partner.partner" + "formula": "$companero.companero.companero.companero" }], ["BulkUpdateRecord", "_grist_Tables_column", [14, 15], { "colId": ["companero", "partner4"], - "formula": [self.partner, "$companero.companero.partner.partner"] - }], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { - "partner4": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "formula": [self.partner, "$companero.companero.companero.companero"] + }] ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("People", "partner4", formula="$companero.companero.companero.companero") - _replace_col_name(self.people_data, "partner", "companero") self.assertTableData("People", cols="subset", data=self.people_data) self.assertTableData("Games", cols="subset", data=self.games_data) @@ -331,21 +315,13 @@ class TestRenames2(test_engine.EngineTestCase): self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "win", "pwin"], ["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.pwin.win.name"}], - # TODO: the omission of the 4th win's update is due to the same astroid bug mentioned above. - ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.win.name"}], + ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.pwin.name"}], ["BulkUpdateRecord", "_grist_Tables_column", [16, 18, 19], { "colId": ["pwin", "win3_person_name", "win4_game_name"], "formula": ["Entries.lookupOne(person=$id, rank=1).game", - "$win.pwin.win.name", "$win.pwin.win.win.name"]}], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "$win.pwin.win.name", "$win.pwin.win.pwin.name"]}], ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("Games", "win4_game_name", formula="$win.pwin.win.pwin.name") - _replace_col_name(self.people_data, "win", "pwin") self.assertTableData("People", cols="subset", data=self.people_data) self.assertTableData("Games", cols="subset", data=self.games_data) diff --git a/sandbox/pyodide/Makefile b/sandbox/pyodide/Makefile new file mode 100644 index 00000000..2adc18c5 --- /dev/null +++ b/sandbox/pyodide/Makefile @@ -0,0 +1,19 @@ +default: + echo "Welcome to the pyodide sandbox" + echo "make fetch_packages # gets python packages prepared earlier" + echo "make build_packages # build python packages from scratch" + echo "make save_packages # upload python packages to fetch later" + echo "setup # get pyodide node package" + +fetch_packages: + node ./packages.js https://s3.amazonaws.com/grist-pynbox/pyodide/packages/ _build/packages/ + +build_packages: + ./build_packages.sh + +save_packages: + aws s3 sync _build/packages s3://grist-pynbox/pyodide/packages/ + +setup: + ./setup.sh + make fetch_packages diff --git a/sandbox/pyodide/README.md b/sandbox/pyodide/README.md new file mode 100644 index 00000000..4384d3ec --- /dev/null +++ b/sandbox/pyodide/README.md @@ -0,0 +1,28 @@ +This is a collection of scripts for running a pyodide-based "sandbox" for +Grist. + +I put "sandbox" in quotes since pyodide isn't built with sandboxing +in mind. It was written to run in a browser, where the browser does +sandboxing. I don't know how much of node's API ends up being exposed +to the "sandbox" - in previous versions of pyodide it seems the answer is +"a lot". See the back-and-forth between dalcde and hoodmane in: + https://github.com/pyodide/pyodide/issues/960 +See specifically: + https://github.com/pyodide/pyodide/issues/960#issuecomment-752305257 +I looked at hiwire and its treatment of js globals has changed a +lot. On the surface it looks like there is good control of what is +exposed, but there may be other routes. + +Still, some wasm-based solution is likely to be helpful, whether from +pyodide or elsewhere, and this is good practice for that. + +*** + +To run, we need specific versions of the Python packages that Grist uses +to be prepared. It should suffice to do: + +``` +make setup +``` + +In this directory. See the `Makefile` for other options. diff --git a/sandbox/pyodide/build_packages.sh b/sandbox/pyodide/build_packages.sh new file mode 100644 index 00000000..2354c831 --- /dev/null +++ b/sandbox/pyodide/build_packages.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +echo "" +echo "###############################################################" +echo "## Get pyodide repository, for transpiling python packages" + +if [[ ! -e _build/pyodide ]]; then + cd _build + git clone https://github.com/pyodide/pyodide + cd .. +fi + +echo "" +echo "###############################################################" +echo "## Prepare python packages" + +cd _build/pyodide +./run_docker make +cp ../../../requirements3.txt . +./run_docker pyodide build -r requirements3.txt --output-lockfile result.txt +cat result.txt +cd ../.. + +echo "" +echo "###############################################################" +echo "## Copy out python packages" + +node ./packages.js _build/pyodide/dist/ _build/packages/ diff --git a/sandbox/pyodide/packages.js b/sandbox/pyodide/packages.js new file mode 100644 index 00000000..fa0f03c6 --- /dev/null +++ b/sandbox/pyodide/packages.js @@ -0,0 +1,117 @@ +const path = require('path'); +const fs = require('fs'); +const fetch = require('node-fetch'); + +async function listLibs(src) { + const txt = fs.readFileSync(path.join(__dirname, '..', 'requirements3.txt'), 'utf8'); + const libs = {}; + for (const line of txt.split(/\r?\n/)) { + const raw = line.split('#')[0]; + if (!raw.includes('==')) { continue; } + const [name, version] = line.split('=='); + libs[name] = version; + } + const hits = []; + const misses = []; + const toLoad = []; + const material = fs.readdirSync(src); + for (const [lib, version] of Object.entries(libs)) { + const nlib = lib.replace(/-/g, '_'); + const info = { + name: lib, + standardName: nlib, + version: version, + } + try { + const found = material.filter(m => m.startsWith(`${nlib}-${version}-`)); + if (found.length !== 1) { + throw new Error('did not find 1'); + } + const fname = found[0]; + info.fullName = path.join(src, fname); + info.fileName = fname; + toLoad.push(info); + hits.push(lib); + } catch (e) { + misses.push(info); + } + } + return { + available: toLoad, + misses, + }; +} +exports.listLibs = listLibs; + +async function findOnDisk(src, dest) { + console.log(`Organizing packages on disk`, {src, dest}); + fs.mkdirSync(dest, {recursive: true}); + let libs = (await listLibs(src)); + for (const lib of libs.available) { + fs.copyFileSync(lib.fullName, path.join(dest, lib.fileName)); + fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`), + JSON.stringify({ + name: lib.name, + version: lib.version, + fileName: lib.fileName, + }, null, 2)); + console.log("Copied", { + content: path.join(dest, lib.fileName), + meta: path.join(dest, `${lib.name}-${lib.version}.json`), + }); + } + libs = await listLibs(dest); + console.log(`Cached`, {libs: libs.available.map(lib => lib.name)}); + console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)}); +} + +async function findOnNet(src, dest) { + console.log(`Caching packages on disk`, {src, dest}); + fs.mkdirSync(dest, {recursive: true}); + let libs = await listLibs(dest); + console.log(`Cached`, {libs: libs.available.map(lib => lib.name)}); + for (const lib of libs.misses) { + console.log('Fetching', lib); + const url = new URL(src); + url.pathname = url.pathname + lib.name + '-' + lib.version + '.json'; + const result = await fetch(url.href); + if (result.status === 200) { + const data = await result.json(); + const url2 = new URL(src); + url2.pathname = url2.pathname + data.fileName; + const result2 = await fetch(url2.href); + if (result2.status === 200) { + fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`), + JSON.stringify(data, null, 2)); + fs.writeFileSync(path.join(dest, data.fileName), + await result2.buffer()); + } else { + console.error("No payload available", {lib}); + } + } else { + console.error("No metadata available", {lib}); + } + } + libs = await listLibs(dest); + console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)}); +} + +async function main(src, dest) { + if (!src) { + console.error('please supply a source'); + process.exit(1); + } + if (!dest) { + console.error('please supply a destination'); + process.exit(1); + } + if (src.startsWith('http:') || src.startsWith('https:')) { + await findOnNet(src, dest); + return; + } + await findOnDisk(src, dest); +} + +if (require.main === module) { + main(...process.argv.slice(2)).catch(e => console.error(e)); +} diff --git a/sandbox/pyodide/pipe.js b/sandbox/pyodide/pipe.js new file mode 100644 index 00000000..09ffeea7 --- /dev/null +++ b/sandbox/pyodide/pipe.js @@ -0,0 +1,147 @@ +const path = require('path'); +const fs = require('fs'); + +const { loadPyodide } = require('./_build/worker/node_modules/pyodide'); +const { listLibs } = require('./packages'); + +const INCOMING_FD = 4; +const OUTGOING_FD = 5; + +class GristPipe { + constructor() { + this.pyodide = null; + this.incomingBuffer = Buffer.alloc(65536); + this.addedBlob = false; + this.adminMode = false; + } + + async init() { + const self = this; + this.setAdminMode(true); + this.pyodide = await loadPyodide({ + jsglobals: { + Object: {}, + setTimeout: function(code, delay) { + if (self.adminMode) { + setTimeout(code, delay); + // Seems to be OK not to return anything, so we don't. + } else { + throw new Error('setTimeout not available'); + } + }, + sendFromSandbox: (data) => { + return fs.writeSync(OUTGOING_FD, Buffer.from(data.toJs())); + } + }, + }); + this.setAdminMode(false); + this.pyodide.setStdin({ + stdin: () => { + const result = fs.readSync(INCOMING_FD, this.incomingBuffer, 0, + this.incomingBuffer.byteLength); + if (result > 0) { + const buf = Buffer.allocUnsafe(result, 0, 0, result); + this.incomingBuffer.copy(buf); + return buf; + } + return null; + }, + }); + this.pyodide.setStderr({ + batched: (data) => { + this.log("[py]", data); + } + }); + } + + async loadCode() { + // Load python packages. + const src = path.join(__dirname, '_build', 'packages'); + const lsty = (await listLibs(src)).available.map(item => item.fullName); + await this.pyodide.loadPackage(lsty, { + messageCallback: (msg) => this.log('[package]', msg), + }); + + // Load Grist data engine code. + // We mount it as /grist_src, copy to /grist, then unmount. + // Note that path to source must be a realpath. + const root = fs.realpathSync(path.join(__dirname, '../grist')); + await this.pyodide.FS.mkdir("/grist_src"); + // careful, needs to be a realpath + await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, { root }, "/grist_src"); + // Now want to copy /grist_src to /grist. + // For some reason shutil.copytree doesn't work on Windows in this situation, so + // we reimplement it crudely. + await this.pyodide.runPython(` +import os, shutil +def copytree(src, dst): + os.makedirs(dst, exist_ok=True) + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + copytree(s, d) + else: + shutil.copy2(s, d) +copytree('/grist_src', '/grist')`); + await this.pyodide.FS.unmount("/grist_src"); + await this.pyodide.FS.rmdir("/grist_src"); + } + + async mountImportDirIfNeeded() { + if (process.env.IMPORTDIR) { + this.log("Setting up import from", process.env.IMPORTDIR); + // Ideally would be read-only; don't see a way to do that, + // other than copying like for Grist code. + await this.pyodide.FS.mkdir("/import"); + await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, { + root: process.env.IMPORTDIR, + }, "/import"); + } + } + + async runCode() { + await this.pyodide.runPython(` + import sys + sys.path.append('/') + sys.path.append('/grist') + import grist + import main + import os + os.environ['PIPE_MODE'] = 'pyodide' + os.environ['IMPORTDIR'] = '/import' + main.main() +`); + } + + setAdminMode(active) { + this.adminMode = active; + // Lack of Blob may result in a message on console.log that hurts us. + if (active && !globalThis.Blob) { + globalThis.Blob = String; + this.addedBlob = true; + } + if (!active && this.addedBlob) { + delete globalThis.Blob; + this.addedBlob = false; + } + } + + log(...args) { + console.error("[pyodide sandbox]", ...args); + } +} + +async function main() { + try { + const pipe = new GristPipe(); + await pipe.init(); + await pipe.loadCode(); + await pipe.mountImportDirIfNeeded(); + await pipe.runCode(); + } finally { + process.stdin.removeAllListeners(); + } +} + +main().catch(err => console.error("[pyodide error]", err)); diff --git a/sandbox/pyodide/setup.sh b/sandbox/pyodide/setup.sh new file mode 100755 index 00000000..ce28cdc8 --- /dev/null +++ b/sandbox/pyodide/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +echo "" +echo "###############################################################" +echo "## Get pyodide node package" + +if [[ ! -e _build/worker ]]; then + mkdir -p _build/worker + cd _build/worker + yarn init --yes + yarn add pyodide@0.22.1 + cd ../.. +fi diff --git a/sandbox/requirements3.txt b/sandbox/requirements3.txt index 6fec0ea4..22f2995a 100644 --- a/sandbox/requirements3.txt +++ b/sandbox/requirements3.txt @@ -12,7 +12,11 @@ jdcal==1.4.1 et-xmlfile==1.0.1 # Different astroid version for python 3 -astroid==2.5.7 +astroid==2.14.2 +typing_extensions==4.4.0 + +# Different roman version for python 3 +roman==3.3 # Everything after this is the same for python 2 and 3 asttokens==2.0.5 @@ -23,7 +27,6 @@ iso8601==0.1.12 lazy_object_proxy==1.6.0 phonenumberslite==8.12.57 python_dateutil==2.8.2 -roman==2.0.0 singledispatch==3.6.2 six==1.16.0 sortedcontainers==2.4.0