diff --git a/.dockerignore b/.dockerignore index 1434837b..f2e04e8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,4 @@ !plugins !test !ext +**/_build diff --git a/.gitignore b/.gitignore index d26d0f07..48663300 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ jspm_packages/ # dotenv environment variables file .env + +**/_build diff --git a/README.md b/README.md index 56334441..27cde72a 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,7 @@ GRIST_SUPPORT_ANON | if set to 'true', show UI for anonymous access (not shown b GRIST_SUPPORT_EMAIL | if set, give a user with the specified email support powers. The main extra power is the ability to share sites, workspaces, and docs with all users in a listed way. GRIST_THROTTLE_CPU | if set, CPU throttling is enabled GRIST_USER_ROOT | an extra path to look for plugins in. +GRIST_WIDGET_LIST_URL | a url pointing to a widget manifest COOKIE_MAX_AGE | session cookie max age, defaults to 90 days; can be set to "none" to make it a session cookie HOME_PORT | port number to listen on for REST API server; if set to "share", add API endpoints to regular grist port. PORT | port number to listen on for Grist server diff --git a/app/server/lib/NSandbox.ts b/app/server/lib/NSandbox.ts index aff564f8..35e09ae1 100644 --- a/app/server/lib/NSandbox.ts +++ b/app/server/lib/NSandbox.ts @@ -15,7 +15,7 @@ import { } from 'app/server/lib/SandboxControl'; import * as sandboxUtil from 'app/server/lib/sandboxUtil'; import * as shutdown from 'app/server/lib/shutdown'; -import {ChildProcess, spawn} from 'child_process'; +import {ChildProcess, fork, spawn} from 'child_process'; import * as fs from 'fs'; import * as _ from 'lodash'; import * as path from 'path'; @@ -73,6 +73,8 @@ export interface ISandboxOptions { interface SandboxProcess { child: ChildProcess; control: ISandboxControl; + dataToSandboxDescriptor?: number; // override sandbox's 'stdin' for data + dataFromSandboxDescriptor?: number; // override sandbox's 'stdout' for data } type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void]; @@ -131,10 +133,23 @@ export class NSandbox implements ISandbox { if (options.minimalPipeMode) { log.rawDebug("3-pipe Sandbox started", this._logMeta); - this._streamToSandbox = this.childProc.stdin!; - this._streamFromSandbox = this.childProc.stdout!; + if (sandboxProcess.dataToSandboxDescriptor) { + this._streamToSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataToSandboxDescriptor] as Writable; + } else { + this._streamToSandbox = this.childProc.stdin!; + } + if (sandboxProcess.dataFromSandboxDescriptor) { + this._streamFromSandbox = + (this.childProc.stdio as Stream[])[sandboxProcess.dataFromSandboxDescriptor]; + } else { + this._streamFromSandbox = this.childProc.stdout!; + } } else { log.rawDebug("5-pipe Sandbox started", this._logMeta); + if (sandboxProcess.dataFromSandboxDescriptor || sandboxProcess.dataToSandboxDescriptor) { + throw new Error('cannot override file descriptors in 5 pipe mode'); + } this._streamToSandbox = (this.childProc.stdio as Stream[])[3] as Writable; this._streamFromSandbox = (this.childProc.stdio as Stream[])[4]; this.childProc.stdout!.on('data', sandboxUtil.makeLinePrefixer('Sandbox stdout: ', this._logMeta)); @@ -206,10 +221,17 @@ export class NSandbox implements ISandbox { * @param args Arguments to pass to the given function. * @returns A promise for the return value from the Python function. */ - public pyCall(funcName: string, ...varArgs: unknown[]): Promise { + public async pyCall(funcName: string, ...varArgs: unknown[]): Promise { const startTime = Date.now(); this._sendData(sandboxUtil.CALL, Array.from(arguments)); - return this._pyCallWait(funcName, startTime); + const slowCallCheck = setTimeout(() => { + // Log calls that take some time, can be a useful symptom of misconfiguration + // (or just benign if the doc is big). + log.rawWarn('Slow pyCall', {...this._logMeta, funcName}); + }, 10000); + const result = await this._pyCallWait(funcName, startTime); + clearTimeout(slowCallCheck); + return result; } /** @@ -371,6 +393,7 @@ const spawners = { docker, // Run sandboxes in distinct docker containers. gvisor, // Gvisor's runsc sandbox. macSandboxExec, // Use "sandbox-exec" on Mac. + pyodide, // Run data engine using pyodide. }; function isFlavor(flavor: string): flavor is keyof typeof spawners { @@ -528,6 +551,44 @@ function unsandboxed(options: ISandboxOptions): SandboxProcess { return {child, control: new DirectProcessControl(child, options.logMeta)}; } +function pyodide(options: ISandboxOptions): SandboxProcess { + const paths = getAbsolutePaths(options); + // We will fork with three regular pipes (stdin, stdout, stderr), then + // ipc (mandatory for calling fork), and a replacement pipe for stdin + // and for stdout. + // The regular stdin always opens non-blocking in node, which is a pain + // in this case, so we just use a different pipe. There's a different + // problem with stdout, with the same solution. + const spawnOptions = { + stdio: ['ignore', 'ignore', 'pipe', 'ipc', 'pipe', 'pipe'] as Array<'pipe'|'ipc'>, + env: { + PYTHONPATH: paths.engine, + IMPORTDIR: options.importDir, + ...getInsertedEnv(options), + ...getWrappingEnv(options), + } + }; + const base = getUnpackedAppRoot(); + const child = fork(path.join(base, 'sandbox', 'pyodide', 'pipe.js'), + {cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions}); + return { + child, + control: new DirectProcessControl(child, options.logMeta), + dataToSandboxDescriptor: 4, // Cannot use normal descriptor, node + // makes it non-blocking. Can be worked around in linux and osx, but + // for windows just using a different file descriptor seems simplest. + // In the sandbox, calling async methods from emscripten code is + // possible but would require more changes to the data engine code + // than seems reasonable at this time. The top level sandbox.run + // can be tweaked to step operations, which actually works for a + // lot of things, but not for cases where the sandbox calls back + // into node (e.g. for column type guessing). TLDR: just switching + // to FD 4 and reading synchronously is more practical solution. + dataFromSandboxDescriptor: 5, // There's an equally long but different + // story about why stdout is a bit messed up under pyodide right now. + }; +} + /** * Helper function to run python in gvisor's runsc, with multiple * sandboxes run within the same container. GRIST_SANDBOX should diff --git a/sandbox/grist/sandbox.py b/sandbox/grist/sandbox.py index fb4c4fb6..ff9727e2 100644 --- a/sandbox/grist/sandbox.py +++ b/sandbox/grist/sandbox.py @@ -14,6 +14,32 @@ import marshal import sys import traceback +class CarefulReader(object): + """ + Wrap a pipe when reading from Pyodide, to work around marshaling + panicking if fewer bytes are read in a block than it was expecting. + Just wait for more. + """ + + def __init__(self, file_): + self._file = file_ + + def write(self, data): + return self._file.write(data) + + def read(self, size): + return self._file.read(size) + + def readinto(self, b): + result = self._file.readinto(b) + while result is not None and result < len(b): + bview = memoryview(b) + result += self._file.readinto(bview[result:]) + return result + + def __getattr__(self, attr): + return getattr(self._file, attr) + def log(msg): sys.stderr.write(str(msg) + "\n") sys.stderr.flush() @@ -23,22 +49,25 @@ class Sandbox(object): This class works in conjunction with Sandbox.js to allow function calls between the Node process and this sandbox. - The sandbox provides two pipes (on fds 3 and 4) to send data to and from the sandboxed + The sandbox provides two pipes to send data to and from the sandboxed process. Data on these is serialized using `marshal` module. All messages are comprised of a msgCode followed immediatedly by msgBody, with the following msgCodes: CALL = call to the other side. The data must be an array of [func_name, arguments...] DATA = data must be a value to return to a call from the other side EXC = data must be an exception to return to a call from the other side + + Optionally, a callback can be supplied instead of an output pipe. """ CALL = None DATA = True EXC = False - def __init__(self, external_input, external_output): + def __init__(self, external_input, external_output, external_output_method=None): self._functions = {} self._external_input = external_input self._external_output = external_output + self._external_output_method = external_output_method @classmethod def connected_to_js_pipes(cls): @@ -62,6 +91,14 @@ class Sandbox(object): sys.stdout = sys.stderr return Sandbox.connected_to_js_pipes() + @classmethod + def use_pyodide(cls): + import js # Get pyodide object. + external_input = CarefulReader(sys.stdin.buffer) + external_output_method = lambda data: js.sendFromSandbox(data) + sys.stdout = sys.stderr + return cls(external_input, None, external_output_method) + def _send_to_js(self, msgCode, msgBody): # (Note that marshal version 2 is the default; we specify it explicitly for clarity. The # difference with version 0 is that version 2 uses a faster binary format for floats.) @@ -70,8 +107,14 @@ class Sandbox(object): # It's much better to ensure the whole blob is sent as one write. We marshal the resulting # buffer again so that the reader can quickly tell how many bytes to expect. buf = marshal.dumps((msgCode, msgBody), 2) - marshal.dump(buf, self._external_output, 2) - self._external_output.flush() + if self._external_output: + marshal.dump(buf, self._external_output, 2) + self._external_output.flush() + elif self._external_output_method: + buf = marshal.dumps(buf, 2) + self._external_output_method(buf) + else: + raise Exception('no data output method') def call_external(self, name, *args): self._send_to_js(Sandbox.CALL, (name,) + args) @@ -115,6 +158,8 @@ def get_default_sandbox(): if default_sandbox is None: if os.environ.get('PIPE_MODE') == 'minimal': default_sandbox = Sandbox.use_common_pipes() + elif os.environ.get('PIPE_MODE') == 'pyodide': + default_sandbox = Sandbox.use_pyodide() else: default_sandbox = Sandbox.connected_to_js_pipes() return default_sandbox diff --git a/sandbox/grist/test_renames2.py b/sandbox/grist/test_renames2.py index 57a58713..d453297f 100644 --- a/sandbox/grist/test_renames2.py +++ b/sandbox/grist/test_renames2.py @@ -189,23 +189,21 @@ class TestRenames2(test_engine.EngineTestCase): def test_renames_b(self): # Rename Games.name: affects People.Games_Won, Games.win4_game_name - # TODO: win4_game_name isn't updated due to astroid avoidance of looking up the same attr on - # the same class during inference. out_actions = self.apply_user_action(["RenameColumn", "Games", "name", "nombre"]) self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "Games", "name", "nombre"], ["ModifyColumn", "People", "Games_Won", { "formula": "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))" }], - ["BulkUpdateRecord", "_grist_Tables_column", [4, 12], { - "colId": ["nombre", "Games_Won"], + ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.win.win.win.nombre"}], + ["BulkUpdateRecord", "_grist_Tables_column", [4, 12, 19], { + "colId": ["nombre", "Games_Won", "win4_game_name"], "formula": [ - "", "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))"] - }], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "", + "' '.join(e.game.nombre for e in Entries.lookupRecords(person=$id, rank=1))", + "$win.win.win.win.nombre" + ] + }] ]}) # Fix up things missed due to the TODOs above. @@ -264,22 +262,16 @@ class TestRenames2(test_engine.EngineTestCase): def test_renames_d(self): # Rename People.name: affects People.N, People.ParnerNames - # TODO: win3_person_name ($win.win.win.name) does NOT get updated correctly with astroid - # because of a limitation in astroid inference: it refuses to look up the same attr on the - # same class during inference (in order to protect against too much recursion). # TODO: PartnerNames does NOT get updated correctly because astroid doesn't infer meanings of # lists very well. out_actions = self.apply_user_action(["RenameColumn", "People", "name", "nombre"]) self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "name", "nombre"], ["ModifyColumn", "People", "N", {"formula": "$nombre.upper()"}], - ["BulkUpdateRecord", "_grist_Tables_column", [2, 11], { - "colId": ["nombre", "N"], - "formula": ["", "$nombre.upper()"] - }], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win3_person_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] + ["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.win.win.nombre"}], + ["BulkUpdateRecord", "_grist_Tables_column", [2, 11, 18], { + "colId": ["nombre", "N", "win3_person_name"], + "formula": ["", "$nombre.upper()", "$win.win.win.nombre"] }], ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { "PartnerNames": [["E", "AttributeError"], ["E", "AttributeError"], @@ -287,8 +279,7 @@ class TestRenames2(test_engine.EngineTestCase): }], ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("Games", "win3_person_name", formula="$win.win.win.nombre") + # Fix up things missed due to the TODO above. self.modify_column("People", "PartnerNames", formula=self.partner_names.replace("name", "nombre")) @@ -305,21 +296,14 @@ class TestRenames2(test_engine.EngineTestCase): self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "partner", "companero"], ["ModifyColumn", "People", "partner4", { - "formula": "$companero.companero.partner.partner" + "formula": "$companero.companero.companero.companero" }], ["BulkUpdateRecord", "_grist_Tables_column", [14, 15], { "colId": ["companero", "partner4"], - "formula": [self.partner, "$companero.companero.partner.partner"] - }], - ["BulkUpdateRecord", "People", [1, 2, 3, 4, 5], { - "partner4": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "formula": [self.partner, "$companero.companero.companero.companero"] + }] ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("People", "partner4", formula="$companero.companero.companero.companero") - _replace_col_name(self.people_data, "partner", "companero") self.assertTableData("People", cols="subset", data=self.people_data) self.assertTableData("Games", cols="subset", data=self.games_data) @@ -331,21 +315,13 @@ class TestRenames2(test_engine.EngineTestCase): self.assertPartialOutActions(out_actions, { "stored": [ ["RenameColumn", "People", "win", "pwin"], ["ModifyColumn", "Games", "win3_person_name", {"formula": "$win.pwin.win.name"}], - # TODO: the omission of the 4th win's update is due to the same astroid bug mentioned above. - ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.win.name"}], + ["ModifyColumn", "Games", "win4_game_name", {"formula": "$win.pwin.win.pwin.name"}], ["BulkUpdateRecord", "_grist_Tables_column", [16, 18, 19], { "colId": ["pwin", "win3_person_name", "win4_game_name"], "formula": ["Entries.lookupOne(person=$id, rank=1).game", - "$win.pwin.win.name", "$win.pwin.win.win.name"]}], - ["BulkUpdateRecord", "Games", [1, 2, 3, 4], { - "win4_game_name": [["E", "AttributeError"], ["E", "AttributeError"], - ["E", "AttributeError"], ["E", "AttributeError"]] - }], + "$win.pwin.win.name", "$win.pwin.win.pwin.name"]}], ]}) - # Fix up things missed due to the TODOs above. - self.modify_column("Games", "win4_game_name", formula="$win.pwin.win.pwin.name") - _replace_col_name(self.people_data, "win", "pwin") self.assertTableData("People", cols="subset", data=self.people_data) self.assertTableData("Games", cols="subset", data=self.games_data) diff --git a/sandbox/pyodide/Makefile b/sandbox/pyodide/Makefile new file mode 100644 index 00000000..2adc18c5 --- /dev/null +++ b/sandbox/pyodide/Makefile @@ -0,0 +1,19 @@ +default: + echo "Welcome to the pyodide sandbox" + echo "make fetch_packages # gets python packages prepared earlier" + echo "make build_packages # build python packages from scratch" + echo "make save_packages # upload python packages to fetch later" + echo "setup # get pyodide node package" + +fetch_packages: + node ./packages.js https://s3.amazonaws.com/grist-pynbox/pyodide/packages/ _build/packages/ + +build_packages: + ./build_packages.sh + +save_packages: + aws s3 sync _build/packages s3://grist-pynbox/pyodide/packages/ + +setup: + ./setup.sh + make fetch_packages diff --git a/sandbox/pyodide/README.md b/sandbox/pyodide/README.md new file mode 100644 index 00000000..4384d3ec --- /dev/null +++ b/sandbox/pyodide/README.md @@ -0,0 +1,28 @@ +This is a collection of scripts for running a pyodide-based "sandbox" for +Grist. + +I put "sandbox" in quotes since pyodide isn't built with sandboxing +in mind. It was written to run in a browser, where the browser does +sandboxing. I don't know how much of node's API ends up being exposed +to the "sandbox" - in previous versions of pyodide it seems the answer is +"a lot". See the back-and-forth between dalcde and hoodmane in: + https://github.com/pyodide/pyodide/issues/960 +See specifically: + https://github.com/pyodide/pyodide/issues/960#issuecomment-752305257 +I looked at hiwire and its treatment of js globals has changed a +lot. On the surface it looks like there is good control of what is +exposed, but there may be other routes. + +Still, some wasm-based solution is likely to be helpful, whether from +pyodide or elsewhere, and this is good practice for that. + +*** + +To run, we need specific versions of the Python packages that Grist uses +to be prepared. It should suffice to do: + +``` +make setup +``` + +In this directory. See the `Makefile` for other options. diff --git a/sandbox/pyodide/build_packages.sh b/sandbox/pyodide/build_packages.sh new file mode 100644 index 00000000..2354c831 --- /dev/null +++ b/sandbox/pyodide/build_packages.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +echo "" +echo "###############################################################" +echo "## Get pyodide repository, for transpiling python packages" + +if [[ ! -e _build/pyodide ]]; then + cd _build + git clone https://github.com/pyodide/pyodide + cd .. +fi + +echo "" +echo "###############################################################" +echo "## Prepare python packages" + +cd _build/pyodide +./run_docker make +cp ../../../requirements3.txt . +./run_docker pyodide build -r requirements3.txt --output-lockfile result.txt +cat result.txt +cd ../.. + +echo "" +echo "###############################################################" +echo "## Copy out python packages" + +node ./packages.js _build/pyodide/dist/ _build/packages/ diff --git a/sandbox/pyodide/packages.js b/sandbox/pyodide/packages.js new file mode 100644 index 00000000..fa0f03c6 --- /dev/null +++ b/sandbox/pyodide/packages.js @@ -0,0 +1,117 @@ +const path = require('path'); +const fs = require('fs'); +const fetch = require('node-fetch'); + +async function listLibs(src) { + const txt = fs.readFileSync(path.join(__dirname, '..', 'requirements3.txt'), 'utf8'); + const libs = {}; + for (const line of txt.split(/\r?\n/)) { + const raw = line.split('#')[0]; + if (!raw.includes('==')) { continue; } + const [name, version] = line.split('=='); + libs[name] = version; + } + const hits = []; + const misses = []; + const toLoad = []; + const material = fs.readdirSync(src); + for (const [lib, version] of Object.entries(libs)) { + const nlib = lib.replace(/-/g, '_'); + const info = { + name: lib, + standardName: nlib, + version: version, + } + try { + const found = material.filter(m => m.startsWith(`${nlib}-${version}-`)); + if (found.length !== 1) { + throw new Error('did not find 1'); + } + const fname = found[0]; + info.fullName = path.join(src, fname); + info.fileName = fname; + toLoad.push(info); + hits.push(lib); + } catch (e) { + misses.push(info); + } + } + return { + available: toLoad, + misses, + }; +} +exports.listLibs = listLibs; + +async function findOnDisk(src, dest) { + console.log(`Organizing packages on disk`, {src, dest}); + fs.mkdirSync(dest, {recursive: true}); + let libs = (await listLibs(src)); + for (const lib of libs.available) { + fs.copyFileSync(lib.fullName, path.join(dest, lib.fileName)); + fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`), + JSON.stringify({ + name: lib.name, + version: lib.version, + fileName: lib.fileName, + }, null, 2)); + console.log("Copied", { + content: path.join(dest, lib.fileName), + meta: path.join(dest, `${lib.name}-${lib.version}.json`), + }); + } + libs = await listLibs(dest); + console.log(`Cached`, {libs: libs.available.map(lib => lib.name)}); + console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)}); +} + +async function findOnNet(src, dest) { + console.log(`Caching packages on disk`, {src, dest}); + fs.mkdirSync(dest, {recursive: true}); + let libs = await listLibs(dest); + console.log(`Cached`, {libs: libs.available.map(lib => lib.name)}); + for (const lib of libs.misses) { + console.log('Fetching', lib); + const url = new URL(src); + url.pathname = url.pathname + lib.name + '-' + lib.version + '.json'; + const result = await fetch(url.href); + if (result.status === 200) { + const data = await result.json(); + const url2 = new URL(src); + url2.pathname = url2.pathname + data.fileName; + const result2 = await fetch(url2.href); + if (result2.status === 200) { + fs.writeFileSync(path.join(dest, `${lib.name}-${lib.version}.json`), + JSON.stringify(data, null, 2)); + fs.writeFileSync(path.join(dest, data.fileName), + await result2.buffer()); + } else { + console.error("No payload available", {lib}); + } + } else { + console.error("No metadata available", {lib}); + } + } + libs = await listLibs(dest); + console.log(`Missing`, {libs: libs.misses.map(lib => lib.name)}); +} + +async function main(src, dest) { + if (!src) { + console.error('please supply a source'); + process.exit(1); + } + if (!dest) { + console.error('please supply a destination'); + process.exit(1); + } + if (src.startsWith('http:') || src.startsWith('https:')) { + await findOnNet(src, dest); + return; + } + await findOnDisk(src, dest); +} + +if (require.main === module) { + main(...process.argv.slice(2)).catch(e => console.error(e)); +} diff --git a/sandbox/pyodide/pipe.js b/sandbox/pyodide/pipe.js new file mode 100644 index 00000000..09ffeea7 --- /dev/null +++ b/sandbox/pyodide/pipe.js @@ -0,0 +1,147 @@ +const path = require('path'); +const fs = require('fs'); + +const { loadPyodide } = require('./_build/worker/node_modules/pyodide'); +const { listLibs } = require('./packages'); + +const INCOMING_FD = 4; +const OUTGOING_FD = 5; + +class GristPipe { + constructor() { + this.pyodide = null; + this.incomingBuffer = Buffer.alloc(65536); + this.addedBlob = false; + this.adminMode = false; + } + + async init() { + const self = this; + this.setAdminMode(true); + this.pyodide = await loadPyodide({ + jsglobals: { + Object: {}, + setTimeout: function(code, delay) { + if (self.adminMode) { + setTimeout(code, delay); + // Seems to be OK not to return anything, so we don't. + } else { + throw new Error('setTimeout not available'); + } + }, + sendFromSandbox: (data) => { + return fs.writeSync(OUTGOING_FD, Buffer.from(data.toJs())); + } + }, + }); + this.setAdminMode(false); + this.pyodide.setStdin({ + stdin: () => { + const result = fs.readSync(INCOMING_FD, this.incomingBuffer, 0, + this.incomingBuffer.byteLength); + if (result > 0) { + const buf = Buffer.allocUnsafe(result, 0, 0, result); + this.incomingBuffer.copy(buf); + return buf; + } + return null; + }, + }); + this.pyodide.setStderr({ + batched: (data) => { + this.log("[py]", data); + } + }); + } + + async loadCode() { + // Load python packages. + const src = path.join(__dirname, '_build', 'packages'); + const lsty = (await listLibs(src)).available.map(item => item.fullName); + await this.pyodide.loadPackage(lsty, { + messageCallback: (msg) => this.log('[package]', msg), + }); + + // Load Grist data engine code. + // We mount it as /grist_src, copy to /grist, then unmount. + // Note that path to source must be a realpath. + const root = fs.realpathSync(path.join(__dirname, '../grist')); + await this.pyodide.FS.mkdir("/grist_src"); + // careful, needs to be a realpath + await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, { root }, "/grist_src"); + // Now want to copy /grist_src to /grist. + // For some reason shutil.copytree doesn't work on Windows in this situation, so + // we reimplement it crudely. + await this.pyodide.runPython(` +import os, shutil +def copytree(src, dst): + os.makedirs(dst, exist_ok=True) + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + copytree(s, d) + else: + shutil.copy2(s, d) +copytree('/grist_src', '/grist')`); + await this.pyodide.FS.unmount("/grist_src"); + await this.pyodide.FS.rmdir("/grist_src"); + } + + async mountImportDirIfNeeded() { + if (process.env.IMPORTDIR) { + this.log("Setting up import from", process.env.IMPORTDIR); + // Ideally would be read-only; don't see a way to do that, + // other than copying like for Grist code. + await this.pyodide.FS.mkdir("/import"); + await this.pyodide.FS.mount(this.pyodide.FS.filesystems.NODEFS, { + root: process.env.IMPORTDIR, + }, "/import"); + } + } + + async runCode() { + await this.pyodide.runPython(` + import sys + sys.path.append('/') + sys.path.append('/grist') + import grist + import main + import os + os.environ['PIPE_MODE'] = 'pyodide' + os.environ['IMPORTDIR'] = '/import' + main.main() +`); + } + + setAdminMode(active) { + this.adminMode = active; + // Lack of Blob may result in a message on console.log that hurts us. + if (active && !globalThis.Blob) { + globalThis.Blob = String; + this.addedBlob = true; + } + if (!active && this.addedBlob) { + delete globalThis.Blob; + this.addedBlob = false; + } + } + + log(...args) { + console.error("[pyodide sandbox]", ...args); + } +} + +async function main() { + try { + const pipe = new GristPipe(); + await pipe.init(); + await pipe.loadCode(); + await pipe.mountImportDirIfNeeded(); + await pipe.runCode(); + } finally { + process.stdin.removeAllListeners(); + } +} + +main().catch(err => console.error("[pyodide error]", err)); diff --git a/sandbox/pyodide/setup.sh b/sandbox/pyodide/setup.sh new file mode 100755 index 00000000..ce28cdc8 --- /dev/null +++ b/sandbox/pyodide/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +echo "" +echo "###############################################################" +echo "## Get pyodide node package" + +if [[ ! -e _build/worker ]]; then + mkdir -p _build/worker + cd _build/worker + yarn init --yes + yarn add pyodide@0.22.1 + cd ../.. +fi diff --git a/sandbox/requirements3.txt b/sandbox/requirements3.txt index 6fec0ea4..22f2995a 100644 --- a/sandbox/requirements3.txt +++ b/sandbox/requirements3.txt @@ -12,7 +12,11 @@ jdcal==1.4.1 et-xmlfile==1.0.1 # Different astroid version for python 3 -astroid==2.5.7 +astroid==2.14.2 +typing_extensions==4.4.0 + +# Different roman version for python 3 +roman==3.3 # Everything after this is the same for python 2 and 3 asttokens==2.0.5 @@ -23,7 +27,6 @@ iso8601==0.1.12 lazy_object_proxy==1.6.0 phonenumberslite==8.12.57 python_dateutil==2.8.2 -roman==2.0.0 singledispatch==3.6.2 six==1.16.0 sortedcontainers==2.4.0 diff --git a/static/locales/de.client.json b/static/locales/de.client.json index a66521e2..b85f1b73 100644 --- a/static/locales/de.client.json +++ b/static/locales/de.client.json @@ -723,7 +723,9 @@ "Open configuration": "Konfiguration öffnen", "Print widget": "Widget drucken", "Show raw data": "Rohdaten anzeigen", - "Widget options": "Widget Optionen" + "Widget options": "Widget Optionen", + "Add to page": "Zur Seite hinzufügen", + "Collapse widget": "Widget einklappen" }, "ViewSectionMenu": { "(customized)": "(angepasst)", @@ -1021,5 +1023,8 @@ "entire": "gesamte", "Access Rules": "Zugriffsregeln", "Access rules give you the power to create nuanced rules to determine who can see or edit which parts of your document.": "Zugriffsregeln geben Ihnen die Möglichkeit, nuancierte Regeln zu erstellen, um festzulegen, wer welche Teile Ihres Dokuments sehen oder bearbeiten kann." + }, + "DescriptionConfig": { + "DESCRIPTION": "BESCHREIBUNG" } } diff --git a/static/locales/en.client.json b/static/locales/en.client.json index b3366170..fdcab730 100644 --- a/static/locales/en.client.json +++ b/static/locales/en.client.json @@ -671,7 +671,9 @@ "Open configuration": "Open configuration", "Print widget": "Print widget", "Show raw data": "Show raw data", - "Widget options": "Widget options" + "Widget options": "Widget options", + "Add to page": "Add to page", + "Collapse widget": "Collapse widget" }, "ViewSectionMenu": { "(customized)": "(customized)", @@ -957,5 +959,8 @@ "Access rules give you the power to create nuanced rules to determine who can see or edit which parts of your document.": "Access rules give you the power to create nuanced rules to determine who can see or edit which parts of your document.", "Add New": "Add New", "Use the 𝚺 icon to create summary (or pivot) tables, for totals or subtotals.": "Use the 𝚺 icon to create summary (or pivot) tables, for totals or subtotals." + }, + "DescriptionConfig": { + "DESCRIPTION": "DESCRIPTION" } } diff --git a/static/locales/es.client.json b/static/locales/es.client.json index b74568df..8cdf8318 100644 --- a/static/locales/es.client.json +++ b/static/locales/es.client.json @@ -580,7 +580,9 @@ "Open configuration": "Abrir configuración", "Print widget": "Imprimir widget", "Show raw data": "Mostrar datos brutos", - "Widget options": "Opciones de Widget" + "Widget options": "Opciones de Widget", + "Collapse widget": "Ocultar el widget", + "Add to page": "Añadir a la página" }, "ViewSectionMenu": { "(customized)": "(personalizado)", @@ -1011,5 +1013,8 @@ "Access Rules": "Reglas de acceso", "Use the 𝚺 icon to create summary (or pivot) tables, for totals or subtotals.": "Utilice el icono 𝚺 para crear tablas resumen (o dinámicas), para totales o subtotales.", "Unpin to hide the the button while keeping the filter.": "Desancla para ocultar el botón mientras mantienes el filtro." + }, + "DescriptionConfig": { + "DESCRIPTION": "DESCRIPCIÓN" } } diff --git a/static/locales/pl.client.json b/static/locales/pl.client.json index 4c479bd7..aed67610 100644 --- a/static/locales/pl.client.json +++ b/static/locales/pl.client.json @@ -457,9 +457,371 @@ "Ask for help": "Zapytaj o pomoc", "Cannot find personal site, sorry!": "Nie mogę znaleźć osobistej strony, przepraszam!", "Give feedback": "Przekaż opinię", - "Notifications": "Powiadomienia" + "Notifications": "Powiadomienia", + "Upgrade Plan": "Plan aktualizacji", + "Renew": "Odnów", + "Report a problem": "Zgłoś problem" }, "LeftPanelCommon": { "Help Center": "Centrum pomocy" + }, + "PageWidgetPicker": { + "Add to Page": "Dodaj do strony", + "Select Widget": "Wybierz widżet", + "Group by": "Grupuj według", + "Building {{- label}} widget": "Budowa widżetu {{- label}}", + "Select Data": "Wybierz Dane" + }, + "OpenVideoTour": { + "Grist Video Tour": "Wideo Przewodnik Grist", + "YouTube video player": "Odtwarzacz wideo YouTube", + "Video Tour": "Prezentacja wideo" + }, + "Pages": { + "Delete": "Usunąć", + "Delete data and this page.": "Usuń dane i tę stronę.", + "The following tables will no longer be visible_one": "Poniższa tabela nie będzie już widoczna", + "The following tables will no longer be visible_other": "Poniższa tabela nie będzie już widoczna" + }, + "PermissionsWidget": { + "Allow All": "Pozwól wszystkim", + "Deny All": "Odmówić wszystkim", + "Read Only": "Tylko do odczytu" + }, + "RecordLayoutEditor": { + "Cancel": "Anuluj", + "Add Field": "Dodaj pole", + "Show field {{- label}}": "Pokaż pole {{- label}}", + "Save Layout": "Zapisz układ", + "Create New Field": "Utwórz nowe pole" + }, + "RefSelect": { + "Add Column": "Dodaj kolumnę", + "No columns to add": "Brak kolumn do dodania" + }, + "RightPanel": { + "Data": "Dane", + "Detach": "Odłączyć", + "Fields_other": "Pola", + "SOURCE DATA": "DANE ŹRÓDŁOWE", + "Select Widget": "Wybierz widżet", + "CUSTOM": "NIESTANDARDOWE", + "Save": "Zapisz", + "Widget": "Widżet", + "Columns_other": "Kolumny", + "Columns_one": "Kolumna", + "DATA TABLE": "TABELA DANYCH", + "DATA TABLE NAME": "NAZWA TABELI DANYCH", + "GROUPED BY": "POGRUPOWANE WEDŁUG", + "ROW STYLE": "STYL WIERSZA", + "Row Style": "Styl wiersza", + "SELECT BY": "WYBIERZ WEDŁUG", + "TRANSFORM": "PRZEKSZTAŁCAĆ", + "Theme": "Motyw", + "WIDGET TITLE": "TYTUŁ WIDŻETU", + "Series_one": "Seria", + "CHART TYPE": "TYP WYKRESU", + "COLUMN TYPE": "TYP KOLUMNY", + "Change Widget": "Zmień widżet", + "Edit Data Selection": "Edytuj zaznaczenie danych", + "Fields_one": "Pole", + "SELECTOR FOR": "WYBÓR DLA", + "Sort & Filter": "Sortowanie i filtrowanie", + "Series_other": "Seria", + "You do not have edit access to this document": "nie masz uprawnień do edytowania tego dokumentu" + }, + "RowContextMenu": { + "Delete": "Usuń", + "Insert row": "Wstawić wiersz", + "Insert row above": "Wstawić wiersz powyżej", + "Insert row below": "Wstawić wiersz poniżej", + "Copy anchor link": "Kopiowanie łącza zakotwiczenia", + "Duplicate rows_one": "Duplikuj wiersz", + "Duplicate rows_other": "Duplikuj wiersze" + }, + "ShareMenu": { + "Back to Current": "Powrót do aktualnego", + "Download": "Pobierz", + "Send to Google Drive": "Wyślij na Dysk Google", + "Work on a Copy": "Praca nad kopią", + "Edit without affecting the original": "Edycja bez wpływu na oryginał", + "Duplicate Document": "Duplikat dokumentu", + "Show in folder": "Pokaż w folderze", + "Unsaved": "Niezapisane", + "Access Details": "Szczegóły dostępu", + "Original": "Oryginał", + "Replace {{termToUse}}...": "Zastąp {{termToUse}}…", + "Return to {{termToUse}}": "Wróć do {{termToUse}}", + "Save Copy": "Zapisz kopię", + "Save Document": "Zapisz dokument", + "Compare to {{termToUse}}": "Porównaj z {{termToUse}}", + "Export CSV": "Eksportuj plik CSV", + "Current Version": "Obecna wersja", + "Export XLSX": "Eksportuj XLSX", + "Manage Users": "Zarządzanie użytkownikami" + }, + "SiteSwitcher": { + "Switch Sites": "Przełączanie witryn", + "Create new team site": "Utwórz nową witrynę zespołu" + }, + "SortConfig": { + "Add Column": "Dodaj kolumnę", + "Empty values last": "Puste wartości jako ostatnie", + "Natural sort": "Naturalne sortowanie", + "Update Data": "Aktualizowanie danych", + "Search Columns": "Wyszukaj kolumny", + "Use choice position": "Użyj pozycji wyboru" + }, + "Tools": { + "Document History": "Historia dokumentu", + "Return to viewing as yourself": "Powrót do oglądania jako Ty", + "Access Rules": "Zasady dostępu", + "Code View": "Widok kodu", + "Delete": "Usuń", + "Delete document tour?": "Usunąć wycieczkę po dokumencie?", + "Raw Data": "Surowe dane", + "Validate Data": "Sprawdzanie poprawności danych", + "TOOLS": "NARZĘDZIA", + "Settings": "Ustawienia", + "How-to Tutorial": "Samouczek", + "Tour of this Document": "Przewodnik po tym dokumencie" + }, + "TriggerFormulas": { + "Apply on changes to:": "Zastosuj w przypadku zmian do:", + "Apply on record changes": "Zastosuj zmiany w rekordzie", + "Cancel": "Anuluj", + "Any field": "Dowolne pole", + "Close": "Zamknij", + "OK": "OK", + "Current field ": "Bieżące pole ", + "Apply to new records": "Zastosuj do nowych rekordów" + }, + "TypeTransformation": { + "Apply": "Zastosuj", + "Cancel": "Anuluj", + "Preview": "Podgląd", + "Revise": "Zrewidować", + "Update formula (Shift+Enter)": "Aktualizacja formuły (Shift+Enter)" + }, + "UserManagerModel": { + "Editor": "Edytor", + "View & Edit": "Wyświetlanie i edytowanie", + "None": "Żaden", + "In Full": "W pełni", + "No Default Access": "Brak dostępu domyślnego", + "Owner": "Właściciel", + "View Only": "Tylko widok", + "Viewer": "Przeglądarka" + }, + "ValidationPanel": { + "Update formula (Shift+Enter)": "Aktualizuj formułę (Shift+Enter)", + "Rule {{length}}": "Zasada {{length}}" + }, + "ViewConfigTab": { + "Make On-Demand": "Twórz na żądanie", + "Plugin: ": "Wtyczka: ", + "Advanced settings": "Ustawienia zaawansowane", + "Blocks": "Bloki", + "Compact": "Kompaktowy", + "Form": "Formularz", + "Edit Card Layout": "Edytuj układ karty", + "Section: ": "Sekcja: ", + "Unmark On-Demand": "Odznacz opcję Na żądanie", + "Big tables may be marked as \"on-demand\" to avoid loading them into the data engine.": "Duże tabele mogą być oznaczone jako \"na żądanie\", aby uniknąć ładowania ich do silnika danych." + }, + "ViewLayoutMenu": { + "Copy anchor link": "Kopiowanie łącza zakotwiczenia", + "Data selection": "Wybór danych", + "Delete record": "Usuń rekord", + "Open configuration": "Otwórz konfiguracje", + "Print widget": "Drukuj widżet", + "Add to page": "Dodaj do strony", + "Collapse widget": "Zwiń widżet", + "Download as XLSX": "Pobierz jako XLSX", + "Edit Card Layout": "Edytuj układ karty", + "Delete widget": "Usuń widżet", + "Download as CSV": "Pobierz jako CSV", + "Show raw data": "Pokaż surowe dane", + "Widget options": "Opcje widżetów", + "Advanced Sort & Filter": "Zaawansowane sortowanie i filtrowanie" + }, + "ViewSectionMenu": { + "Custom options": "Opcje niestandardowe", + "Revert": "Przywrócić", + "SORT": "SORTOWAĆ", + "Save": "Zapisz", + "(customized)": "(dostosowane)", + "(empty)": "(pusty)", + "(modified)": "(zmodyfikowany)", + "FILTER": "FILTR", + "Update Sort&Filter settings": "Zaktualizuj ustawienia sortowania i filtrowania" + }, + "VisibleFieldsConfig": { + "Clear": "Wyczyść", + "Select All": "Wybierz wszystko", + "Visible {{label}}": "Widoczny {{label}}", + "Hide {{label}}": "Ukryj {{label}}", + "Hidden {{label}}": "Ukryty {{label}}", + "Show {{label}}": "Pokaż {{label}}", + "Cannot drop items into Hidden Fields": "Nie można upuszczać elementów do ukrytych pól", + "Hidden Fields cannot be reordered": "Pola ukryte nie mogą być ponownie uporządkowane" + }, + "WelcomeQuestions": { + "Education": "Edukacja", + "Research": "Badania", + "Type here": "Wpisz tutaj", + "Welcome to Grist!": "Witamy w Grist!", + "What brings you to Grist? Please help us serve you better.": "Co sprowadza cię do Grist? Pomóż nam lepiej Ci służyć.", + "Product Development": "Rozwój produktu", + "IT & Technology": "IT i technologia", + "Marketing": "Marketing", + "Media Production": "Produkcja medialna", + "Other": "Inne", + "Sales": "Sprzedaż", + "Finance & Accounting": "Finanse i Księgowość", + "HR & Management": "HR i zarządzanie" + }, + "WidgetTitle": { + "DATA TABLE NAME": "NAZWA TABELI DANYCH", + "Override widget title": "Zastąp tytuł widżetu", + "WIDGET TITLE": "TYTUŁ WIDŻETU", + "Provide a table name": "Podaj nazwę tabeli", + "Cancel": "Anulować", + "Save": "Zapisz" + }, + "breadcrumbs": { + "fiddle": "skrzypce", + "override": "nadpisać", + "snapshot": "migawka", + "unsaved": "niezapisany", + "recovery mode": "tryb odzyskiwania", + "You may make edits, but they will create a new copy and will\nnot affect the original document.": "Możesz wprowadzać zmiany, ale one utworzą nową kopię i\nnie mają wpływu na oryginalny dokument." + }, + "duplicatePage": { + "Note that this does not copy data, but creates another view of the same data.": "Zauważ, że to nie kopiuje danych, ale tworzy inny widok tych samych danych.", + "Duplicate page {{pageName}}": "Duplikat strony {{pageName}}" + }, + "errorPages": { + "Access denied{{suffix}}": "Odmowa dostępu{{suffix}}", + "Go to main page": "Przejdź do strony głównej", + "Page not found{{suffix}}": "Nie znaleziono strony{{suffix}}", + "Sign in again": "Zaloguj się ponownie", + "Signed out{{suffix}}": "Wylogowano{{suffix}}", + "Something went wrong": "Coś poszło nie tak", + "There was an error: {{message}}": "Wystąpił błąd: {{message}}", + "There was an unknown error.": "Wystąpił nieznany błąd.", + "You are now signed out.": "Jesteś teraz wylogowany.", + "You do not have access to this organization's documents.": "Nie ma dostępu do dokumentów tej organizacji.", + "Error{{suffix}}": "Błąd{{suffix}}", + "Sign in": "Zaloguj się", + "Add account": "Dodaj konto", + "Contact support": "Skontaktuj się z pomocą techniczną", + "Sign in to access this organization's documents.": "Zaloguj się, aby uzyskać dostęp do dokumentów tej organizacji.", + "The requested page could not be found.{{separator}}Please check the URL and try again.": "Nie można odnaleźć żądanej strony. {{separator}} Sprawdź adres URL i spróbuj ponownie.", + "You are signed in as {{email}}. You can sign in with a different account, or ask an administrator for access.": "Jesteś zalogowany jako {{email}}. Możesz zalogować się na inne konto lub poprosić administratora o dostęp." + }, + "menus": { + "* Workspaces are available on team plans. ": "* Obszary robocze są dostępne w planach zespołowych. ", + "Upgrade now": "Uaktualnij teraz", + "Any": "Dowolny", + "Integer": "Liczba całkowita", + "Toggle": "Przełączać", + "Date": "Data", + "Choice": "Wybór", + "Choice List": "Lista wyboru", + "Reference": "Odnośnik", + "Reference List": "Lista referencyjna", + "Select fields": "Wybierz pola", + "Numeric": "Numeryczny", + "Text": "Tekst", + "DateTime": "DataGodzina", + "Attachment": "Załącznik" + }, + "modals": { + "Cancel": "Anuluj", + "Save": "Zapisz", + "Ok": "OK" + }, + "pages": { + "You do not have edit access to this document": "Nie masz dostępu do edycji tego dokumentu", + "Duplicate Page": "Duplikat strony", + "Remove": "Usuń", + "Rename": "Zmień nazwę" + }, + "search": { + "Search in document": "Szukaj w dokumencie", + "Find Next ": "Znajdź następny ", + "Find Previous ": "Znajdź poprzedni ", + "No results": "Brak wyników" + }, + "NTextBox": { + "true": "prawdziwy", + "false": "fałszywy" + }, + "ACLUsers": { + "Users from table": "Użytkownicy z tabeli", + "View As": "Wyświetl jako", + "Example Users": "Przykładowi Użytkownicy" + }, + "TypeTransform": { + "Cancel": "Anulować", + "Preview": "Podgląd", + "Revise": "Zrewidować", + "Apply": "Zastosuj", + "Update formula (Shift+Enter)": "Aktualizuj formułę (Shift+Enter)" + }, + "CellStyle": { + "Cell Style": "Styl komórki", + "Default cell style": "Domyślny styl komórki", + "Mixed style": "Styl mieszany", + "Open row styles": "Otwórz style wierszy", + "CELL STYLE": "STYL KOMÓRKI" + }, + "ColumnEditor": { + "COLUMN DESCRIPTION": "OPIS KOLUMNY", + "COLUMN LABEL": "ETYKIETA KOLUMNY" + }, + "ColumnInfo": { + "COLUMN ID: ": "ID KOLUMNY: ", + "COLUMN LABEL": "ETYKIETA KOLUMNY", + "Cancel": "Anuluj", + "Save": "Zapisz", + "COLUMN DESCRIPTION": "OPIS KOLUMNY" + }, + "OnBoardingPopups": { + "Next": "Następny", + "Finish": "Skończyć" + }, + "SortFilterConfig": { + "Revert": "Przywrócić", + "Filter": "FILTR", + "Save": "Zapisz", + "Sort": "SORTOWAĆ", + "Update Sort & Filter settings": "Zaktualizuj ustawienia sortowania i filtrowania" + }, + "ViewAsBanner": { + "UnknownUser": "Nieznany użytkownik" + }, + "SelectionSummary": { + "Copied to clipboard": "Skopiowane do schowka" + }, + "PluginScreen": { + "Import failed: ": "Importowanie nie powiodło się: " + }, + "RecordLayout": { + "Updating record layout.": "Aktualizowanie układu rekordu." + }, + "ThemeConfig": { + "Appearance ": "Wygląd ", + "Switch appearance automatically to match system": "Automatyczne przełączanie wyglądu w celu dopasowania do systemu" + }, + "TopBar": { + "Manage Team": "Zarządzaj zespołem" + }, + "sendToDrive": { + "Sending file to Google Drive": "Wysyłanie pliku do Google Drive" + }, + "ChoiceTextBox": { + "CHOICES": "WYBORY" } } diff --git a/static/locales/pt_BR.client.json b/static/locales/pt_BR.client.json index 2c4be4a1..2bbb9810 100644 --- a/static/locales/pt_BR.client.json +++ b/static/locales/pt_BR.client.json @@ -723,7 +723,9 @@ "Open configuration": "Abrir configuração", "Print widget": "Imprimir Widget", "Show raw data": "Mostrar dados primários", - "Widget options": "Opções do Widget" + "Widget options": "Opções do Widget", + "Collapse widget": "Colapsar widget", + "Add to page": "Adicionar à página" }, "ViewSectionMenu": { "(customized)": "(personalizado)", @@ -1021,5 +1023,8 @@ "Use the 𝚺 icon to create summary (or pivot) tables, for totals or subtotals.": "Use o ícone 𝚺 para criar tabelas resumidas (ou dinâmicas), para totais ou subtotais.", "relational": "relacionais", "Unpin to hide the the button while keeping the filter.": "Desfixe para ocultar o botão enquanto mantém o filtro." + }, + "DescriptionConfig": { + "DESCRIPTION": "DESCRIÇÃO" } } diff --git a/static/locales/ru.client.json b/static/locales/ru.client.json index 71fd3ab8..3780f271 100644 --- a/static/locales/ru.client.json +++ b/static/locales/ru.client.json @@ -820,7 +820,7 @@ "IT & Technology": "IT & Технология" }, "breadcrumbs": { - "fiddle": "ответвление", + "fiddle": "ветка", "override": "переопределение", "recovery mode": "режим восстановления", "snapshot": "снимок",