mirror of
https://github.com/gristlabs/grist-core.git
synced 2024-10-27 20:44:07 +00:00
(core) Generic tools for recording pycalls, deterministic mode.
Summary: Replaces https://phab.getgrist.com/D2854 Refactoring of NSandbox: - Simplify arguments to NSandbox.spawn. Only half the arguments were used depending on the flavour, adding a layer of confusion. - Ensure the same environment variables are passed to both flavours of sandbox - Simplify passing down environment variables. Implement deterministic mode with libfaketime and a seeded random instance. - Include static prebuilt libfaketime.so.1, may need another solution in future for other platforms. Recording pycalls: - Add script recordDocumentPyCalls.js to open a single document outside of tests. - Refactor out recordPyCalls.ts to support various uses. - Add afterEach hook to save all pycalls from server tests under $PYCALLS_DIR - Make docTools usable without mocha. - Add useLocalDoc and loadLocalDoc for loading non-fixture documents Test Plan: Made a document with formulas NOW() and UUID() Compare two document openings in normal mode: diff <(test/recordDocumentPyCalls.js samples/d4W6NrzCMNVSVD6nWgNrGC.grist /dev/stdout) \ <(test/recordDocumentPyCalls.js samples/d4W6NrzCMNVSVD6nWgNrGC.grist /dev/stdout) Output: < 1623407499.58132, --- > 1623407499.60376, 1195c1195 < "B": "bd2487f6-63c9-4f02-bbbc-5c0d674a2dc6" --- > "B": "22e1a4fd-297f-4b86-91a2-bc42cc6da4b2" `export DETERMINISTIC_MODE=1` and repeat. diff is empty! Reviewers: paulfitz Reviewed By: paulfitz Differential Revision: https://phab.getgrist.com/D2857
This commit is contained in:
Alex Hall
parent
f613b68a9e
commit
8a940676e9
2
app/server/declarations.d.ts
vendored
2
app/server/declarations.d.ts
vendored
@ -31,7 +31,7 @@ declare module "app/server/lib/shutdown" {
|
||||
export function addCleanupHandler<T>(context: T, method: (this: T) => void, timeout?: number, name?: string): void;
|
||||
export function removeCleanupHandlers<T>(context: T): void;
|
||||
export function cleanupOnSignals(...signalNames: string[]): void;
|
||||
export function exit(optExitCode?: number): void;
|
||||
export function exit(optExitCode?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// There is a @types/bluebird, but it's not great, and breaks for some of our usages.
|
||||
|
||||
@ -11,15 +11,10 @@ import {Throttle} from 'app/server/lib/Throttle';
|
||||
import {ChildProcess, spawn, SpawnOptions} from 'child_process';
|
||||
import * as path from 'path';
|
||||
import {Stream, Writable} from 'stream';
|
||||
import * as _ from 'lodash';
|
||||
|
||||
type SandboxMethod = (...args: any[]) => any;
|
||||
|
||||
export interface ISandboxCommand {
|
||||
process: string;
|
||||
libraryPath: string;
|
||||
docUrl: string;
|
||||
}
|
||||
|
||||
export interface ISandboxOptions {
|
||||
args: string[]; // The arguments to pass to the python process.
|
||||
exports?: {[name: string]: SandboxMethod}; // Functions made available to the sandboxed process.
|
||||
@ -29,13 +24,8 @@ export interface ISandboxOptions {
|
||||
selLdrArgs?: string[]; // Arguments passed to selLdr, for instance the following sets an
|
||||
// environment variable `{ ... selLdrArgs: ['-E', 'PYTHONPATH=grist'] ... }`.
|
||||
logMeta?: log.ILogMeta; // Log metadata (e.g. including docId) to report in all log messages.
|
||||
command?: ISandboxCommand;
|
||||
}
|
||||
|
||||
// Options for low-level spawning of selLdr sandbox process.
|
||||
export interface ISpawnOptions extends SpawnOptions {
|
||||
unsilenceLog?: boolean; // Don't silence the sel_ldr logging.
|
||||
command?: ISandboxCommand;
|
||||
command?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}
|
||||
|
||||
type ResolveRejectPair = [(value?: any) => void, (reason?: unknown) => void];
|
||||
@ -49,24 +39,30 @@ export class NSandbox implements ISandbox {
|
||||
* nacl/bin/run script, but without the reliance on bash. We can't use bash when -r/-w options
|
||||
* because on Windows it doesn't pass along the open file descriptors. Bash is also unavailable
|
||||
* when installing a standalone version on Windows.
|
||||
* @param selLdrArgs: Arguments to pass to sel_ldr;
|
||||
* @param pythonArgs: Arguments to pass to python within the sandbox.
|
||||
* @param spawnOptions: extra options for child_process.spawn(), such as 'stdio'.
|
||||
*/
|
||||
public static spawn(selLdrArgs: string[], pythonArgs: string[], spawnOptions: ISpawnOptions = {}): ChildProcess {
|
||||
const unsilenceLog = spawnOptions.unsilenceLog;
|
||||
delete spawnOptions.unsilenceLog;
|
||||
const command = spawnOptions.command;
|
||||
delete spawnOptions.command;
|
||||
|
||||
public static spawn(options: ISandboxOptions): ChildProcess {
|
||||
const {command, args: pythonArgs, unsilenceLog, env} = options;
|
||||
const spawnOptions: SpawnOptions = {
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
||||
env,
|
||||
};
|
||||
if (command) {
|
||||
return spawn(command.process, pythonArgs,
|
||||
{env: {PYTHONPATH: command.libraryPath, DOC_URL: command.docUrl},
|
||||
cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
|
||||
return spawn(command, pythonArgs,
|
||||
{cwd: path.join(process.cwd(), 'sandbox'), ...spawnOptions});
|
||||
}
|
||||
|
||||
const noLog = unsilenceLog ? [] :
|
||||
(process.env.OS === 'Windows_NT' ? ['-l', 'NUL'] : ['-l', '/dev/null']);
|
||||
// We use these options to set up communication with the sandbox:
|
||||
// -r 3:3 to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the
|
||||
// inside, for reading from the inside. This becomes `this._streamToSandbox`.
|
||||
// -w 4:4 to associate FD 4 on the outside with FD 4 on the inside for writing from the inside.
|
||||
// This becomes `this._streamFromSandbox`
|
||||
const selLdrArgs = ['-r', '3:3', '-w', '4:4', ...options.selLdrArgs || []];
|
||||
for (const [key, value] of _.toPairs(env)) {
|
||||
selLdrArgs.push("-E");
|
||||
selLdrArgs.push(`${key}=${value}`);
|
||||
}
|
||||
return spawn('sandbox/nacl/bin/sel_ldr', [
|
||||
'-B', './sandbox/nacl/lib/irt_core.nexe', '-m', './sandbox/nacl/root:/:ro',
|
||||
...noLog,
|
||||
@ -75,7 +71,7 @@ export class NSandbox implements ISandbox {
|
||||
'--library-path', '/slib', '/python/bin/python2.7.nexe',
|
||||
...pythonArgs
|
||||
],
|
||||
{env: {}, ...spawnOptions},
|
||||
spawnOptions,
|
||||
);
|
||||
}
|
||||
|
||||
@ -104,18 +100,7 @@ export class NSandbox implements ISandbox {
|
||||
this._logTimes = Boolean(options.logTimes || options.logCalls);
|
||||
this._exportedFunctions = options.exports || {};
|
||||
|
||||
const selLdrArgs = options.selLdrArgs || [];
|
||||
|
||||
// We use these options to set up communication with the sandbox:
|
||||
// -r 3:3 to associate a file descriptor 3 on the outside of the sandbox with FD 3 on the
|
||||
// inside, for reading from the inside. This becomes `this._streamToSandbox`.
|
||||
// -w 4:4 to associate FD 4 on the outside with FD 4 on the inside for writing from the inside.
|
||||
// This becomes `this._streamFromSandbox`
|
||||
this.childProc = NSandbox.spawn(['-r', '3:3', '-w', '4:4', ...selLdrArgs], options.args, {
|
||||
stdio: ['pipe', 'pipe', 'pipe', 'pipe', 'pipe'],
|
||||
unsilenceLog: options.unsilenceLog,
|
||||
command: options.command
|
||||
});
|
||||
this.childProc = NSandbox.spawn(options);
|
||||
|
||||
this._logMeta = {sandboxPid: this.childProc.pid, ...options.logMeta};
|
||||
log.rawDebug("Sandbox started", this._logMeta);
|
||||
@ -322,15 +307,9 @@ export class NSandboxCreator implements ISandboxCreator {
|
||||
}
|
||||
|
||||
public create(options: ISandboxCreationOptions): ISandbox {
|
||||
const pynbox = this._flavor === 'pynbox';
|
||||
// Main script to run.
|
||||
const defaultEntryPoint = this._flavor === 'pynbox' ? 'grist/main.pyc' : 'grist/main.py';
|
||||
// Python library path is only configurable when flavor is unsandboxed.
|
||||
// In this case, expect to find library files in a virtualenv built by core
|
||||
// buildtools/prepare_python.sh
|
||||
const pythonVersion = 'python2.7';
|
||||
const libraryPath =
|
||||
path.join(process.cwd(), 'sandbox', 'grist') + ':' +
|
||||
path.join(process.cwd(), 'venv', 'lib', pythonVersion, 'site-packages');
|
||||
const defaultEntryPoint = pynbox ? 'grist/main.pyc' : 'grist/main.py';
|
||||
const args = [options.entryPoint || defaultEntryPoint];
|
||||
if (!options.entryPoint && options.comment) {
|
||||
// When using default entry point, we can add on a comment as an argument - it isn't
|
||||
@ -343,30 +322,44 @@ export class NSandboxCreator implements ISandboxCreator {
|
||||
selLdrArgs.push(
|
||||
// TODO: Only modules that we share with plugins should be mounted. They could be gathered in
|
||||
// a "$APPROOT/sandbox/plugin" folder, only which get mounted.
|
||||
// TODO: These settings only make sense for pynbox flavor.
|
||||
'-E', 'PYTHONPATH=grist:thirdparty',
|
||||
'-m', `${options.sandboxMount}:/sandbox:ro`);
|
||||
}
|
||||
if (options.importMount) {
|
||||
selLdrArgs.push('-m', `${options.importMount}:/importdir:ro`);
|
||||
}
|
||||
const docUrl = (options.docUrl || '').replace(/[^-a-zA-Z0-9_:/?&.]/, '');
|
||||
if (this._flavor === 'pynbox') {
|
||||
selLdrArgs.push('-E', `DOC_URL=${docUrl}`);
|
||||
}
|
||||
const pythonVersion = 'python2.7';
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
// Python library path is only configurable when flavor is unsandboxed.
|
||||
// In this case, expect to find library files in a virtualenv built by core
|
||||
// buildtools/prepare_python.sh
|
||||
PYTHONPATH: pynbox ? 'grist:thirdparty' :
|
||||
path.join(process.cwd(), 'sandbox', 'grist') + ':' +
|
||||
path.join(process.cwd(), 'venv', 'lib', pythonVersion, 'site-packages'),
|
||||
|
||||
DOC_URL: (options.docUrl || '').replace(/[^-a-zA-Z0-9_:/?&.]/, ''),
|
||||
|
||||
// Making time and randomness act deterministically for testing purposes.
|
||||
// See test/utils/recordPyCalls.ts
|
||||
...(process.env.LIBFAKETIME_PATH ? { // path to compiled binary
|
||||
DETERMINISTIC_MODE: '1', // tells python to seed the random module
|
||||
FAKETIME: "2020-01-01 00:00:00", // setting for libfaketime
|
||||
|
||||
// For Linux
|
||||
LD_PRELOAD: process.env.LIBFAKETIME_PATH,
|
||||
|
||||
// For Mac (https://github.com/wolfcw/libfaketime/blob/master/README.OSX)
|
||||
DYLD_INSERT_LIBRARIES: process.env.LIBFAKETIME_PATH,
|
||||
DYLD_FORCE_FLAT_NAMESPACE: '1',
|
||||
} : {}),
|
||||
};
|
||||
return new NSandbox({
|
||||
args,
|
||||
logCalls: options.logCalls,
|
||||
logMeta: options.logMeta,
|
||||
logTimes: options.logTimes,
|
||||
selLdrArgs,
|
||||
...(this._flavor === 'pynbox' ? {} : {
|
||||
command: {
|
||||
process: pythonVersion,
|
||||
libraryPath,
|
||||
docUrl,
|
||||
}
|
||||
})
|
||||
env,
|
||||
...(pynbox ? {} : {command: pythonVersion}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ from __future__ import absolute_import
|
||||
import itertools
|
||||
import math as _math
|
||||
import operator
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
|
||||
@ -11,6 +12,10 @@ from functions.info import ISNUMBER, ISLOGICAL
|
||||
from functions.unimplemented import unimplemented
|
||||
import roman
|
||||
|
||||
if os.environ.get("DETERMINISTIC_MODE"):
|
||||
random.seed(1)
|
||||
|
||||
|
||||
# Iterates through elements of iterable arguments, or through individual args when not iterable.
|
||||
def _chain(*values_or_iterables):
|
||||
for v in values_or_iterables:
|
||||
|
||||
@ -260,19 +260,29 @@ exports.fixturesRoot = fixturesRoot;
|
||||
exports.appRoot = getAppRoot();
|
||||
|
||||
/**
|
||||
* Copy the given filename from the fixtures directory (test/fixtures) to the provided copyPath.
|
||||
* Copy the given filename from the fixtures directory (test/fixtures)
|
||||
* to the storage manager root.
|
||||
* @param {string} alias - Optional alias that lets you rename the document on disk.
|
||||
*/
|
||||
function useFixtureDoc(fileName, storageManager, alias = fileName) {
|
||||
var srcPath = path.resolve(fixturesRoot, "docs", fileName);
|
||||
var docName = path.basename(alias ? alias : fileName, ".grist");
|
||||
return docUtils.createNumbered(docName, "-",
|
||||
name => docUtils.createExclusive(storageManager.getPath(name))
|
||||
)
|
||||
.tap(docName => log.info("Using fixture %s as %s", fileName, docName + ".grist"))
|
||||
.tap(docName => docUtils.copyFile(srcPath, storageManager.getPath(docName)))
|
||||
.tap(docName => storageManager.markAsChanged(docName));
|
||||
return useLocalDoc(srcPath, storageManager, alias)
|
||||
.tap(docName => log.info("Using fixture %s as %s", fileName, docName + ".grist"))
|
||||
}
|
||||
exports.useFixtureDoc = useFixtureDoc;
|
||||
|
||||
/**
|
||||
* Copy the given filename from srcPath to the storage manager root.
|
||||
* @param {string} alias - Optional alias that lets you rename the document on disk.
|
||||
*/
|
||||
function useLocalDoc(srcPath, storageManager, alias = srcPath) {
|
||||
var docName = path.basename(alias || srcPath, ".grist");
|
||||
return docUtils.createNumbered(docName, "-",
|
||||
name => docUtils.createExclusive(storageManager.getPath(name))
|
||||
)
|
||||
.tap(docName => docUtils.copyFile(srcPath, storageManager.getPath(docName)))
|
||||
.tap(docName => storageManager.markAsChanged(docName));
|
||||
}
|
||||
exports.useLocalDoc = useLocalDoc;
|
||||
|
||||
exports.assert = assert;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user