From e5e44c786ace256575a646994c2fa1bc3845e529 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 15:54:41 -0500 Subject: [PATCH] add a script for copying schema information from python to typescript There was no script for updating typescript schema information after a python-based document migration. Moving one in here, along with its test. Tweaked the code slightly to work with grist-core's directory structure. Also fixed a formatting error in mocha calls that was resulting in some root tests not running. --- buildtools/update_schema.sh | 48 ++++++++++++++++++++++++++++ package.json | 13 ++++---- sandbox/gen_js_schema.py | 5 ++- test/server/generateInitialDocSql.ts | 41 ++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 7 deletions(-) create mode 100755 buildtools/update_schema.sh create mode 100644 test/server/generateInitialDocSql.ts diff --git a/buildtools/update_schema.sh b/buildtools/update_schema.sh new file mode 100755 index 00000000..13635e11 --- /dev/null +++ b/buildtools/update_schema.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Regenerates typescript files with schema and sql for grist documents. +# This needs to run whenever the document schema is changed in the data +# engine, maintained in python code. It propagates the schema information +# to a typescript file, and updates SQL code for initializing new documents. +# +# To preview what it will do, call as: +# buildtools/update_schema.sh schema.ts sql.ts +# This will put schema.ts and sql.ts files in your working directory. +# Run without any arguments to modify application files. +# buildtools/update_schema.sh +# (you can see the differences with git diff if in a git repository). + +set -e + +schema_ts=$1 +sql_ts=$2 +if [[ -z "$schema_ts" ]]; then + # Default to regenerating regular suspects. + schema_ts=app/common/schema.ts + sql_ts=app/server/lib/initialDocSql.ts +fi +if [[ -z "$sql_ts" ]]; then + echo "Need both a schema and sql target" + exit 1 +fi + +# Prepare new version of schema file. +# Define custom python path locally, do not let it bleed over to node, since it +# could interfere with sandbox operation. +if [[ -e sandbox_venv3/bin/python ]]; then + # Use our virtual env if available. + PYTHON=sandbox_venv3/bin/python +else + # Fall back on system. + PYTHON=python +fi +PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp + +# Prepare new version of sql file. +export NODE_PATH=_build:_build/core:_build/ext +BUILD=$(test -e _build/core && echo "_build/core" || echo "_build") +node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp + +rm $sql_ts.tmpdoc.grist +mv $schema_ts.tmp $schema_ts +mv $sql_ts.tmp $sql_ts diff --git a/package.json b/package.json index 73f46645..f94a79a2 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,11 @@ "install:python3": "buildtools/prepare_python3.sh", "build:prod": "buildtools/build.sh", "start:prod": "sandbox/run.sh", - "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js", - "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 _build/test/nbrowser/**/*.js", - "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/client/**/*.js", - "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/common/**/*.js", - "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/server/**/*.js _build/test/gen-server/**/*.js", + "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", + "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 '_build/test/nbrowser/**/*.js'", + "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'", + "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'", + "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js", "test:docker": "./test/test_under_docker.sh", "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}", @@ -24,7 +24,8 @@ "lint": "eslint --cache --cache-strategy content .", "lint:fix": "eslint --cache --cache-strategy=content --fix .", "lint:ci": "eslint --max-warnings=0 .", - "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js" + "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js", + "generate:schema:ts": "buildtools/update_schema.sh" }, "keywords": [ "grist", diff --git a/sandbox/gen_js_schema.py b/sandbox/gen_js_schema.py index 0c4547b7..41ff6efb 100644 --- a/sandbox/gen_js_schema.py +++ b/sandbox/gen_js_schema.py @@ -32,7 +32,10 @@ import { GristObjCode } from "app/plugin/GristData"; export const SCHEMA_VERSION = %d; export const schema = { -""" % (__file__, schema.SCHEMA_VERSION)) +""" % ('core/sandbox/gen_js_schema.py', schema.SCHEMA_VERSION)) + # The script name is hardcoded since the Grist sandbox can be + # at different paths depending on how Grist is installed, and + # we don't want unnecessary changes to generated files. for table in schema.schema_create_actions(): print(' "%s": {' % table.table_id) diff --git a/test/server/generateInitialDocSql.ts b/test/server/generateInitialDocSql.ts new file mode 100644 index 00000000..9c8e17c9 --- /dev/null +++ b/test/server/generateInitialDocSql.ts @@ -0,0 +1,41 @@ +import { getAppRoot } from 'app/server/lib/places'; +import { createTmpDir } from 'test/server/docTools'; +import * as testUtils from 'test/server/testUtils'; + +import { assert } from 'chai'; +import * as childProcess from 'child_process'; +import * as fse from 'fs-extra'; +import * as path from 'path'; +import * as util from 'util'; + +const execFile = util.promisify(childProcess.execFile); + +describe('generateInitialDocSql', function() { + this.timeout(10000); + + let tmpDir: string; + + testUtils.setTmpLogLevel('fatal'); + + before(async function() { + tmpDir = await createTmpDir(); + }); + + it('confirms schema and sql files are up to date (run "./build schema" on failure)', async function() { + let root = getAppRoot(); + if (await fse.pathExists(path.join(root, 'core'))) { + root = path.join(root, 'core'); + } + const newSchemaTs = path.join(tmpDir, 'schema.ts'); + const newSqlTs = path.join(tmpDir, 'sql.ts'); + const currentSchemaTs = path.join(root, 'app/common/schema.ts'); + const currentSqlTs = path.join(root, 'app/server/lib/initialDocSql.ts'); + await execFile(path.join(getAppRoot(), 'buildtools/update_schema.sh'), [ + newSchemaTs, newSqlTs, + ], { env: process.env }); + assert.equal((await fse.readFile(newSchemaTs)).toString(), + (await fse.readFile(currentSchemaTs)).toString()); + assert.equal((await fse.readFile(newSqlTs)).toString(), + (await fse.readFile(currentSqlTs)).toString()); + }); +});