From e5e44c786ace256575a646994c2fa1bc3845e529 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 15:54:41 -0500 Subject: [PATCH 1/6] add a script for copying schema information from python to typescript There was no script for updating typescript schema information after a python-based document migration. Moving one in here, along with its test. Tweaked the code slightly to work with grist-core's directory structure. Also fixed a formatting error in mocha calls that was resulting in some root tests not running. --- buildtools/update_schema.sh | 48 ++++++++++++++++++++++++++++ package.json | 13 ++++---- sandbox/gen_js_schema.py | 5 ++- test/server/generateInitialDocSql.ts | 41 ++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 7 deletions(-) create mode 100755 buildtools/update_schema.sh create mode 100644 test/server/generateInitialDocSql.ts diff --git a/buildtools/update_schema.sh b/buildtools/update_schema.sh new file mode 100755 index 00000000..13635e11 --- /dev/null +++ b/buildtools/update_schema.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Regenerates typescript files with schema and sql for grist documents. +# This needs to run whenever the document schema is changed in the data +# engine, maintained in python code. It propagates the schema information +# to a typescript file, and updates SQL code for initializing new documents. +# +# To preview what it will do, call as: +# buildtools/update_schema.sh schema.ts sql.ts +# This will put schema.ts and sql.ts files in your working directory. +# Run without any arguments to modify application files. +# buildtools/update_schema.sh +# (you can see the differences with git diff if in a git repository). + +set -e + +schema_ts=$1 +sql_ts=$2 +if [[ -z "$schema_ts" ]]; then + # Default to regenerating regular suspects. + schema_ts=app/common/schema.ts + sql_ts=app/server/lib/initialDocSql.ts +fi +if [[ -z "$sql_ts" ]]; then + echo "Need both a schema and sql target" + exit 1 +fi + +# Prepare new version of schema file. +# Define custom python path locally, do not let it bleed over to node, since it +# could interfere with sandbox operation. +if [[ -e sandbox_venv3/bin/python ]]; then + # Use our virtual env if available. + PYTHON=sandbox_venv3/bin/python +else + # Fall back on system. + PYTHON=python +fi +PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp + +# Prepare new version of sql file. +export NODE_PATH=_build:_build/core:_build/ext +BUILD=$(test -e _build/core && echo "_build/core" || echo "_build") +node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp + +rm $sql_ts.tmpdoc.grist +mv $schema_ts.tmp $schema_ts +mv $sql_ts.tmp $sql_ts diff --git a/package.json b/package.json index 73f46645..f94a79a2 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,11 @@ "install:python3": "buildtools/prepare_python3.sh", "build:prod": "buildtools/build.sh", "start:prod": "sandbox/run.sh", - "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js", - "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 _build/test/nbrowser/**/*.js", - "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/client/**/*.js", - "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/common/**/*.js", - "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/server/**/*.js _build/test/gen-server/**/*.js", + "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", + "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 '_build/test/nbrowser/**/*.js'", + "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'", + "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'", + "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'", "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js", "test:docker": "./test/test_under_docker.sh", "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}", @@ -24,7 +24,8 @@ "lint": "eslint --cache --cache-strategy content .", "lint:fix": "eslint --cache --cache-strategy=content --fix .", "lint:ci": "eslint --max-warnings=0 .", - "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js" + "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js", + "generate:schema:ts": "buildtools/update_schema.sh" }, "keywords": [ "grist", diff --git a/sandbox/gen_js_schema.py b/sandbox/gen_js_schema.py index 0c4547b7..41ff6efb 100644 --- a/sandbox/gen_js_schema.py +++ b/sandbox/gen_js_schema.py @@ -32,7 +32,10 @@ import { GristObjCode } from "app/plugin/GristData"; export const SCHEMA_VERSION = %d; export const schema = { -""" % (__file__, schema.SCHEMA_VERSION)) +""" % ('core/sandbox/gen_js_schema.py', schema.SCHEMA_VERSION)) + # The script name is hardcoded since the Grist sandbox can be + # at different paths depending on how Grist is installed, and + # we don't want unnecessary changes to generated files. for table in schema.schema_create_actions(): print(' "%s": {' % table.table_id) diff --git a/test/server/generateInitialDocSql.ts b/test/server/generateInitialDocSql.ts new file mode 100644 index 00000000..9c8e17c9 --- /dev/null +++ b/test/server/generateInitialDocSql.ts @@ -0,0 +1,41 @@ +import { getAppRoot } from 'app/server/lib/places'; +import { createTmpDir } from 'test/server/docTools'; +import * as testUtils from 'test/server/testUtils'; + +import { assert } from 'chai'; +import * as childProcess from 'child_process'; +import * as fse from 'fs-extra'; +import * as path from 'path'; +import * as util from 'util'; + +const execFile = util.promisify(childProcess.execFile); + +describe('generateInitialDocSql', function() { + this.timeout(10000); + + let tmpDir: string; + + testUtils.setTmpLogLevel('fatal'); + + before(async function() { + tmpDir = await createTmpDir(); + }); + + it('confirms schema and sql files are up to date (run "./build schema" on failure)', async function() { + let root = getAppRoot(); + if (await fse.pathExists(path.join(root, 'core'))) { + root = path.join(root, 'core'); + } + const newSchemaTs = path.join(tmpDir, 'schema.ts'); + const newSqlTs = path.join(tmpDir, 'sql.ts'); + const currentSchemaTs = path.join(root, 'app/common/schema.ts'); + const currentSqlTs = path.join(root, 'app/server/lib/initialDocSql.ts'); + await execFile(path.join(getAppRoot(), 'buildtools/update_schema.sh'), [ + newSchemaTs, newSqlTs, + ], { env: process.env }); + assert.equal((await fse.readFile(newSchemaTs)).toString(), + (await fse.readFile(currentSchemaTs)).toString()); + assert.equal((await fse.readFile(newSqlTs)).toString(), + (await fse.readFile(currentSqlTs)).toString()); + }); +}); From 1b6b56bea184cbeb7037bf6192684922d9ddc9c1 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 15:59:09 -0500 Subject: [PATCH 2/6] update comment --- test/server/generateInitialDocSql.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/server/generateInitialDocSql.ts b/test/server/generateInitialDocSql.ts index 9c8e17c9..a7f4ec2c 100644 --- a/test/server/generateInitialDocSql.ts +++ b/test/server/generateInitialDocSql.ts @@ -21,7 +21,7 @@ describe('generateInitialDocSql', function() { tmpDir = await createTmpDir(); }); - it('confirms schema and sql files are up to date (run "./build schema" on failure)', async function() { + it('confirms schema and sql files are up to date (run "yarn run generate:schema:ts" on failure)', async function() { let root = getAppRoot(); if (await fse.pathExists(path.join(root, 'core'))) { root = path.join(root, 'core'); From 95351fb187d9cdbaff86a0af1480e85bf9c10f91 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 16:14:15 -0500 Subject: [PATCH 3/6] add test/upgradeDocument utility --- test/setupPaths.js | 7 ++++++ test/upgradeDocument | 4 ++++ test/upgradeDocumentImpl.ts | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 test/setupPaths.js create mode 100755 test/upgradeDocument create mode 100644 test/upgradeDocumentImpl.ts diff --git a/test/setupPaths.js b/test/setupPaths.js new file mode 100644 index 00000000..f579a663 --- /dev/null +++ b/test/setupPaths.js @@ -0,0 +1,7 @@ +// enhance require() to support project paths and typescript. +const path = require('path'); +const appModulePath = require('app-module-path'); +const root = path.dirname(__dirname); +appModulePath.addPath(path.join(root, "_build")); +appModulePath.addPath(path.join(root, "_build/core")); +appModulePath.addPath(path.join(root, "_build/ext")); diff --git a/test/upgradeDocument b/test/upgradeDocument new file mode 100755 index 00000000..be3f6cc3 --- /dev/null +++ b/test/upgradeDocument @@ -0,0 +1,4 @@ +#!/usr/bin/env node + +require('./setupPaths'); +require('test/upgradeDocumentImpl').main().catch(e => console.error(String(e))); diff --git a/test/upgradeDocumentImpl.ts b/test/upgradeDocumentImpl.ts new file mode 100644 index 00000000..499c28a0 --- /dev/null +++ b/test/upgradeDocumentImpl.ts @@ -0,0 +1,43 @@ +/** + * Upgrade one or more documents (both the DocStorage and schema migrations). + * + * Usage: + * test/upgradeDocument + */ +import {copyFile} from 'app/server/lib/docUtils'; +import {createDocTools} from 'test/server/docTools'; +import log from 'app/server/lib/log'; +import * as fs from "fs"; + +export async function main() { + const docPaths = process.argv.slice(2); + if (docPaths.length === 0) { + console.log(`Usage:\n test/upgradeDocument path/to/doc.grist ...\n`); + throw new Error("Document argument required"); + } + for (const docPath of docPaths) { + if (!docPath.endsWith('.grist')) { + throw new Error(`Document path should have .grist extension: ${docPath}`); + } + if (!fs.existsSync(docPath)) { + throw new Error(`Document path doesn't exist: ${docPath}`); + } + } + + const prevLogLevel = log.transports.file.level; + log.transports.file.level = 'warn'; + const docTools = createDocTools(); + await docTools.before(); + try { + for (const docPath of docPaths) { + console.log(`Upgrading ${docPath}`); + const activeDoc = await docTools.loadLocalDoc(docPath); + await activeDoc.waitForInitialization(); + await activeDoc.shutdown(); + await copyFile(docTools.getStorageManager().getPath(activeDoc.docName), docPath); + } + } finally { + await docTools.after(); + log.transports.file.level = prevLogLevel; + } +} From 6296423248f8f1df6d9ceddc1c1497d7b35621e5 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 16:16:36 -0500 Subject: [PATCH 4/6] update paths for grist-core layout --- buildtools/update_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/update_schema.sh b/buildtools/update_schema.sh index 13635e11..97d5737d 100755 --- a/buildtools/update_schema.sh +++ b/buildtools/update_schema.sh @@ -39,7 +39,7 @@ fi PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp # Prepare new version of sql file. -export NODE_PATH=_build:_build/core:_build/ext +export NODE_PATH=_build:_build/core:_build/stub:_build/ext BUILD=$(test -e _build/core && echo "_build/core" || echo "_build") node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp From 9d93c2e6c904fb93676ba0760e7b9ccd63452da9 Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Tue, 17 Jan 2023 17:09:59 -0500 Subject: [PATCH 5/6] stub -> stubs --- buildtools/update_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildtools/update_schema.sh b/buildtools/update_schema.sh index 97d5737d..5d5d4ddd 100755 --- a/buildtools/update_schema.sh +++ b/buildtools/update_schema.sh @@ -39,7 +39,7 @@ fi PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp # Prepare new version of sql file. -export NODE_PATH=_build:_build/core:_build/stub:_build/ext +export NODE_PATH=_build:_build/core:_build/stubs:_build/ext BUILD=$(test -e _build/core && echo "_build/core" || echo "_build") node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp From 2bfd8b42f64aad53bdcf5a6e01ec71a7f4d735cb Mon Sep 17 00:00:00 2001 From: Paul Fitzpatrick Date: Wed, 18 Jan 2023 12:19:23 -0500 Subject: [PATCH 6/6] suppress unneeded error message --- app/server/lib/ActiveDoc.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts index 82845766..d8beb8f3 100644 --- a/app/server/lib/ActiveDoc.ts +++ b/app/server/lib/ActiveDoc.ts @@ -1644,7 +1644,17 @@ export class ActiveDoc extends EventEmitter { const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds]; await this.applyUserActions(makeExceptionalDocSession('system'), [action]); } - await this.docStorage.removeUnusedAttachments(); + try { + await this.docStorage.removeUnusedAttachments(); + } catch (e) { + // If document doesn't have _gristsys_Files, don't worry about it; + // if this is an error it will have already been reported, and the + // document can be in this state when updating initial SQL code after + // a schema change. + if (!String(e).match(/no such table: _gristsys_Files/)) { + throw e; + } + } } // Needed for test/server/migrations.js tests