add a script for copying schema information from python to typescript

There was no script for updating typescript schema information after
a python-based document migration. Moving one in here, along with its
test. Tweaked the code slightly to work with grist-core's directory
structure. Also fixed a formatting error in mocha calls that was resulting
in some root tests not running.

buildtools/update_schema.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Regenerates typescript files with schema and sql for grist documents.
+# This needs to run whenever the document schema is changed in the data
+# engine, maintained in python code. It propagates the schema information
+# to a typescript file, and updates SQL code for initializing new documents.
+#
+# To preview what it will do, call as:
+#   buildtools/update_schema.sh schema.ts sql.ts
+# This will put schema.ts and sql.ts files in your working directory.
+# Run without any arguments to modify application files.
+#   buildtools/update_schema.sh
+# (you can see the differences with git diff if in a git repository).
+
+set -e
+
+schema_ts=$1
+sql_ts=$2
+if [[ -z "$schema_ts" ]]; then
+  # Default to regenerating regular suspects.
+  schema_ts=app/common/schema.ts
+  sql_ts=app/server/lib/initialDocSql.ts
+fi
+if [[ -z "$sql_ts" ]]; then
+  echo "Need both a schema and sql target"
+  exit 1
+fi
+
+# Prepare new version of schema file.
+# Define custom python path locally, do not let it bleed over to node, since it
+# could interfere with sandbox operation.
+if [[ -e sandbox_venv3/bin/python ]]; then
+  # Use our virtual env if available.
+  PYTHON=sandbox_venv3/bin/python
+else
+  # Fall back on system.
+  PYTHON=python
+fi
+PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp
+
+# Prepare new version of sql file.
+export NODE_PATH=_build:_build/core:_build/ext
+BUILD=$(test -e _build/core && echo "_build/core" || echo "_build")
+node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp
+
+rm $sql_ts.tmpdoc.grist
+mv $schema_ts.tmp $schema_ts
+mv $sql_ts.tmp $sql_ts

package.json

@@ -12,11 +12,11 @@
     "install:python3": "buildtools/prepare_python3.sh",
     "build:prod": "buildtools/build.sh",
     "start:prod": "sandbox/run.sh",
-    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
-    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 _build/test/nbrowser/**/*.js",
-    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/client/**/*.js",
-    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/common/**/*.js",
-    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
+    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 '_build/test/nbrowser/**/*.js'",
+    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
+    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
+    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
     "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
     "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
@@ -24,7 +24,8 @@
     "lint": "eslint --cache --cache-strategy content .",
     "lint:fix": "eslint  --cache --cache-strategy=content --fix .",
     "lint:ci": "eslint --max-warnings=0 .",
-    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js"
+    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js",
+    "generate:schema:ts": "buildtools/update_schema.sh"
   },
   "keywords": [
     "grist",

sandbox/gen_js_schema.py

@@ -32,7 +32,10 @@ import { GristObjCode } from "app/plugin/GristData";
 export const SCHEMA_VERSION = %d;
 
 export const schema = {
-""" % (__file__, schema.SCHEMA_VERSION))
+""" % ('core/sandbox/gen_js_schema.py', schema.SCHEMA_VERSION))
+  # The script name is hardcoded since the Grist sandbox can be
+  # at different paths depending on how Grist is installed, and
+  # we don't want unnecessary changes to generated files.
 
   for table in schema.schema_create_actions():
     print('  "%s": {' % table.table_id)

test/server/generateInitialDocSql.ts

@@ -0,0 +1,41 @@
+import { getAppRoot } from 'app/server/lib/places';
+import { createTmpDir } from 'test/server/docTools';
+import * as testUtils from 'test/server/testUtils';
+
+import { assert } from 'chai';
+import * as childProcess from 'child_process';
+import * as fse from 'fs-extra';
+import * as path from 'path';
+import * as util from 'util';
+
+const execFile = util.promisify(childProcess.execFile);
+
+describe('generateInitialDocSql', function() {
+  this.timeout(10000);
+
+  let tmpDir: string;
+
+  testUtils.setTmpLogLevel('fatal');
+
+  before(async function() {
+    tmpDir = await createTmpDir();
+  });
+
+  it('confirms schema and sql files are up to date (run "./build schema" on failure)', async function() {
+    let root = getAppRoot();
+    if (await fse.pathExists(path.join(root, 'core'))) {
+      root = path.join(root, 'core');
+    }
+    const newSchemaTs = path.join(tmpDir, 'schema.ts');
+    const newSqlTs = path.join(tmpDir, 'sql.ts');
+    const currentSchemaTs = path.join(root, 'app/common/schema.ts');
+    const currentSqlTs = path.join(root, 'app/server/lib/initialDocSql.ts');
+    await execFile(path.join(getAppRoot(), 'buildtools/update_schema.sh'), [
+      newSchemaTs, newSqlTs,
+    ], { env: process.env });
+    assert.equal((await fse.readFile(newSchemaTs)).toString(),
+                 (await fse.readFile(currentSchemaTs)).toString());
+    assert.equal((await fse.readFile(newSqlTs)).toString(),
+                 (await fse.readFile(currentSqlTs)).toString());
+  });
+});