Merge pull request #403 from gristlabs/schema-script

add a script for copying schema information from python to typescript

app/server/lib/ActiveDoc.ts

@@ -1644,7 +1644,17 @@ export class ActiveDoc extends EventEmitter {
       const action: BulkRemoveRecord = ["BulkRemoveRecord", "_grist_Attachments", rowIds];
       await this.applyUserActions(makeExceptionalDocSession('system'), [action]);
     }
-    await this.docStorage.removeUnusedAttachments();
+    try {
+      await this.docStorage.removeUnusedAttachments();
+    } catch (e) {
+      // If document doesn't have _gristsys_Files, don't worry about it;
+      // if this is an error it will have already been reported, and the
+      // document can be in this state when updating initial SQL code after
+      // a schema change.
+      if (!String(e).match(/no such table: _gristsys_Files/)) {
+        throw e;
+      }
+    }
   }
 
   // Needed for test/server/migrations.js tests

buildtools/update_schema.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Regenerates typescript files with schema and sql for grist documents.
+# This needs to run whenever the document schema is changed in the data
+# engine, maintained in python code. It propagates the schema information
+# to a typescript file, and updates SQL code for initializing new documents.
+#
+# To preview what it will do, call as:
+#   buildtools/update_schema.sh schema.ts sql.ts
+# This will put schema.ts and sql.ts files in your working directory.
+# Run without any arguments to modify application files.
+#   buildtools/update_schema.sh
+# (you can see the differences with git diff if in a git repository).
+
+set -e
+
+schema_ts=$1
+sql_ts=$2
+if [[ -z "$schema_ts" ]]; then
+  # Default to regenerating regular suspects.
+  schema_ts=app/common/schema.ts
+  sql_ts=app/server/lib/initialDocSql.ts
+fi
+if [[ -z "$sql_ts" ]]; then
+  echo "Need both a schema and sql target"
+  exit 1
+fi
+
+# Prepare new version of schema file.
+# Define custom python path locally, do not let it bleed over to node, since it
+# could interfere with sandbox operation.
+if [[ -e sandbox_venv3/bin/python ]]; then
+  # Use our virtual env if available.
+  PYTHON=sandbox_venv3/bin/python
+else
+  # Fall back on system.
+  PYTHON=python
+fi
+PYTHONPATH=sandbox/grist:sandbox/thirdparty $PYTHON -B sandbox/gen_js_schema.py > $schema_ts.tmp
+
+# Prepare new version of sql file.
+export NODE_PATH=_build:_build/core:_build/stubs:_build/ext
+BUILD=$(test -e _build/core && echo "_build/core" || echo "_build")
+node $BUILD/app/server/generateInitialDocSql.js $sql_ts.tmpdoc > $sql_ts.tmp
+
+rm $sql_ts.tmpdoc.grist
+mv $schema_ts.tmp $schema_ts
+mv $sql_ts.tmp $sql_ts

package.json

@@ -12,11 +12,11 @@
     "install:python3": "buildtools/prepare_python3.sh",
     "build:prod": "buildtools/build.sh",
     "start:prod": "sandbox/run.sh",
-    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} _build/test/common/*.js _build/test/client/*.js _build/test/nbrowser/*.js _build/test/server/**/*.js _build/test/gen-server/**/*.js",
-    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 _build/test/nbrowser/**/*.js",
-    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/client/**/*.js",
-    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/common/**/*.js",
-    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} _build/test/server/**/*.js _build/test/gen-server/**/*.js",
+    "test": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} --slow 8000 ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} '_build/test/common/*.js' '_build/test/client/*.js' '_build/test/nbrowser/*.js' '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
+    "test:nbrowser": "GRIST_SESSION_COOKIE=grist_test_cookie GRIST_TEST_LOGIN=1 TEST_SUPPORT_API_KEY=api_key_for_support TEST_CLEAN_DATABASE=true NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+-b --no-exit} ${DEBUG:---forbid-only} -g ${GREP_TESTS:-''} --slow 8000 '_build/test/nbrowser/**/*.js'",
+    "test:client": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/client/**/*.js'",
+    "test:common": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/common/**/*.js'",
+    "test:server": "GRIST_SESSION_COOKIE=grist_test_cookie NODE_PATH=_build:_build/stubs:_build/ext mocha ${DEBUG:+'-b'} '_build/test/server/**/*.js' '_build/test/gen-server/**/*.js'",
     "test:smoke": "NODE_PATH=_build:_build/stubs:_build/ext mocha _build/test/nbrowser/Smoke.js",
     "test:docker": "./test/test_under_docker.sh",
     "test:python": "sandbox_venv3/bin/python sandbox/grist/runtests.py ${GREP_TESTS:+discover -p \"test*${GREP_TESTS}*.py\"}",
@@ -24,7 +24,8 @@
     "lint": "eslint --cache --cache-strategy content .",
     "lint:fix": "eslint  --cache --cache-strategy=content --fix .",
     "lint:ci": "eslint --max-warnings=0 .",
-    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js"
+    "generate:translation": "NODE_PATH=_build:_build/stubs:_build/ext node buildtools/generate_translation_keys.js",
+    "generate:schema:ts": "buildtools/update_schema.sh"
   },
   "keywords": [
     "grist",

sandbox/gen_js_schema.py

@@ -32,7 +32,10 @@ import { GristObjCode } from "app/plugin/GristData";
 export const SCHEMA_VERSION = %d;
 
 export const schema = {
-""" % (__file__, schema.SCHEMA_VERSION))
+""" % ('core/sandbox/gen_js_schema.py', schema.SCHEMA_VERSION))
+  # The script name is hardcoded since the Grist sandbox can be
+  # at different paths depending on how Grist is installed, and
+  # we don't want unnecessary changes to generated files.
 
   for table in schema.schema_create_actions():
     print('  "%s": {' % table.table_id)

test/server/generateInitialDocSql.ts

@@ -0,0 +1,41 @@
+import { getAppRoot } from 'app/server/lib/places';
+import { createTmpDir } from 'test/server/docTools';
+import * as testUtils from 'test/server/testUtils';
+
+import { assert } from 'chai';
+import * as childProcess from 'child_process';
+import * as fse from 'fs-extra';
+import * as path from 'path';
+import * as util from 'util';
+
+const execFile = util.promisify(childProcess.execFile);
+
+describe('generateInitialDocSql', function() {
+  this.timeout(10000);
+
+  let tmpDir: string;
+
+  testUtils.setTmpLogLevel('fatal');
+
+  before(async function() {
+    tmpDir = await createTmpDir();
+  });
+
+  it('confirms schema and sql files are up to date (run "yarn run generate:schema:ts" on failure)', async function() {
+    let root = getAppRoot();
+    if (await fse.pathExists(path.join(root, 'core'))) {
+      root = path.join(root, 'core');
+    }
+    const newSchemaTs = path.join(tmpDir, 'schema.ts');
+    const newSqlTs = path.join(tmpDir, 'sql.ts');
+    const currentSchemaTs = path.join(root, 'app/common/schema.ts');
+    const currentSqlTs = path.join(root, 'app/server/lib/initialDocSql.ts');
+    await execFile(path.join(getAppRoot(), 'buildtools/update_schema.sh'), [
+      newSchemaTs, newSqlTs,
+    ], { env: process.env });
+    assert.equal((await fse.readFile(newSchemaTs)).toString(),
+                 (await fse.readFile(currentSchemaTs)).toString());
+    assert.equal((await fse.readFile(newSqlTs)).toString(),
+                 (await fse.readFile(currentSqlTs)).toString());
+  });
+});

test/setupPaths.js

@@ -0,0 +1,7 @@
+// enhance require() to support project paths and typescript.
+const path = require('path');
+const appModulePath = require('app-module-path');
+const root = path.dirname(__dirname);
+appModulePath.addPath(path.join(root, "_build"));
+appModulePath.addPath(path.join(root, "_build/core"));
+appModulePath.addPath(path.join(root, "_build/ext"));

test/upgradeDocument

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require('./setupPaths');
+require('test/upgradeDocumentImpl').main().catch(e => console.error(String(e)));

test/upgradeDocumentImpl.ts

@@ -0,0 +1,43 @@
+/**
+ * Upgrade one or more documents (both the DocStorage and schema migrations).
+ *
+ * Usage:
+ *    test/upgradeDocument <docPaths...>
+ */
+import {copyFile} from 'app/server/lib/docUtils';
+import {createDocTools} from 'test/server/docTools';
+import log from 'app/server/lib/log';
+import * as fs from "fs";
+
+export async function main() {
+  const docPaths = process.argv.slice(2);
+  if (docPaths.length === 0) {
+    console.log(`Usage:\n    test/upgradeDocument path/to/doc.grist ...\n`);
+    throw new Error("Document argument required");
+  }
+  for (const docPath of docPaths) {
+    if (!docPath.endsWith('.grist')) {
+      throw new Error(`Document path should have .grist extension: ${docPath}`);
+    }
+    if (!fs.existsSync(docPath)) {
+      throw new Error(`Document path doesn't exist: ${docPath}`);
+    }
+  }
+
+  const prevLogLevel = log.transports.file.level;
+  log.transports.file.level = 'warn';
+  const docTools = createDocTools();
+  await docTools.before();
+  try {
+    for (const docPath of docPaths) {
+      console.log(`Upgrading ${docPath}`);
+      const activeDoc = await docTools.loadLocalDoc(docPath);
+      await activeDoc.waitForInitialization();
+      await activeDoc.shutdown();
+      await copyFile(docTools.getStorageManager().getPath(activeDoc.docName), docPath);
+    }
+  } finally {
+    await docTools.after();
+    log.transports.file.level = prevLogLevel;
+  }
+}